Files
mostovik-backend/src/apps/parsers/serializers.py
2026-10-05 17:33:33 +02:00

1295 lines
47 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Сериализаторы для приложения парсеров."""
import ipaddress
import socket
from urllib.parse import urlsplit
from apps.core.exceptions import ValidationError as APIValidationError
from apps.core.serializers import (
BackgroundJobListSerializer,
SnapshotRunMetadataSerializer,
)
from apps.parsers.clients.common.structured import (
MAX_FILE_SIZE_BYTES,
SUPPORTED_EXCEL_EXTENSIONS,
SUPPORTED_ZIP_EXTENSIONS,
)
from apps.parsers.fns_upload import FNS_XLSX_FILENAME_RE
from apps.parsers.models import (
FinancialReport,
FinancialReportLine,
GenericParserRecord,
IndustrialCertificateRecord,
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParserLoadLog,
ParserSourceArtifact,
ParsingSettings,
ProcurementRecord,
)
from apps.parsers.organization_enrichment import (
ORGANIZATION_ENRICHMENT_CONTEXT_KEY,
OrganizationFieldSet,
build_canonical_organization_enrichment,
enrich_organization_fields,
)
from apps.parsers.snapshot_metadata import get_snapshot_artifact, snapshot_metadata
from apps.parsers.source_cards import SOURCE_CARD_BY_SLUG
from drf_yasg.utils import swagger_serializer_method
from rest_framework import serializers
BLOCKED_FILE_HOSTS = {"localhost", "localhost.localdomain"}
BLOCKED_FILE_HOST_SUFFIXES = (".localhost", ".local", ".internal")
SUPPORTED_UPLOAD_EXTENSIONS = tuple(
sorted({*SUPPORTED_EXCEL_EXTENSIONS, *SUPPORTED_ZIP_EXTENSIONS, ".zip"})
)
def _is_blocked_ip(address: str) -> bool:
ip = ipaddress.ip_address(address)
return (
ip.is_loopback
or ip.is_private
or ip.is_link_local
or ip.is_multicast
or ip.is_reserved
or ip.is_unspecified
)
def _validate_public_file_host(host: str) -> None:
normalized_host = host.strip("[]").rstrip(".").lower()
if normalized_host in BLOCKED_FILE_HOSTS or normalized_host.endswith(
BLOCKED_FILE_HOST_SUFFIXES
):
raise serializers.ValidationError("file_url host is not allowed")
try:
if _is_blocked_ip(normalized_host):
raise serializers.ValidationError("file_url IP address is not allowed")
return
except ValueError:
pass
try:
address_info = socket.getaddrinfo(
normalized_host,
443,
type=socket.SOCK_STREAM,
)
except socket.gaierror as exc:
raise serializers.ValidationError("file_url host cannot be resolved") from exc
for item in address_info:
if _is_blocked_ip(item[4][0]):
raise serializers.ValidationError("file_url host resolves to private IP")
# =============================================================================
# Минпромторг - Сертификаты промышленного производства
# =============================================================================
class CanonicalOrganizationListSerializer(serializers.ListSerializer):
"""Adds one canonical organization lookup for the current serialized page."""
def to_representation(self, data):
items = list(data.all() if hasattr(data, "all") else data)
field_sets = getattr(self.child, "canonical_organization_fields", ())
self.context[
ORGANIZATION_ENRICHMENT_CONTEXT_KEY
] = build_canonical_organization_enrichment(items, field_sets)
return super().to_representation(items)
class CanonicalOrganizationEnrichmentMixin:
"""Fill blank v1 organization fields while preserving existing contract."""
canonical_organization_fields: tuple[OrganizationFieldSet, ...] = ()
def to_representation(self, instance):
data = super().to_representation(instance)
if not self.canonical_organization_fields:
return data
enrichment = self.context.get(ORGANIZATION_ENRICHMENT_CONTEXT_KEY)
if enrichment is None:
enrichment = build_canonical_organization_enrichment(
[instance],
self.canonical_organization_fields,
)
return enrich_organization_fields(
data,
source=instance,
field_sets=self.canonical_organization_fields,
enrichment=enrichment,
)
class IndustrialCertificateSerializer(
CanonicalOrganizationEnrichmentMixin,
serializers.ModelSerializer,
):
"""
Сертификат промышленного производства РФ.
Данные загружаются из Минпромторга.
"""
canonical_organization_fields = (
OrganizationFieldSet(
name="organisation_name",
inn="inn",
ogrn="ogrn",
),
)
class Meta:
model = IndustrialCertificateRecord
list_serializer_class = CanonicalOrganizationListSerializer
fields = [
"id",
"load_batch",
"issue_date",
"issue_date_normalized",
"certificate_number",
"expiry_date",
"expiry_date_normalized",
"certificate_file_url",
"organisation_name",
"inn",
"ogrn",
"registry_organization",
"created_at",
"updated_at",
]
read_only_fields = fields
# =============================================================================
# Минпромторг - Реестр производителей
# =============================================================================
class ManufacturerSerializer(
CanonicalOrganizationEnrichmentMixin,
serializers.ModelSerializer,
):
"""
Производитель из реестра Минпромторга.
Данные загружаются из Минпромторга.
"""
canonical_organization_fields = (
OrganizationFieldSet(
name="full_legal_name",
inn="inn",
ogrn="ogrn",
),
)
class Meta:
model = ManufacturerRecord
list_serializer_class = CanonicalOrganizationListSerializer
fields = [
"id",
"load_batch",
"full_legal_name",
"inn",
"ogrn",
"address",
"registry_organization",
"created_at",
"updated_at",
]
read_only_fields = fields
class IndustrialProductSerializer(
CanonicalOrganizationEnrichmentMixin,
serializers.ModelSerializer,
):
"""
Промышленная продукция из реестра Минпромторга.
Данные загружаются из Минпромторга.
"""
canonical_organization_fields = (
OrganizationFieldSet(
name="full_organisation_name",
inn="inn",
ogrn="ogrn",
),
)
class Meta:
model = IndustrialProductRecord
list_serializer_class = CanonicalOrganizationListSerializer
fields = [
"id",
"load_batch",
"full_organisation_name",
"ogrn",
"inn",
"registry_number",
"product_name",
"product_model",
"okpd2_code",
"tnved_code",
"regulatory_document",
"registry_organization",
"created_at",
"updated_at",
]
read_only_fields = fields
# =============================================================================
# Единый реестр проверок (proverki.gov.ru)
# =============================================================================
class InspectionSerializer(
CanonicalOrganizationEnrichmentMixin,
serializers.ModelSerializer,
):
"""
Проверка из Единого реестра проверок.
Поддерживает ФЗ-294 и ФЗ-248.
"""
canonical_organization_fields = (
OrganizationFieldSet(
name="organisation_name",
inn="inn",
ogrn="ogrn",
),
)
class Meta:
model = InspectionRecord
list_serializer_class = CanonicalOrganizationListSerializer
fields = [
"id",
"load_batch",
"registration_number",
"inn",
"ogrn",
"organisation_name",
"control_authority",
"inspection_type",
"inspection_form",
"start_date",
"start_date_normalized",
"end_date",
"end_date_normalized",
"status",
"legal_basis",
"result",
"is_federal_law_248",
"data_year",
"data_month",
"registry_organization",
"created_at",
"updated_at",
]
read_only_fields = fields
# =============================================================================
# Государственные закупки (zakupki.gov.ru)
# =============================================================================
class ProcurementSerializer(
CanonicalOrganizationEnrichmentMixin,
serializers.ModelSerializer,
):
"""
Государственная закупка из ЕИС zakupki.gov.ru.
Поддерживает 44-ФЗ и 223-ФЗ.
"""
canonical_organization_fields = (
OrganizationFieldSet(
name="customer_name",
inn="customer_inn",
kpp="customer_kpp",
ogrn="customer_ogrn",
),
)
class Meta:
model = ProcurementRecord
list_serializer_class = CanonicalOrganizationListSerializer
fields = [
"id",
"load_batch",
"purchase_number",
"purchase_name",
"customer_inn",
"customer_kpp",
"customer_ogrn",
"customer_name",
"max_price",
"max_price_amount",
"currency_code",
"placement_method",
"publish_date",
"publish_date_normalized",
"end_date",
"end_date_normalized",
"status",
"law_type",
"purchase_object_info",
"href",
"region_code",
"data_year",
"data_month",
"registry_organization",
"created_at",
"updated_at",
]
read_only_fields = fields
# =============================================================================
# ФНС - Бухгалтерская отчетность
# =============================================================================
class FinancialReportLineSerializer(serializers.ModelSerializer):
"""Строка финансового отчета."""
class Meta:
model = FinancialReportLine
fields = [
"id",
"form_code",
"line_code",
"line_name",
"year",
"period_start",
"period_end",
]
class FinancialReportSerializer(serializers.ModelSerializer):
"""
Финансовый отчет ФНС.
Данные загружаются из Excel файлов.
"""
lines_count = serializers.SerializerMethodField()
class Meta:
model = FinancialReport
fields = [
"id",
"external_id",
"ogrn",
"registry_organization",
"file_name",
"file_hash",
"load_batch",
"status",
"source",
"error_message",
"created_at",
"updated_at",
"lines_count",
]
read_only_fields = fields
def get_lines_count(self, obj) -> int:
return getattr(obj, "lines_count", obj.lines.count())
class FinancialReportDetailSerializer(FinancialReportSerializer):
"""Финансовый отчет с детализацией строк."""
lines = FinancialReportLineSerializer(many=True, read_only=True)
class Meta(FinancialReportSerializer.Meta):
fields = FinancialReportSerializer.Meta.fields + ["lines"]
class FNSFileUploadSerializer(serializers.Serializer):
"""
Сериализатор для загрузки файлов FNS.
Принимает список Excel файлов в формате fin_{id}_{ogrn}.xlsx
"""
file = serializers.FileField(
required=False,
help_text="Одиночный файл для загрузки (fin_*.xlsx)",
)
files = serializers.ListField(
child=serializers.FileField(),
required=False,
allow_empty=False,
help_text="Список файлов для загрузки (fin_*.xlsx)",
)
def validate(self, attrs):
files = attrs.get("files")
single_file = attrs.get("file")
if single_file and files:
raise serializers.ValidationError(
{"file": "Используйте либо file, либо files."}
)
if single_file:
files = [single_file]
if not files:
raise serializers.ValidationError(
{"file": "Нужно передать file или files."}
)
attrs["files"] = self._validate_uploaded_files(files)
return attrs
def _validate_uploaded_files(self, files):
"""Валидация файлов."""
for file in files:
if not FNS_XLSX_FILENAME_RE.match(file.name):
raise serializers.ValidationError(
f"Неверный формат имени файла: {file.name}. "
"Ожидается: fin_{{id}}_{{ogrn}}.xlsx"
)
return files
class FNSZipUploadSerializer(serializers.Serializer):
"""Сериализатор для загрузки ZIP архива с FNS Excel файлами."""
file = serializers.FileField(help_text="ZIP архив с файлами fin_*.xlsx")
def validate_file(self, value):
if not value.name.lower().endswith(".zip"):
raise serializers.ValidationError("Поддерживаются только ZIP архивы")
return value
class FNSServerPathUploadSerializer(serializers.Serializer):
"""Сериализатор для запуска обработки файла, уже лежащего на сервере."""
server_path = serializers.CharField(
help_text="Абсолютный путь к ZIP/XLSX внутри FNS watch directory",
)
class FNSFileUploadSuccessSerializer(serializers.Serializer):
"""Ответ одиночной загрузки FNS в формате frontend."""
success = serializers.BooleanField(read_only=True)
message = serializers.CharField(read_only=True)
class ParsingSettingsSerializer(serializers.ModelSerializer):
"""Настройки периодичности обновления источников парсинга."""
class Meta:
model = ParsingSettings
fields = [
"manufacturers_and_products",
"public_procurements",
"defense_unreliable_suppliers",
"planned_inspections",
"arbitration_cases",
"bankruptcy_procedures",
"information_security_registries",
]
class ParserSourceSerializer(serializers.Serializer):
"""Описание доступного parser source для dashboard/API."""
key = serializers.CharField()
source = serializers.ChoiceField(choices=ParserLoadLog.Source.choices)
title = serializers.CharField()
agency = serializers.CharField()
data_scope = serializers.CharField()
task_name = serializers.CharField()
is_existing = serializers.BooleanField()
requires_file_url = serializers.BooleanField()
mode = serializers.CharField()
status = serializers.CharField()
owner = serializers.CharField(allow_blank=True)
upstream_url = serializers.URLField(allow_blank=True)
access_method = serializers.CharField()
parser_strategy = serializers.CharField()
source_notes = serializers.CharField(allow_blank=True)
supports_file_upload = serializers.BooleanField()
api_route = serializers.CharField(allow_blank=True)
result_list_url = serializers.CharField()
result_detail_url = serializers.CharField()
upload_url = serializers.CharField(allow_blank=True)
supports_refresh = serializers.BooleanField()
admin_only = serializers.BooleanField()
class ParserRunRequestSerializer(serializers.Serializer):
"""Параметры запуска Celery-задачи парсера."""
params = serializers.DictField(
required=False,
help_text="For registry snapshot sources only: empty object; loader settings are server-controlled.",
)
mode = serializers.ChoiceField(
choices=("incremental", "full"),
required=False,
help_text="SRO parser only; defaults to incremental.",
)
file_url = serializers.URLField(required=False, allow_blank=True)
file_path = serializers.CharField(required=False, allow_blank=True)
proxies = serializers.ListField(
child=serializers.CharField(),
required=False,
allow_empty=True,
)
vacancy_sources = serializers.ListField(
child=serializers.ChoiceField(choices=["trudvsem"]),
required=False,
allow_empty=False,
)
registry_organizations_only = serializers.BooleanField(required=False)
registry_organization_limit = serializers.IntegerField(required=False, min_value=1)
year = serializers.IntegerField(required=False, min_value=2000, max_value=2100)
month = serializers.IntegerField(required=False, min_value=1, max_value=12)
limit = serializers.IntegerField(required=False, min_value=1, max_value=1000)
offset = serializers.IntegerField(required=False, min_value=0)
max_months_per_law = serializers.IntegerField(
required=False,
min_value=1,
max_value=36,
)
start_year = serializers.IntegerField(
required=False,
min_value=2000,
max_value=2100,
)
start_month = serializers.IntegerField(required=False, min_value=1, max_value=12)
include_fz294 = serializers.BooleanField(required=False)
include_fz248 = serializers.BooleanField(required=False)
current_year = serializers.IntegerField(
required=False,
min_value=2000,
max_value=2100,
)
current_month = serializers.IntegerField(required=False, min_value=1, max_value=12)
region_code = serializers.CharField(required=False, allow_blank=True)
law_type = serializers.CharField(required=False, allow_blank=True)
company_inn = serializers.CharField(required=False, allow_blank=True)
text = serializers.CharField(required=False, allow_blank=True)
def validate_file_url(self, value: str) -> str:
"""Разрешить worker-download только с публичных HTTPS URL."""
if not value:
return value
parsed = urlsplit(value)
if parsed.scheme != "https":
raise serializers.ValidationError("file_url must use https")
if parsed.username or parsed.password:
raise serializers.ValidationError("file_url credentials are not allowed")
if not parsed.hostname:
raise serializers.ValidationError("file_url host is required")
_validate_public_file_host(parsed.hostname)
return value
def validate(self, attrs):
attrs = super().validate(attrs)
if ("start_year" in attrs) != ("start_month" in attrs):
raise serializers.ValidationError(
{
"start_month": (
"start_year and start_month must be provided together"
)
}
)
return attrs
class ParserUploadRequestSerializer(serializers.Serializer):
"""Файл ручной загрузки реестра или финансовой выгрузки."""
file = serializers.FileField()
def validate_file(self, value):
file_name = (value.name or "").lower()
if not file_name.endswith(SUPPORTED_UPLOAD_EXTENSIONS):
extensions = ", ".join(SUPPORTED_UPLOAD_EXTENSIONS)
raise serializers.ValidationError(
f"Unsupported file extension. Allowed: {extensions}"
)
if value.size > MAX_FILE_SIZE_BYTES:
raise serializers.ValidationError(
f"File exceeds size limit: {MAX_FILE_SIZE_BYTES} bytes"
)
return value
class ParserScheduleRequestSerializer(ParserRunRequestSerializer):
"""Параметры создания/обновления периодической Celery-задачи парсера."""
SCHEDULE_TYPES = ("interval", "crontab")
PERIODS = ("seconds", "minutes", "hours", "days")
source_key = serializers.CharField(required=False)
name = serializers.CharField(required=False, allow_blank=True, max_length=200)
enabled = serializers.BooleanField(required=False, default=True)
schedule_type = serializers.ChoiceField(
choices=SCHEDULE_TYPES,
required=False,
default="interval",
)
every = serializers.IntegerField(required=False, min_value=1)
period = serializers.ChoiceField(choices=PERIODS, required=False, default="hours")
minute = serializers.CharField(required=False, allow_blank=True, default="0")
hour = serializers.CharField(required=False, allow_blank=True, default="*")
day_of_week = serializers.CharField(required=False, allow_blank=True, default="*")
day_of_month = serializers.CharField(required=False, allow_blank=True, default="*")
month_of_year = serializers.CharField(required=False, allow_blank=True, default="*")
def validate(self, attrs):
attrs = super().validate(attrs)
if attrs.get("schedule_type", "interval") == "interval" and not attrs.get(
"every"
):
raise serializers.ValidationError(
{"every": "Required for interval schedule"}
)
return attrs
class ParserScheduleSerializer(serializers.Serializer):
"""Описание периодической задачи парсера."""
id = serializers.IntegerField()
name = serializers.CharField()
source_key = serializers.CharField()
source = serializers.CharField()
title = serializers.CharField()
task_name = serializers.CharField()
enabled = serializers.BooleanField()
schedule_type = serializers.CharField()
schedule = serializers.DictField()
params = serializers.DictField()
last_run_at = serializers.DateTimeField(allow_null=True)
total_run_count = serializers.IntegerField()
date_changed = serializers.DateTimeField()
class ParserListQuerySerializer(serializers.Serializer):
"""Общие query-параметры списков парсеров."""
limit = serializers.IntegerField(
required=False,
default=50,
min_value=1,
max_value=200,
)
class ParserResultQuerySerializer(serializers.Serializer):
"""Query-параметры per-source result endpoints."""
page = serializers.IntegerField(required=False, default=1, min_value=1)
page_size = serializers.IntegerField(
required=False,
default=20,
min_value=1,
max_value=100,
)
limit = serializers.IntegerField(required=False, min_value=1, max_value=100)
id = serializers.IntegerField(required=False, min_value=1)
source = serializers.CharField(required=False, allow_blank=True)
external_id = serializers.CharField(required=False, allow_blank=True)
inn = serializers.CharField(required=False, allow_blank=True)
ogrn = serializers.CharField(required=False, allow_blank=True)
load_batch = serializers.IntegerField(required=False, min_value=1)
batch_id = serializers.IntegerField(required=False, min_value=1)
status = serializers.CharField(required=False, allow_blank=True)
record_date = serializers.CharField(required=False, allow_blank=True)
date_from = serializers.DateField(required=False)
date_to = serializers.DateField(required=False)
search = serializers.CharField(required=False, allow_blank=True)
ordering = serializers.CharField(required=False, allow_blank=True)
include_payload = serializers.BooleanField(required=False, default=True)
def validate(self, attrs):
attrs = super().validate(attrs)
if attrs.get("limit"):
attrs["page"] = 1
attrs["page_size"] = attrs["limit"]
if attrs.get("batch_id") and not attrs.get("load_batch"):
attrs["load_batch"] = attrs["batch_id"]
if (
attrs.get("date_from")
and attrs.get("date_to")
and attrs["date_from"] > attrs["date_to"]
):
raise serializers.ValidationError(
{"date_to": "date_to must be greater than or equal to date_from"}
)
return attrs
class ParserRunResponseSerializer(serializers.Serializer):
"""Ответ API на запуск задачи."""
task_id = serializers.CharField()
task_ids = serializers.ListField(child=serializers.CharField())
source = serializers.CharField()
task_name = serializers.CharField()
status = serializers.ChoiceField(
choices=("queued", "pending", "started", "running", "skipped"), required=False
)
class IntegratedStructureUploadResponseSerializer(serializers.Serializer):
status = serializers.ChoiceField(choices=("published", "skipped"))
artifact_id = serializers.UUIDField()
task_id = serializers.CharField()
batch_id = serializers.IntegerField()
records_count = serializers.IntegerField()
structures_count = serializers.IntegerField()
event_rows_count = serializers.IntegerField()
document_link_rows_count = serializers.IntegerField()
unique_documents_count = serializers.IntegerField()
events_without_documents_count = serializers.IntegerField()
reason = serializers.CharField(required=False)
# =============================================================================
# Служебные модели
# =============================================================================
class SnapshotMetadataSerializer(serializers.Serializer):
"""Provenance of the exact artifact published by this load."""
artifact_id = serializers.UUIDField(read_only=True)
snapshot_date = serializers.DateField(read_only=True, allow_null=True)
version = serializers.CharField(read_only=True, allow_blank=True)
checksum_sha256 = serializers.CharField(read_only=True, allow_blank=True)
published_at = serializers.DateTimeField(read_only=True, allow_null=True)
class ParserLoadLogSerializer(serializers.ModelSerializer):
"""
Лог загрузки парсера.
Информация о каждой загрузке данных из внешнего источника.
"""
source_display = serializers.CharField(source="get_source_display", read_only=True)
source_label = serializers.CharField(source="get_source_display", read_only=True)
snapshot = serializers.SerializerMethodField()
meta = serializers.SerializerMethodField()
organizations_count = serializers.SerializerMethodField()
artifact_uid = serializers.SerializerMethodField()
parsed_count = serializers.SerializerMethodField()
published_count = serializers.SerializerMethodField()
quarantined_count = serializers.SerializerMethodField()
rejection_reasons = serializers.SerializerMethodField()
class Meta:
model = ParserLoadLog
fields = [
"id",
"batch_id",
"source",
"source_display",
"source_label",
"snapshot",
"meta",
"records_count",
"organizations_count",
"artifact_uid",
"parsed_count",
"published_count",
"quarantined_count",
"rejection_reasons",
"status",
"error_message",
"created_at",
"updated_at",
]
read_only_fields = fields
def get_organizations_count(self, obj) -> int:
artifact = self._get_artifact(obj)
if artifact is not None:
return artifact.published_organizations_count
if obj.source == ParserLoadLog.Source.FNS_REPORTS:
return (
FinancialReport.objects.filter(load_batch=obj.batch_id)
.exclude(ogrn="")
.values("ogrn")
.distinct()
.count()
)
if obj.source == ParserLoadLog.Source.INDUSTRIAL:
return (
IndustrialCertificateRecord.objects.filter(load_batch=obj.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if obj.source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS:
return (
IndustrialProductRecord.objects.filter(load_batch=obj.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if obj.source == ParserLoadLog.Source.MANUFACTURES:
return (
ManufacturerRecord.objects.filter(load_batch=obj.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if obj.source == ParserLoadLog.Source.INSPECTIONS:
return (
InspectionRecord.objects.filter(load_batch=obj.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if obj.source == ParserLoadLog.Source.PROCUREMENTS:
return (
ProcurementRecord.objects.filter(load_batch=obj.batch_id)
.exclude(customer_inn="")
.values("customer_inn")
.distinct()
.count()
)
return 0
@swagger_serializer_method(
serializer_or_field=SnapshotRunMetadataSerializer(allow_null=True)
)
def get_meta(self, obj):
artifact = self._get_artifact(obj)
return (
SnapshotRunMetadataSerializer(artifact.metadata).data
if artifact is not None
else None
)
@swagger_serializer_method(
serializer_or_field=SnapshotMetadataSerializer(allow_null=True)
)
def get_snapshot(self, obj):
metadata = snapshot_metadata(obj)
return (
SnapshotMetadataSerializer(metadata).data if metadata is not None else None
)
@staticmethod
def _get_artifact(obj) -> ParserSourceArtifact | None:
return get_snapshot_artifact(obj)
def get_artifact_uid(self, obj) -> str | None:
artifact = self._get_artifact(obj)
return str(artifact.uid) if artifact is not None else None
def get_parsed_count(self, obj) -> int:
artifact = self._get_artifact(obj)
return artifact.parsed_count if artifact is not None else obj.records_count
def get_published_count(self, obj) -> int:
artifact = self._get_artifact(obj)
return artifact.published_count if artifact is not None else obj.records_count
def get_quarantined_count(self, obj) -> int:
artifact = self._get_artifact(obj)
return artifact.quarantined_count if artifact is not None else 0
def get_rejection_reasons(self, obj) -> dict:
artifact = self._get_artifact(obj)
return dict(artifact.rejection_reasons) if artifact is not None else {}
class GenericParserRecordSerializer(
CanonicalOrganizationEnrichmentMixin,
serializers.ModelSerializer,
):
"""Сериализатор универсальных записей новых источников."""
canonical_organization_fields = (
OrganizationFieldSet(
name="organisation_name",
inn="inn",
ogrn="ogrn",
),
)
class Meta:
model = GenericParserRecord
list_serializer_class = CanonicalOrganizationListSerializer
fields = [
"id",
"load_batch",
"source",
"external_id",
"inn",
"ogrn",
"organisation_name",
"title",
"record_date",
"amount",
"status",
"url",
"payload",
"registry_organization",
"created_at",
"updated_at",
]
read_only_fields = fields
class ParserResultRecordSerializer(serializers.Serializer):
"""Унифицированная запись результата конкретного источника."""
id = serializers.IntegerField()
load_batch = serializers.IntegerField()
source = serializers.CharField()
vacancy_source = serializers.CharField(allow_blank=True, required=False)
external_id = serializers.CharField(allow_blank=True)
inn = serializers.CharField(allow_blank=True)
ogrn = serializers.CharField(allow_blank=True)
organisation_name = serializers.CharField(allow_blank=True)
title = serializers.CharField(allow_blank=True)
record_date = serializers.CharField(allow_blank=True)
amount = serializers.DecimalField(
max_digits=20,
decimal_places=2,
allow_null=True,
)
status = serializers.CharField(allow_blank=True)
url = serializers.CharField(allow_blank=True)
payload = serializers.DictField()
created_at = serializers.DateTimeField()
updated_at = serializers.DateTimeField()
class ParserLoadLogListSerializer(serializers.Serializer):
"""Строка списка логов в frontend-friendly формате."""
id = serializers.IntegerField(read_only=True)
batch_id = serializers.IntegerField(read_only=True)
source = serializers.ChoiceField(
choices=[*ParserLoadLog.Source.values, *SOURCE_CARD_BY_SLUG], read_only=True
)
source_label = serializers.CharField(read_only=True, allow_null=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
artifact_uid = serializers.UUIDField(read_only=True, allow_null=True)
meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True)
snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True)
parsed_count = serializers.IntegerField(read_only=True)
published_count = serializers.IntegerField(read_only=True)
quarantined_count = serializers.IntegerField(read_only=True)
rejection_reasons = serializers.DictField(read_only=True)
status = serializers.ChoiceField(
choices=ParserLoadLog.Status.choices, read_only=True
)
status_label = serializers.CharField(read_only=True)
error_message = serializers.CharField(read_only=True, allow_blank=True)
created_at = serializers.DateTimeField(read_only=True)
updated_at = serializers.DateTimeField(read_only=True)
class ParserLoadLogPageSerializer(serializers.Serializer):
"""Пагинированный ответ списка логов для frontend."""
count = serializers.IntegerField(read_only=True)
next = serializers.CharField(read_only=True, allow_null=True)
previous = serializers.CharField(read_only=True, allow_null=True)
results = ParserLoadLogListSerializer(many=True, read_only=True)
class SourceCardRefreshParamSerializer(serializers.Serializer):
"""Описание параметра ручного обновления карточки источника."""
name = serializers.CharField(read_only=True)
label = serializers.CharField(read_only=True)
description = serializers.CharField(read_only=True)
required = serializers.BooleanField(read_only=True)
type = serializers.CharField(read_only=True)
default = serializers.JSONField(read_only=True)
class SourceCardLoadSerializer(serializers.Serializer):
"""Сводка по последней загрузке источника."""
meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True)
snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True)
batch_id = serializers.IntegerField(read_only=True)
source = serializers.ChoiceField(
choices=ParserLoadLog.Source.choices, read_only=True
)
source_display = serializers.CharField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
source_label = serializers.CharField(read_only=True)
records_count = serializers.IntegerField(read_only=True)
status = serializers.ChoiceField(
choices=ParserLoadLog.Status.choices, read_only=True
)
error_message = serializers.CharField(read_only=True)
created_at = serializers.DateTimeField(read_only=True)
updated_at = serializers.DateTimeField(read_only=True)
class SourceCardTaskSerializer(serializers.Serializer):
"""Краткая информация по фоновой задаче карточки."""
task_id = serializers.CharField(read_only=True)
task_name = serializers.CharField(read_only=True)
status = serializers.CharField(read_only=True)
progress = serializers.IntegerField(read_only=True)
progress_message = serializers.CharField(read_only=True)
started_at = serializers.DateTimeField(read_only=True, allow_null=True)
created_at = serializers.DateTimeField(read_only=True)
meta = serializers.JSONField(read_only=True)
class SourceCardItemSerializer(serializers.Serializer):
"""Подисточник внутри агрегированной карточки."""
code = serializers.CharField(read_only=True)
refresh_key = serializers.CharField(read_only=True)
title = serializers.CharField(read_only=True)
description = serializers.CharField(read_only=True)
parser_source = serializers.CharField(read_only=True, allow_null=True)
parser_source_display = serializers.CharField(read_only=True, allow_null=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True)
upload_url = serializers.CharField(read_only=True, allow_blank=True)
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
class SourceCardSerializer(serializers.Serializer):
"""Serializer for frontend source cards list."""
slug = serializers.CharField(read_only=True)
title = serializers.CharField(read_only=True)
description = serializers.CharField(read_only=True)
order = serializers.IntegerField(read_only=True)
is_available = serializers.BooleanField(read_only=True)
status = serializers.CharField(read_only=True)
status_label = serializers.CharField(read_only=True)
progress = serializers.IntegerField(read_only=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True)
next_update_at = serializers.DateTimeField(read_only=True, allow_null=True)
error_message = serializers.CharField(read_only=True)
task_names = serializers.ListField(child=serializers.CharField(), read_only=True)
refresh_requires_params = serializers.BooleanField(read_only=True)
supports_refresh = serializers.BooleanField(read_only=True)
upload_url = serializers.CharField(read_only=True, allow_blank=True)
refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True)
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
source_items = SourceCardItemSerializer(many=True, read_only=True)
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
class SourceTaskStatusSerializer(serializers.Serializer):
"""Табличная строка статуса источника для экрана задач парсинга."""
row_number = serializers.IntegerField(read_only=True)
slug = serializers.CharField(read_only=True)
source = serializers.CharField(read_only=True)
status = serializers.CharField(read_only=True)
status_label = serializers.CharField(read_only=True)
actualized_at = serializers.DateTimeField(read_only=True, allow_null=True)
next_update_at = serializers.DateTimeField(read_only=True, allow_null=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
progress = serializers.IntegerField(read_only=True)
error_message = serializers.CharField(read_only=True)
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
class SourceCardDetailSerializer(SourceCardSerializer):
"""List and detail deliberately share the complete card contract."""
class SourceCardRefreshRequestSerializer(serializers.Serializer):
"""Request body for manual card refresh."""
params = serializers.DictField(required=False)
def to_internal_value(self, data):
if isinstance(data, dict) and set(data) - set(self.fields):
raise serializers.ValidationError({"detail": "Only params is supported."})
return super().to_internal_value(data)
class SroParserRunRequestSerializer(SourceCardRefreshRequestSerializer):
mode = serializers.ChoiceField(
choices=("incremental", "full"), required=False, default="incremental"
)
def to_internal_value(self, data):
if isinstance(data, dict) and data.get("mode", "incremental") not in (
"incremental",
"full",
):
raise APIValidationError(
message="Допустимые режимы: incremental, full.", code="invalid_mode"
)
return super().to_internal_value(data)
class FedresursParserRunRequestSerializer(SourceCardRefreshRequestSerializer):
mode = serializers.ChoiceField(
choices=("full", "targeted"), required=False, default="full"
)
organization_uids = serializers.ListField(
child=serializers.UUIDField(), required=False, max_length=500
)
def validate(self, attrs):
mode = attrs.get("mode", "full")
uids = attrs.get("organization_uids")
if (mode == "targeted") != bool(uids):
raise serializers.ValidationError(
{"organization_uids": "Required only for targeted refresh."}
)
attrs["params"] = {
**attrs.get("params", {}),
**({"organization_uids": [str(uid) for uid in uids]} if uids else {}),
}
return attrs
class SourceCardRefreshTaskSerializer(serializers.Serializer):
"""Queued task info after refresh start."""
task_id = serializers.CharField(read_only=True)
task_name = serializers.CharField(read_only=True)
class SourceCardRefreshResponseSerializer(serializers.Serializer):
"""Response serializer for manual card refresh."""
source_card = serializers.CharField(read_only=True)
status = serializers.CharField(read_only=True)
requested_at = serializers.DateTimeField(read_only=True)
tasks = SourceCardRefreshTaskSerializer(many=True, read_only=True)
class FrontendApiResponseSerializer(serializers.Serializer):
"""Общая форма success/data/errors/meta для frontend sources API."""
success = serializers.BooleanField(read_only=True)
errors = serializers.JSONField(read_only=True, allow_null=True)
meta = serializers.JSONField(read_only=True, allow_null=True)
class ParserRunEnvelopeSerializer(FrontendApiResponseSerializer):
"""Pollable parser task inside the standard API envelope."""
data = ParserRunResponseSerializer(read_only=True)
class SourceCardListResponseSerializer(FrontendApiResponseSerializer):
"""Envelope для списка карточек источников."""
data = SourceCardSerializer(many=True, read_only=True)
class SourceTaskStatusListResponseSerializer(FrontendApiResponseSerializer):
"""Envelope для списка статусов источников."""
data = SourceTaskStatusSerializer(many=True, read_only=True)
class SourceCardDetailResponseSerializer(FrontendApiResponseSerializer):
"""Envelope для детальной карточки источника."""
data = SourceCardDetailSerializer(read_only=True)
class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
"""Минимальный ответ запуска обновления карточки по md."""
task_id = serializers.CharField(read_only=True, allow_null=True)
task_ids = serializers.ListField(child=serializers.CharField(), read_only=True)
status = serializers.ChoiceField(choices=("queued", "accepted"), read_only=True)
class SourceRefreshErrorItemSerializer(serializers.Serializer):
code = serializers.CharField(read_only=True)
message = serializers.CharField(read_only=True)
details = serializers.JSONField(read_only=True, required=False)
class SourceRefreshErrorSerializer(serializers.Serializer):
success = serializers.BooleanField(read_only=True)
data = serializers.JSONField(read_only=True, allow_null=True)
errors = SourceRefreshErrorItemSerializer(many=True, read_only=True)
meta = serializers.DictField(read_only=True, allow_null=True)
class SourceRefreshConflictItemSerializer(serializers.Serializer):
code = serializers.ChoiceField(
choices=("refresh_already_running", "upstream_access_not_approved"),
read_only=True,
)
message = serializers.CharField(read_only=True)
class SourceRefreshConflictSerializer(serializers.Serializer):
success = serializers.BooleanField(read_only=True)
data = serializers.JSONField(read_only=True, allow_null=True)
errors = SourceRefreshConflictItemSerializer(many=True, read_only=True)
meta = serializers.DictField(read_only=True, allow_null=True)
class ParserDashboardGroupsSerializer(serializers.Serializer):
api = ParserSourceSerializer(many=True, read_only=True)
uploads = ParserSourceSerializer(many=True, read_only=True)
class ParserDashboardSerializer(serializers.Serializer):
sources = ParserSourceSerializer(many=True, read_only=True)
api_sources = ParserSourceSerializer(many=True, read_only=True)
file_sources = ParserSourceSerializer(many=True, read_only=True)
groups = ParserDashboardGroupsSerializer(read_only=True)
schedules = ParserScheduleSerializer(many=True, read_only=True)
jobs = BackgroundJobListSerializer(many=True, read_only=True)
source_counts = serializers.DictField(
child=serializers.IntegerField(), read_only=True
)
registry_data_coverage = serializers.DictField(read_only=True)
registry_enrichment_analytics = serializers.DictField(read_only=True)
load_logs = ParserLoadLogSerializer(many=True, read_only=True)
class ParserDashboardResponseSerializer(FrontendApiResponseSerializer):
data = ParserDashboardSerializer(read_only=True)