"""Сериализаторы для приложения парсеров.""" import ipaddress import socket from urllib.parse import urlsplit from apps.core.exceptions import ValidationError as APIValidationError from apps.core.serializers import ( BackgroundJobListSerializer, SnapshotRunMetadataSerializer, ) from apps.parsers.clients.common.structured import ( MAX_FILE_SIZE_BYTES, SUPPORTED_EXCEL_EXTENSIONS, SUPPORTED_ZIP_EXTENSIONS, ) from apps.parsers.fns_upload import FNS_XLSX_FILENAME_RE from apps.parsers.models import ( FinancialReport, FinancialReportLine, GenericParserRecord, IndustrialCertificateRecord, IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, ParserSourceArtifact, ParsingSettings, ProcurementRecord, ) from apps.parsers.organization_enrichment import ( ORGANIZATION_ENRICHMENT_CONTEXT_KEY, OrganizationFieldSet, build_canonical_organization_enrichment, enrich_organization_fields, ) from apps.parsers.snapshot_metadata import get_snapshot_artifact, snapshot_metadata from apps.parsers.source_cards import SOURCE_CARD_BY_SLUG from drf_yasg.utils import swagger_serializer_method from rest_framework import serializers BLOCKED_FILE_HOSTS = {"localhost", "localhost.localdomain"} BLOCKED_FILE_HOST_SUFFIXES = (".localhost", ".local", ".internal") SUPPORTED_UPLOAD_EXTENSIONS = tuple( sorted({*SUPPORTED_EXCEL_EXTENSIONS, *SUPPORTED_ZIP_EXTENSIONS, ".zip"}) ) def _is_blocked_ip(address: str) -> bool: ip = ipaddress.ip_address(address) return ( ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified ) def _validate_public_file_host(host: str) -> None: normalized_host = host.strip("[]").rstrip(".").lower() if normalized_host in BLOCKED_FILE_HOSTS or normalized_host.endswith( BLOCKED_FILE_HOST_SUFFIXES ): raise serializers.ValidationError("file_url host is not allowed") try: if _is_blocked_ip(normalized_host): raise serializers.ValidationError("file_url IP address is not allowed") return except ValueError: pass try: address_info = socket.getaddrinfo( normalized_host, 443, type=socket.SOCK_STREAM, ) except socket.gaierror as exc: raise serializers.ValidationError("file_url host cannot be resolved") from exc for item in address_info: if _is_blocked_ip(item[4][0]): raise serializers.ValidationError("file_url host resolves to private IP") # ============================================================================= # Минпромторг - Сертификаты промышленного производства # ============================================================================= class CanonicalOrganizationListSerializer(serializers.ListSerializer): """Adds one canonical organization lookup for the current serialized page.""" def to_representation(self, data): items = list(data.all() if hasattr(data, "all") else data) field_sets = getattr(self.child, "canonical_organization_fields", ()) self.context[ ORGANIZATION_ENRICHMENT_CONTEXT_KEY ] = build_canonical_organization_enrichment(items, field_sets) return super().to_representation(items) class CanonicalOrganizationEnrichmentMixin: """Fill blank v1 organization fields while preserving existing contract.""" canonical_organization_fields: tuple[OrganizationFieldSet, ...] = () def to_representation(self, instance): data = super().to_representation(instance) if not self.canonical_organization_fields: return data enrichment = self.context.get(ORGANIZATION_ENRICHMENT_CONTEXT_KEY) if enrichment is None: enrichment = build_canonical_organization_enrichment( [instance], self.canonical_organization_fields, ) return enrich_organization_fields( data, source=instance, field_sets=self.canonical_organization_fields, enrichment=enrichment, ) class IndustrialCertificateSerializer( CanonicalOrganizationEnrichmentMixin, serializers.ModelSerializer, ): """ Сертификат промышленного производства РФ. Данные загружаются из Минпромторга. """ canonical_organization_fields = ( OrganizationFieldSet( name="organisation_name", inn="inn", ogrn="ogrn", ), ) class Meta: model = IndustrialCertificateRecord list_serializer_class = CanonicalOrganizationListSerializer fields = [ "id", "load_batch", "issue_date", "issue_date_normalized", "certificate_number", "expiry_date", "expiry_date_normalized", "certificate_file_url", "organisation_name", "inn", "ogrn", "registry_organization", "created_at", "updated_at", ] read_only_fields = fields # ============================================================================= # Минпромторг - Реестр производителей # ============================================================================= class ManufacturerSerializer( CanonicalOrganizationEnrichmentMixin, serializers.ModelSerializer, ): """ Производитель из реестра Минпромторга. Данные загружаются из Минпромторга. """ canonical_organization_fields = ( OrganizationFieldSet( name="full_legal_name", inn="inn", ogrn="ogrn", ), ) class Meta: model = ManufacturerRecord list_serializer_class = CanonicalOrganizationListSerializer fields = [ "id", "load_batch", "full_legal_name", "inn", "ogrn", "address", "registry_organization", "created_at", "updated_at", ] read_only_fields = fields class IndustrialProductSerializer( CanonicalOrganizationEnrichmentMixin, serializers.ModelSerializer, ): """ Промышленная продукция из реестра Минпромторга. Данные загружаются из Минпромторга. """ canonical_organization_fields = ( OrganizationFieldSet( name="full_organisation_name", inn="inn", ogrn="ogrn", ), ) class Meta: model = IndustrialProductRecord list_serializer_class = CanonicalOrganizationListSerializer fields = [ "id", "load_batch", "full_organisation_name", "ogrn", "inn", "registry_number", "product_name", "product_model", "okpd2_code", "tnved_code", "regulatory_document", "registry_organization", "created_at", "updated_at", ] read_only_fields = fields # ============================================================================= # Единый реестр проверок (proverki.gov.ru) # ============================================================================= class InspectionSerializer( CanonicalOrganizationEnrichmentMixin, serializers.ModelSerializer, ): """ Проверка из Единого реестра проверок. Поддерживает ФЗ-294 и ФЗ-248. """ canonical_organization_fields = ( OrganizationFieldSet( name="organisation_name", inn="inn", ogrn="ogrn", ), ) class Meta: model = InspectionRecord list_serializer_class = CanonicalOrganizationListSerializer fields = [ "id", "load_batch", "registration_number", "inn", "ogrn", "organisation_name", "control_authority", "inspection_type", "inspection_form", "start_date", "start_date_normalized", "end_date", "end_date_normalized", "status", "legal_basis", "result", "is_federal_law_248", "data_year", "data_month", "registry_organization", "created_at", "updated_at", ] read_only_fields = fields # ============================================================================= # Государственные закупки (zakupki.gov.ru) # ============================================================================= class ProcurementSerializer( CanonicalOrganizationEnrichmentMixin, serializers.ModelSerializer, ): """ Государственная закупка из ЕИС zakupki.gov.ru. Поддерживает 44-ФЗ и 223-ФЗ. """ canonical_organization_fields = ( OrganizationFieldSet( name="customer_name", inn="customer_inn", kpp="customer_kpp", ogrn="customer_ogrn", ), ) class Meta: model = ProcurementRecord list_serializer_class = CanonicalOrganizationListSerializer fields = [ "id", "load_batch", "purchase_number", "purchase_name", "customer_inn", "customer_kpp", "customer_ogrn", "customer_name", "max_price", "max_price_amount", "currency_code", "placement_method", "publish_date", "publish_date_normalized", "end_date", "end_date_normalized", "status", "law_type", "purchase_object_info", "href", "region_code", "data_year", "data_month", "registry_organization", "created_at", "updated_at", ] read_only_fields = fields # ============================================================================= # ФНС - Бухгалтерская отчетность # ============================================================================= class FinancialReportLineSerializer(serializers.ModelSerializer): """Строка финансового отчета.""" class Meta: model = FinancialReportLine fields = [ "id", "form_code", "line_code", "line_name", "year", "period_start", "period_end", ] class FinancialReportSerializer(serializers.ModelSerializer): """ Финансовый отчет ФНС. Данные загружаются из Excel файлов. """ lines_count = serializers.SerializerMethodField() class Meta: model = FinancialReport fields = [ "id", "external_id", "ogrn", "registry_organization", "file_name", "file_hash", "load_batch", "status", "source", "error_message", "created_at", "updated_at", "lines_count", ] read_only_fields = fields def get_lines_count(self, obj) -> int: return getattr(obj, "lines_count", obj.lines.count()) class FinancialReportDetailSerializer(FinancialReportSerializer): """Финансовый отчет с детализацией строк.""" lines = FinancialReportLineSerializer(many=True, read_only=True) class Meta(FinancialReportSerializer.Meta): fields = FinancialReportSerializer.Meta.fields + ["lines"] class FNSFileUploadSerializer(serializers.Serializer): """ Сериализатор для загрузки файлов FNS. Принимает список Excel файлов в формате fin_{id}_{ogrn}.xlsx """ file = serializers.FileField( required=False, help_text="Одиночный файл для загрузки (fin_*.xlsx)", ) files = serializers.ListField( child=serializers.FileField(), required=False, allow_empty=False, help_text="Список файлов для загрузки (fin_*.xlsx)", ) def validate(self, attrs): files = attrs.get("files") single_file = attrs.get("file") if single_file and files: raise serializers.ValidationError( {"file": "Используйте либо file, либо files."} ) if single_file: files = [single_file] if not files: raise serializers.ValidationError( {"file": "Нужно передать file или files."} ) attrs["files"] = self._validate_uploaded_files(files) return attrs def _validate_uploaded_files(self, files): """Валидация файлов.""" for file in files: if not FNS_XLSX_FILENAME_RE.match(file.name): raise serializers.ValidationError( f"Неверный формат имени файла: {file.name}. " "Ожидается: fin_{{id}}_{{ogrn}}.xlsx" ) return files class FNSZipUploadSerializer(serializers.Serializer): """Сериализатор для загрузки ZIP архива с FNS Excel файлами.""" file = serializers.FileField(help_text="ZIP архив с файлами fin_*.xlsx") def validate_file(self, value): if not value.name.lower().endswith(".zip"): raise serializers.ValidationError("Поддерживаются только ZIP архивы") return value class FNSServerPathUploadSerializer(serializers.Serializer): """Сериализатор для запуска обработки файла, уже лежащего на сервере.""" server_path = serializers.CharField( help_text="Абсолютный путь к ZIP/XLSX внутри FNS watch directory", ) class FNSFileUploadSuccessSerializer(serializers.Serializer): """Ответ одиночной загрузки FNS в формате frontend.""" success = serializers.BooleanField(read_only=True) message = serializers.CharField(read_only=True) class ParsingSettingsSerializer(serializers.ModelSerializer): """Настройки периодичности обновления источников парсинга.""" class Meta: model = ParsingSettings fields = [ "manufacturers_and_products", "public_procurements", "defense_unreliable_suppliers", "planned_inspections", "arbitration_cases", "bankruptcy_procedures", "information_security_registries", ] class ParserSourceSerializer(serializers.Serializer): """Описание доступного parser source для dashboard/API.""" key = serializers.CharField() source = serializers.ChoiceField(choices=ParserLoadLog.Source.choices) title = serializers.CharField() agency = serializers.CharField() data_scope = serializers.CharField() task_name = serializers.CharField() is_existing = serializers.BooleanField() requires_file_url = serializers.BooleanField() mode = serializers.CharField() status = serializers.CharField() owner = serializers.CharField(allow_blank=True) upstream_url = serializers.URLField(allow_blank=True) access_method = serializers.CharField() parser_strategy = serializers.CharField() source_notes = serializers.CharField(allow_blank=True) supports_file_upload = serializers.BooleanField() api_route = serializers.CharField(allow_blank=True) result_list_url = serializers.CharField() result_detail_url = serializers.CharField() upload_url = serializers.CharField(allow_blank=True) supports_refresh = serializers.BooleanField() admin_only = serializers.BooleanField() class ParserRunRequestSerializer(serializers.Serializer): """Параметры запуска Celery-задачи парсера.""" params = serializers.DictField( required=False, help_text="For registry snapshot sources only: empty object; loader settings are server-controlled.", ) mode = serializers.ChoiceField( choices=("incremental", "full"), required=False, help_text="SRO parser only; defaults to incremental.", ) file_url = serializers.URLField(required=False, allow_blank=True) file_path = serializers.CharField(required=False, allow_blank=True) proxies = serializers.ListField( child=serializers.CharField(), required=False, allow_empty=True, ) vacancy_sources = serializers.ListField( child=serializers.ChoiceField(choices=["trudvsem"]), required=False, allow_empty=False, ) registry_organizations_only = serializers.BooleanField(required=False) registry_organization_limit = serializers.IntegerField(required=False, min_value=1) year = serializers.IntegerField(required=False, min_value=2000, max_value=2100) month = serializers.IntegerField(required=False, min_value=1, max_value=12) limit = serializers.IntegerField(required=False, min_value=1, max_value=1000) offset = serializers.IntegerField(required=False, min_value=0) max_months_per_law = serializers.IntegerField( required=False, min_value=1, max_value=36, ) start_year = serializers.IntegerField( required=False, min_value=2000, max_value=2100, ) start_month = serializers.IntegerField(required=False, min_value=1, max_value=12) include_fz294 = serializers.BooleanField(required=False) include_fz248 = serializers.BooleanField(required=False) current_year = serializers.IntegerField( required=False, min_value=2000, max_value=2100, ) current_month = serializers.IntegerField(required=False, min_value=1, max_value=12) region_code = serializers.CharField(required=False, allow_blank=True) law_type = serializers.CharField(required=False, allow_blank=True) company_inn = serializers.CharField(required=False, allow_blank=True) text = serializers.CharField(required=False, allow_blank=True) def validate_file_url(self, value: str) -> str: """Разрешить worker-download только с публичных HTTPS URL.""" if not value: return value parsed = urlsplit(value) if parsed.scheme != "https": raise serializers.ValidationError("file_url must use https") if parsed.username or parsed.password: raise serializers.ValidationError("file_url credentials are not allowed") if not parsed.hostname: raise serializers.ValidationError("file_url host is required") _validate_public_file_host(parsed.hostname) return value def validate(self, attrs): attrs = super().validate(attrs) if ("start_year" in attrs) != ("start_month" in attrs): raise serializers.ValidationError( { "start_month": ( "start_year and start_month must be provided together" ) } ) return attrs class ParserUploadRequestSerializer(serializers.Serializer): """Файл ручной загрузки реестра или финансовой выгрузки.""" file = serializers.FileField() def validate_file(self, value): file_name = (value.name or "").lower() if not file_name.endswith(SUPPORTED_UPLOAD_EXTENSIONS): extensions = ", ".join(SUPPORTED_UPLOAD_EXTENSIONS) raise serializers.ValidationError( f"Unsupported file extension. Allowed: {extensions}" ) if value.size > MAX_FILE_SIZE_BYTES: raise serializers.ValidationError( f"File exceeds size limit: {MAX_FILE_SIZE_BYTES} bytes" ) return value class ParserScheduleRequestSerializer(ParserRunRequestSerializer): """Параметры создания/обновления периодической Celery-задачи парсера.""" SCHEDULE_TYPES = ("interval", "crontab") PERIODS = ("seconds", "minutes", "hours", "days") source_key = serializers.CharField(required=False) name = serializers.CharField(required=False, allow_blank=True, max_length=200) enabled = serializers.BooleanField(required=False, default=True) schedule_type = serializers.ChoiceField( choices=SCHEDULE_TYPES, required=False, default="interval", ) every = serializers.IntegerField(required=False, min_value=1) period = serializers.ChoiceField(choices=PERIODS, required=False, default="hours") minute = serializers.CharField(required=False, allow_blank=True, default="0") hour = serializers.CharField(required=False, allow_blank=True, default="*") day_of_week = serializers.CharField(required=False, allow_blank=True, default="*") day_of_month = serializers.CharField(required=False, allow_blank=True, default="*") month_of_year = serializers.CharField(required=False, allow_blank=True, default="*") def validate(self, attrs): attrs = super().validate(attrs) if attrs.get("schedule_type", "interval") == "interval" and not attrs.get( "every" ): raise serializers.ValidationError( {"every": "Required for interval schedule"} ) return attrs class ParserScheduleSerializer(serializers.Serializer): """Описание периодической задачи парсера.""" id = serializers.IntegerField() name = serializers.CharField() source_key = serializers.CharField() source = serializers.CharField() title = serializers.CharField() task_name = serializers.CharField() enabled = serializers.BooleanField() schedule_type = serializers.CharField() schedule = serializers.DictField() params = serializers.DictField() last_run_at = serializers.DateTimeField(allow_null=True) total_run_count = serializers.IntegerField() date_changed = serializers.DateTimeField() class ParserListQuerySerializer(serializers.Serializer): """Общие query-параметры списков парсеров.""" limit = serializers.IntegerField( required=False, default=50, min_value=1, max_value=200, ) class ParserResultQuerySerializer(serializers.Serializer): """Query-параметры per-source result endpoints.""" page = serializers.IntegerField(required=False, default=1, min_value=1) page_size = serializers.IntegerField( required=False, default=20, min_value=1, max_value=100, ) limit = serializers.IntegerField(required=False, min_value=1, max_value=100) id = serializers.IntegerField(required=False, min_value=1) source = serializers.CharField(required=False, allow_blank=True) external_id = serializers.CharField(required=False, allow_blank=True) inn = serializers.CharField(required=False, allow_blank=True) ogrn = serializers.CharField(required=False, allow_blank=True) load_batch = serializers.IntegerField(required=False, min_value=1) batch_id = serializers.IntegerField(required=False, min_value=1) status = serializers.CharField(required=False, allow_blank=True) record_date = serializers.CharField(required=False, allow_blank=True) date_from = serializers.DateField(required=False) date_to = serializers.DateField(required=False) search = serializers.CharField(required=False, allow_blank=True) ordering = serializers.CharField(required=False, allow_blank=True) include_payload = serializers.BooleanField(required=False, default=True) def validate(self, attrs): attrs = super().validate(attrs) if attrs.get("limit"): attrs["page"] = 1 attrs["page_size"] = attrs["limit"] if attrs.get("batch_id") and not attrs.get("load_batch"): attrs["load_batch"] = attrs["batch_id"] if ( attrs.get("date_from") and attrs.get("date_to") and attrs["date_from"] > attrs["date_to"] ): raise serializers.ValidationError( {"date_to": "date_to must be greater than or equal to date_from"} ) return attrs class ParserRunResponseSerializer(serializers.Serializer): """Ответ API на запуск задачи.""" task_id = serializers.CharField() task_ids = serializers.ListField(child=serializers.CharField()) source = serializers.CharField() task_name = serializers.CharField() status = serializers.ChoiceField( choices=("queued", "pending", "started", "running", "skipped"), required=False ) class IntegratedStructureUploadResponseSerializer(serializers.Serializer): status = serializers.ChoiceField(choices=("published", "skipped")) artifact_id = serializers.UUIDField() task_id = serializers.CharField() batch_id = serializers.IntegerField() records_count = serializers.IntegerField() structures_count = serializers.IntegerField() event_rows_count = serializers.IntegerField() document_link_rows_count = serializers.IntegerField() unique_documents_count = serializers.IntegerField() events_without_documents_count = serializers.IntegerField() reason = serializers.CharField(required=False) # ============================================================================= # Служебные модели # ============================================================================= class SnapshotMetadataSerializer(serializers.Serializer): """Provenance of the exact artifact published by this load.""" artifact_id = serializers.UUIDField(read_only=True) snapshot_date = serializers.DateField(read_only=True, allow_null=True) version = serializers.CharField(read_only=True, allow_blank=True) checksum_sha256 = serializers.CharField(read_only=True, allow_blank=True) published_at = serializers.DateTimeField(read_only=True, allow_null=True) class ParserLoadLogSerializer(serializers.ModelSerializer): """ Лог загрузки парсера. Информация о каждой загрузке данных из внешнего источника. """ source_display = serializers.CharField(source="get_source_display", read_only=True) source_label = serializers.CharField(source="get_source_display", read_only=True) snapshot = serializers.SerializerMethodField() meta = serializers.SerializerMethodField() organizations_count = serializers.SerializerMethodField() artifact_uid = serializers.SerializerMethodField() parsed_count = serializers.SerializerMethodField() published_count = serializers.SerializerMethodField() quarantined_count = serializers.SerializerMethodField() rejection_reasons = serializers.SerializerMethodField() class Meta: model = ParserLoadLog fields = [ "id", "batch_id", "source", "source_display", "source_label", "snapshot", "meta", "records_count", "organizations_count", "artifact_uid", "parsed_count", "published_count", "quarantined_count", "rejection_reasons", "status", "error_message", "created_at", "updated_at", ] read_only_fields = fields def get_organizations_count(self, obj) -> int: artifact = self._get_artifact(obj) if artifact is not None: return artifact.published_organizations_count if obj.source == ParserLoadLog.Source.FNS_REPORTS: return ( FinancialReport.objects.filter(load_batch=obj.batch_id) .exclude(ogrn="") .values("ogrn") .distinct() .count() ) if obj.source == ParserLoadLog.Source.INDUSTRIAL: return ( IndustrialCertificateRecord.objects.filter(load_batch=obj.batch_id) .exclude(inn="") .values("inn") .distinct() .count() ) if obj.source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: return ( IndustrialProductRecord.objects.filter(load_batch=obj.batch_id) .exclude(inn="") .values("inn") .distinct() .count() ) if obj.source == ParserLoadLog.Source.MANUFACTURES: return ( ManufacturerRecord.objects.filter(load_batch=obj.batch_id) .exclude(inn="") .values("inn") .distinct() .count() ) if obj.source == ParserLoadLog.Source.INSPECTIONS: return ( InspectionRecord.objects.filter(load_batch=obj.batch_id) .exclude(inn="") .values("inn") .distinct() .count() ) if obj.source == ParserLoadLog.Source.PROCUREMENTS: return ( ProcurementRecord.objects.filter(load_batch=obj.batch_id) .exclude(customer_inn="") .values("customer_inn") .distinct() .count() ) return 0 @swagger_serializer_method( serializer_or_field=SnapshotRunMetadataSerializer(allow_null=True) ) def get_meta(self, obj): artifact = self._get_artifact(obj) return ( SnapshotRunMetadataSerializer(artifact.metadata).data if artifact is not None else None ) @swagger_serializer_method( serializer_or_field=SnapshotMetadataSerializer(allow_null=True) ) def get_snapshot(self, obj): metadata = snapshot_metadata(obj) return ( SnapshotMetadataSerializer(metadata).data if metadata is not None else None ) @staticmethod def _get_artifact(obj) -> ParserSourceArtifact | None: return get_snapshot_artifact(obj) def get_artifact_uid(self, obj) -> str | None: artifact = self._get_artifact(obj) return str(artifact.uid) if artifact is not None else None def get_parsed_count(self, obj) -> int: artifact = self._get_artifact(obj) return artifact.parsed_count if artifact is not None else obj.records_count def get_published_count(self, obj) -> int: artifact = self._get_artifact(obj) return artifact.published_count if artifact is not None else obj.records_count def get_quarantined_count(self, obj) -> int: artifact = self._get_artifact(obj) return artifact.quarantined_count if artifact is not None else 0 def get_rejection_reasons(self, obj) -> dict: artifact = self._get_artifact(obj) return dict(artifact.rejection_reasons) if artifact is not None else {} class GenericParserRecordSerializer( CanonicalOrganizationEnrichmentMixin, serializers.ModelSerializer, ): """Сериализатор универсальных записей новых источников.""" canonical_organization_fields = ( OrganizationFieldSet( name="organisation_name", inn="inn", ogrn="ogrn", ), ) class Meta: model = GenericParserRecord list_serializer_class = CanonicalOrganizationListSerializer fields = [ "id", "load_batch", "source", "external_id", "inn", "ogrn", "organisation_name", "title", "record_date", "amount", "status", "url", "payload", "registry_organization", "created_at", "updated_at", ] read_only_fields = fields class ParserResultRecordSerializer(serializers.Serializer): """Унифицированная запись результата конкретного источника.""" id = serializers.IntegerField() load_batch = serializers.IntegerField() source = serializers.CharField() vacancy_source = serializers.CharField(allow_blank=True, required=False) external_id = serializers.CharField(allow_blank=True) inn = serializers.CharField(allow_blank=True) ogrn = serializers.CharField(allow_blank=True) organisation_name = serializers.CharField(allow_blank=True) title = serializers.CharField(allow_blank=True) record_date = serializers.CharField(allow_blank=True) amount = serializers.DecimalField( max_digits=20, decimal_places=2, allow_null=True, ) status = serializers.CharField(allow_blank=True) url = serializers.CharField(allow_blank=True) payload = serializers.DictField() created_at = serializers.DateTimeField() updated_at = serializers.DateTimeField() class ParserLoadLogListSerializer(serializers.Serializer): """Строка списка логов в frontend-friendly формате.""" id = serializers.IntegerField(read_only=True) batch_id = serializers.IntegerField(read_only=True) source = serializers.ChoiceField( choices=[*ParserLoadLog.Source.values, *SOURCE_CARD_BY_SLUG], read_only=True ) source_label = serializers.CharField(read_only=True, allow_null=True) records_count = serializers.IntegerField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) artifact_uid = serializers.UUIDField(read_only=True, allow_null=True) meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True) snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True) parsed_count = serializers.IntegerField(read_only=True) published_count = serializers.IntegerField(read_only=True) quarantined_count = serializers.IntegerField(read_only=True) rejection_reasons = serializers.DictField(read_only=True) status = serializers.ChoiceField( choices=ParserLoadLog.Status.choices, read_only=True ) status_label = serializers.CharField(read_only=True) error_message = serializers.CharField(read_only=True, allow_blank=True) created_at = serializers.DateTimeField(read_only=True) updated_at = serializers.DateTimeField(read_only=True) class ParserLoadLogPageSerializer(serializers.Serializer): """Пагинированный ответ списка логов для frontend.""" count = serializers.IntegerField(read_only=True) next = serializers.CharField(read_only=True, allow_null=True) previous = serializers.CharField(read_only=True, allow_null=True) results = ParserLoadLogListSerializer(many=True, read_only=True) class SourceCardRefreshParamSerializer(serializers.Serializer): """Описание параметра ручного обновления карточки источника.""" name = serializers.CharField(read_only=True) label = serializers.CharField(read_only=True) description = serializers.CharField(read_only=True) required = serializers.BooleanField(read_only=True) type = serializers.CharField(read_only=True) default = serializers.JSONField(read_only=True) class SourceCardLoadSerializer(serializers.Serializer): """Сводка по последней загрузке источника.""" meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True) snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True) batch_id = serializers.IntegerField(read_only=True) source = serializers.ChoiceField( choices=ParserLoadLog.Source.choices, read_only=True ) source_display = serializers.CharField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) source_label = serializers.CharField(read_only=True) records_count = serializers.IntegerField(read_only=True) status = serializers.ChoiceField( choices=ParserLoadLog.Status.choices, read_only=True ) error_message = serializers.CharField(read_only=True) created_at = serializers.DateTimeField(read_only=True) updated_at = serializers.DateTimeField(read_only=True) class SourceCardTaskSerializer(serializers.Serializer): """Краткая информация по фоновой задаче карточки.""" task_id = serializers.CharField(read_only=True) task_name = serializers.CharField(read_only=True) status = serializers.CharField(read_only=True) progress = serializers.IntegerField(read_only=True) progress_message = serializers.CharField(read_only=True) started_at = serializers.DateTimeField(read_only=True, allow_null=True) created_at = serializers.DateTimeField(read_only=True) meta = serializers.JSONField(read_only=True) class SourceCardItemSerializer(serializers.Serializer): """Подисточник внутри агрегированной карточки.""" code = serializers.CharField(read_only=True) refresh_key = serializers.CharField(read_only=True) title = serializers.CharField(read_only=True) description = serializers.CharField(read_only=True) parser_source = serializers.CharField(read_only=True, allow_null=True) parser_source_display = serializers.CharField(read_only=True, allow_null=True) records_count = serializers.IntegerField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True) upload_url = serializers.CharField(read_only=True, allow_blank=True) latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True) latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True) class SourceCardSerializer(serializers.Serializer): """Serializer for frontend source cards list.""" slug = serializers.CharField(read_only=True) title = serializers.CharField(read_only=True) description = serializers.CharField(read_only=True) order = serializers.IntegerField(read_only=True) is_available = serializers.BooleanField(read_only=True) status = serializers.CharField(read_only=True) status_label = serializers.CharField(read_only=True) progress = serializers.IntegerField(read_only=True) records_count = serializers.IntegerField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True) next_update_at = serializers.DateTimeField(read_only=True, allow_null=True) error_message = serializers.CharField(read_only=True) task_names = serializers.ListField(child=serializers.CharField(), read_only=True) refresh_requires_params = serializers.BooleanField(read_only=True) supports_refresh = serializers.BooleanField(read_only=True) upload_url = serializers.CharField(read_only=True, allow_blank=True) refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True) active_tasks = SourceCardTaskSerializer(many=True, read_only=True) source_items = SourceCardItemSerializer(many=True, read_only=True) latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True) latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True) class SourceTaskStatusSerializer(serializers.Serializer): """Табличная строка статуса источника для экрана задач парсинга.""" row_number = serializers.IntegerField(read_only=True) slug = serializers.CharField(read_only=True) source = serializers.CharField(read_only=True) status = serializers.CharField(read_only=True) status_label = serializers.CharField(read_only=True) actualized_at = serializers.DateTimeField(read_only=True, allow_null=True) next_update_at = serializers.DateTimeField(read_only=True, allow_null=True) records_count = serializers.IntegerField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) progress = serializers.IntegerField(read_only=True) error_message = serializers.CharField(read_only=True) active_tasks = SourceCardTaskSerializer(many=True, read_only=True) class SourceCardDetailSerializer(SourceCardSerializer): """List and detail deliberately share the complete card contract.""" class SourceCardRefreshRequestSerializer(serializers.Serializer): """Request body for manual card refresh.""" params = serializers.DictField(required=False) def to_internal_value(self, data): if isinstance(data, dict) and set(data) - set(self.fields): raise serializers.ValidationError({"detail": "Only params is supported."}) return super().to_internal_value(data) class SroParserRunRequestSerializer(SourceCardRefreshRequestSerializer): mode = serializers.ChoiceField( choices=("incremental", "full"), required=False, default="incremental" ) def to_internal_value(self, data): if isinstance(data, dict) and data.get("mode", "incremental") not in ( "incremental", "full", ): raise APIValidationError( message="Допустимые режимы: incremental, full.", code="invalid_mode" ) return super().to_internal_value(data) class FedresursParserRunRequestSerializer(SourceCardRefreshRequestSerializer): mode = serializers.ChoiceField( choices=("full", "targeted"), required=False, default="full" ) organization_uids = serializers.ListField( child=serializers.UUIDField(), required=False, max_length=500 ) def validate(self, attrs): mode = attrs.get("mode", "full") uids = attrs.get("organization_uids") if (mode == "targeted") != bool(uids): raise serializers.ValidationError( {"organization_uids": "Required only for targeted refresh."} ) attrs["params"] = { **attrs.get("params", {}), **({"organization_uids": [str(uid) for uid in uids]} if uids else {}), } return attrs class SourceCardRefreshTaskSerializer(serializers.Serializer): """Queued task info after refresh start.""" task_id = serializers.CharField(read_only=True) task_name = serializers.CharField(read_only=True) class SourceCardRefreshResponseSerializer(serializers.Serializer): """Response serializer for manual card refresh.""" source_card = serializers.CharField(read_only=True) status = serializers.CharField(read_only=True) requested_at = serializers.DateTimeField(read_only=True) tasks = SourceCardRefreshTaskSerializer(many=True, read_only=True) class FrontendApiResponseSerializer(serializers.Serializer): """Общая форма success/data/errors/meta для frontend sources API.""" success = serializers.BooleanField(read_only=True) errors = serializers.JSONField(read_only=True, allow_null=True) meta = serializers.JSONField(read_only=True, allow_null=True) class ParserRunEnvelopeSerializer(FrontendApiResponseSerializer): """Pollable parser task inside the standard API envelope.""" data = ParserRunResponseSerializer(read_only=True) class SourceCardListResponseSerializer(FrontendApiResponseSerializer): """Envelope для списка карточек источников.""" data = SourceCardSerializer(many=True, read_only=True) class SourceTaskStatusListResponseSerializer(FrontendApiResponseSerializer): """Envelope для списка статусов источников.""" data = SourceTaskStatusSerializer(many=True, read_only=True) class SourceCardDetailResponseSerializer(FrontendApiResponseSerializer): """Envelope для детальной карточки источника.""" data = SourceCardDetailSerializer(read_only=True) class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer): """Минимальный ответ запуска обновления карточки по md.""" task_id = serializers.CharField(read_only=True, allow_null=True) task_ids = serializers.ListField(child=serializers.CharField(), read_only=True) status = serializers.ChoiceField(choices=("queued", "accepted"), read_only=True) class SourceRefreshErrorItemSerializer(serializers.Serializer): code = serializers.CharField(read_only=True) message = serializers.CharField(read_only=True) details = serializers.JSONField(read_only=True, required=False) class SourceRefreshErrorSerializer(serializers.Serializer): success = serializers.BooleanField(read_only=True) data = serializers.JSONField(read_only=True, allow_null=True) errors = SourceRefreshErrorItemSerializer(many=True, read_only=True) meta = serializers.DictField(read_only=True, allow_null=True) class SourceRefreshConflictItemSerializer(serializers.Serializer): code = serializers.ChoiceField( choices=("refresh_already_running", "upstream_access_not_approved"), read_only=True, ) message = serializers.CharField(read_only=True) class SourceRefreshConflictSerializer(serializers.Serializer): success = serializers.BooleanField(read_only=True) data = serializers.JSONField(read_only=True, allow_null=True) errors = SourceRefreshConflictItemSerializer(many=True, read_only=True) meta = serializers.DictField(read_only=True, allow_null=True) class ParserDashboardGroupsSerializer(serializers.Serializer): api = ParserSourceSerializer(many=True, read_only=True) uploads = ParserSourceSerializer(many=True, read_only=True) class ParserDashboardSerializer(serializers.Serializer): sources = ParserSourceSerializer(many=True, read_only=True) api_sources = ParserSourceSerializer(many=True, read_only=True) file_sources = ParserSourceSerializer(many=True, read_only=True) groups = ParserDashboardGroupsSerializer(read_only=True) schedules = ParserScheduleSerializer(many=True, read_only=True) jobs = BackgroundJobListSerializer(many=True, read_only=True) source_counts = serializers.DictField( child=serializers.IntegerField(), read_only=True ) registry_data_coverage = serializers.DictField(read_only=True) registry_enrichment_analytics = serializers.DictField(read_only=True) load_logs = ParserLoadLogSerializer(many=True, read_only=True) class ParserDashboardResponseSerializer(FrontendApiResponseSerializer): data = ParserDashboardSerializer(read_only=True)