feat: complete published registry contracts and gated SRO ingestion
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s

This commit is contained in:
Aleksandr Meshchryakov
2026-09-14 17:01:02 +02:00
parent 49cbfd265c
commit 18971d33ec
76 changed files with 33156 additions and 311 deletions

View File

@@ -4,6 +4,11 @@ import ipaddress
import socket
from urllib.parse import urlsplit
from apps.core.exceptions import ValidationError as APIValidationError
from apps.core.serializers import (
BackgroundJobListSerializer,
SnapshotRunMetadataSerializer,
)
from apps.parsers.clients.common.structured import (
MAX_FILE_SIZE_BYTES,
SUPPORTED_EXCEL_EXTENSIONS,
@@ -29,6 +34,9 @@ from apps.parsers.organization_enrichment import (
build_canonical_organization_enrichment,
enrich_organization_fields,
)
from apps.parsers.snapshot_metadata import get_snapshot_artifact, snapshot_metadata
from apps.parsers.source_cards import SOURCE_CARD_BY_SLUG
from drf_yasg.utils import swagger_serializer_method
from rest_framework import serializers
BLOCKED_FILE_HOSTS = {"localhost", "localhost.localdomain"}
@@ -506,7 +514,7 @@ class ParserSourceSerializer(serializers.Serializer):
"""Описание доступного parser source для dashboard/API."""
key = serializers.CharField()
source = serializers.CharField()
source = serializers.ChoiceField(choices=ParserLoadLog.Source.choices)
title = serializers.CharField()
agency = serializers.CharField()
data_scope = serializers.CharField()
@@ -532,6 +540,15 @@ class ParserSourceSerializer(serializers.Serializer):
class ParserRunRequestSerializer(serializers.Serializer):
"""Параметры запуска Celery-задачи парсера."""
params = serializers.DictField(
required=False,
help_text="For registry snapshot sources only: empty object; loader settings are server-controlled.",
)
mode = serializers.ChoiceField(
choices=("incremental", "full"),
required=False,
help_text="SRO parser only; defaults to incremental.",
)
file_url = serializers.URLField(required=False, allow_blank=True)
file_path = serializers.CharField(required=False, allow_blank=True)
proxies = serializers.ListField(
@@ -735,7 +752,9 @@ class ParserRunResponseSerializer(serializers.Serializer):
task_ids = serializers.ListField(child=serializers.CharField())
source = serializers.CharField()
task_name = serializers.CharField()
status = serializers.CharField(required=False)
status = serializers.ChoiceField(
choices=("queued", "pending", "started", "running", "skipped"), required=False
)
# =============================================================================
@@ -743,6 +762,16 @@ class ParserRunResponseSerializer(serializers.Serializer):
# =============================================================================
class SnapshotMetadataSerializer(serializers.Serializer):
"""Provenance of the exact artifact published by this load."""
artifact_id = serializers.UUIDField(read_only=True)
snapshot_date = serializers.DateField(read_only=True, allow_null=True)
version = serializers.CharField(read_only=True, allow_blank=True)
checksum_sha256 = serializers.CharField(read_only=True, allow_blank=True)
published_at = serializers.DateTimeField(read_only=True, allow_null=True)
class ParserLoadLogSerializer(serializers.ModelSerializer):
"""
Лог загрузки парсера.
@@ -751,6 +780,9 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
"""
source_display = serializers.CharField(source="get_source_display", read_only=True)
source_label = serializers.CharField(source="get_source_display", read_only=True)
snapshot = serializers.SerializerMethodField()
meta = serializers.SerializerMethodField()
organizations_count = serializers.SerializerMethodField()
artifact_uid = serializers.SerializerMethodField()
parsed_count = serializers.SerializerMethodField()
@@ -765,6 +797,9 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
"batch_id",
"source",
"source_display",
"source_label",
"snapshot",
"meta",
"records_count",
"organizations_count",
"artifact_uid",
@@ -782,15 +817,7 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
def get_organizations_count(self, obj) -> int:
artifact = self._get_artifact(obj)
if artifact is not None:
return (
artifact.staged_records.filter(
disposition="published",
organization__isnull=False,
)
.values("organization_id")
.distinct()
.count()
)
return artifact.published_organizations_count
if obj.source == ParserLoadLog.Source.FNS_REPORTS:
return (
FinancialReport.objects.filter(load_batch=obj.batch_id)
@@ -841,20 +868,29 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
)
return 0
@swagger_serializer_method(
serializer_or_field=SnapshotRunMetadataSerializer(allow_null=True)
)
def get_meta(self, obj):
artifact = self._get_artifact(obj)
return (
SnapshotRunMetadataSerializer(artifact.metadata).data
if artifact is not None
else None
)
@swagger_serializer_method(
serializer_or_field=SnapshotMetadataSerializer(allow_null=True)
)
def get_snapshot(self, obj):
metadata = snapshot_metadata(obj)
return (
SnapshotMetadataSerializer(metadata).data if metadata is not None else None
)
@staticmethod
def _get_artifact(obj) -> ParserSourceArtifact | None:
if hasattr(obj, "_source_artifact_cache"):
return obj._source_artifact_cache
artifact = (
ParserSourceArtifact.objects.filter(
source=obj.source,
load_batch=obj.batch_id,
)
.order_by("-created_at")
.first()
)
obj._source_artifact_cache = artifact
return artifact
return get_snapshot_artifact(obj)
def get_artifact_uid(self, obj) -> str | None:
artifact = self._get_artifact(obj)
@@ -945,16 +981,22 @@ class ParserLoadLogListSerializer(serializers.Serializer):
id = serializers.IntegerField(read_only=True)
batch_id = serializers.IntegerField(read_only=True)
source = serializers.CharField(read_only=True)
source = serializers.ChoiceField(
choices=[*ParserLoadLog.Source.values, *SOURCE_CARD_BY_SLUG], read_only=True
)
source_label = serializers.CharField(read_only=True, allow_null=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
artifact_uid = serializers.UUIDField(read_only=True, allow_null=True)
meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True)
snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True)
parsed_count = serializers.IntegerField(read_only=True)
published_count = serializers.IntegerField(read_only=True)
quarantined_count = serializers.IntegerField(read_only=True)
rejection_reasons = serializers.DictField(read_only=True)
status = serializers.CharField(read_only=True)
status = serializers.ChoiceField(
choices=ParserLoadLog.Status.choices, read_only=True
)
status_label = serializers.CharField(read_only=True)
error_message = serializers.CharField(read_only=True, allow_blank=True)
created_at = serializers.DateTimeField(read_only=True)
@@ -984,11 +1026,19 @@ class SourceCardRefreshParamSerializer(serializers.Serializer):
class SourceCardLoadSerializer(serializers.Serializer):
"""Сводка по последней загрузке источника."""
meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True)
snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True)
batch_id = serializers.IntegerField(read_only=True)
source = serializers.CharField(read_only=True)
source = serializers.ChoiceField(
choices=ParserLoadLog.Source.choices, read_only=True
)
source_display = serializers.CharField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
source_label = serializers.CharField(read_only=True)
records_count = serializers.IntegerField(read_only=True)
status = serializers.CharField(read_only=True)
status = serializers.ChoiceField(
choices=ParserLoadLog.Status.choices, read_only=True
)
error_message = serializers.CharField(read_only=True)
created_at = serializers.DateTimeField(read_only=True)
updated_at = serializers.DateTimeField(read_only=True)
@@ -1002,7 +1052,7 @@ class SourceCardTaskSerializer(serializers.Serializer):
status = serializers.CharField(read_only=True)
progress = serializers.IntegerField(read_only=True)
progress_message = serializers.CharField(read_only=True)
started_at = serializers.DateTimeField(read_only=True)
started_at = serializers.DateTimeField(read_only=True, allow_null=True)
created_at = serializers.DateTimeField(read_only=True)
meta = serializers.JSONField(read_only=True)
@@ -1046,6 +1096,11 @@ class SourceCardSerializer(serializers.Serializer):
upload_url = serializers.CharField(read_only=True, allow_blank=True)
refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True)
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
source_items = SourceCardItemSerializer(many=True, read_only=True)
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
class SourceTaskStatusSerializer(serializers.Serializer):
"""Табличная строка статуса источника для экрана задач парсинга."""
@@ -1065,12 +1120,7 @@ class SourceTaskStatusSerializer(serializers.Serializer):
class SourceCardDetailSerializer(SourceCardSerializer):
"""Detailed serializer for a single frontend source card."""
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
source_items = SourceCardItemSerializer(many=True, read_only=True)
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
"""List and detail deliberately share the complete card contract."""
class SourceCardRefreshRequestSerializer(serializers.Serializer):
@@ -1078,6 +1128,27 @@ class SourceCardRefreshRequestSerializer(serializers.Serializer):
params = serializers.DictField(required=False)
def to_internal_value(self, data):
if isinstance(data, dict) and set(data) - set(self.fields):
raise serializers.ValidationError({"detail": "Only params is supported."})
return super().to_internal_value(data)
class SroParserRunRequestSerializer(SourceCardRefreshRequestSerializer):
mode = serializers.ChoiceField(
choices=("incremental", "full"), required=False, default="incremental"
)
def to_internal_value(self, data):
if isinstance(data, dict) and data.get("mode", "incremental") not in (
"incremental",
"full",
):
raise APIValidationError(
message="Допустимые режимы: incremental, full.", code="invalid_mode"
)
return super().to_internal_value(data)
class SourceCardRefreshTaskSerializer(serializers.Serializer):
"""Queued task info after refresh start."""
@@ -1132,4 +1203,43 @@ class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
task_id = serializers.CharField(read_only=True, allow_null=True)
task_ids = serializers.ListField(child=serializers.CharField(), read_only=True)
status = serializers.CharField(read_only=True)
status = serializers.ChoiceField(choices=("queued", "accepted"), read_only=True)
class SourceRefreshConflictItemSerializer(serializers.Serializer):
code = serializers.ChoiceField(
choices=("refresh_already_running", "upstream_access_not_approved"),
read_only=True,
)
message = serializers.CharField(read_only=True)
class SourceRefreshConflictSerializer(serializers.Serializer):
success = serializers.BooleanField(read_only=True)
data = serializers.JSONField(read_only=True, allow_null=True)
errors = SourceRefreshConflictItemSerializer(many=True, read_only=True)
meta = serializers.DictField(read_only=True, allow_null=True)
class ParserDashboardGroupsSerializer(serializers.Serializer):
api = ParserSourceSerializer(many=True, read_only=True)
uploads = ParserSourceSerializer(many=True, read_only=True)
class ParserDashboardSerializer(serializers.Serializer):
sources = ParserSourceSerializer(many=True, read_only=True)
api_sources = ParserSourceSerializer(many=True, read_only=True)
file_sources = ParserSourceSerializer(many=True, read_only=True)
groups = ParserDashboardGroupsSerializer(read_only=True)
schedules = ParserScheduleSerializer(many=True, read_only=True)
jobs = BackgroundJobListSerializer(many=True, read_only=True)
source_counts = serializers.DictField(
child=serializers.IntegerField(), read_only=True
)
registry_data_coverage = serializers.DictField(read_only=True)
registry_enrichment_analytics = serializers.DictField(read_only=True)
load_logs = ParserLoadLogSerializer(many=True, read_only=True)
class ParserDashboardResponseSerializer(FrontendApiResponseSerializer):
data = ParserDashboardSerializer(read_only=True)