feat: complete published registry contracts and gated SRO ingestion
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s
This commit is contained in:
@@ -4,6 +4,11 @@ import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from apps.core.exceptions import ValidationError as APIValidationError
|
||||
from apps.core.serializers import (
|
||||
BackgroundJobListSerializer,
|
||||
SnapshotRunMetadataSerializer,
|
||||
)
|
||||
from apps.parsers.clients.common.structured import (
|
||||
MAX_FILE_SIZE_BYTES,
|
||||
SUPPORTED_EXCEL_EXTENSIONS,
|
||||
@@ -29,6 +34,9 @@ from apps.parsers.organization_enrichment import (
|
||||
build_canonical_organization_enrichment,
|
||||
enrich_organization_fields,
|
||||
)
|
||||
from apps.parsers.snapshot_metadata import get_snapshot_artifact, snapshot_metadata
|
||||
from apps.parsers.source_cards import SOURCE_CARD_BY_SLUG
|
||||
from drf_yasg.utils import swagger_serializer_method
|
||||
from rest_framework import serializers
|
||||
|
||||
BLOCKED_FILE_HOSTS = {"localhost", "localhost.localdomain"}
|
||||
@@ -506,7 +514,7 @@ class ParserSourceSerializer(serializers.Serializer):
|
||||
"""Описание доступного parser source для dashboard/API."""
|
||||
|
||||
key = serializers.CharField()
|
||||
source = serializers.CharField()
|
||||
source = serializers.ChoiceField(choices=ParserLoadLog.Source.choices)
|
||||
title = serializers.CharField()
|
||||
agency = serializers.CharField()
|
||||
data_scope = serializers.CharField()
|
||||
@@ -532,6 +540,15 @@ class ParserSourceSerializer(serializers.Serializer):
|
||||
class ParserRunRequestSerializer(serializers.Serializer):
|
||||
"""Параметры запуска Celery-задачи парсера."""
|
||||
|
||||
params = serializers.DictField(
|
||||
required=False,
|
||||
help_text="For registry snapshot sources only: empty object; loader settings are server-controlled.",
|
||||
)
|
||||
mode = serializers.ChoiceField(
|
||||
choices=("incremental", "full"),
|
||||
required=False,
|
||||
help_text="SRO parser only; defaults to incremental.",
|
||||
)
|
||||
file_url = serializers.URLField(required=False, allow_blank=True)
|
||||
file_path = serializers.CharField(required=False, allow_blank=True)
|
||||
proxies = serializers.ListField(
|
||||
@@ -735,7 +752,9 @@ class ParserRunResponseSerializer(serializers.Serializer):
|
||||
task_ids = serializers.ListField(child=serializers.CharField())
|
||||
source = serializers.CharField()
|
||||
task_name = serializers.CharField()
|
||||
status = serializers.CharField(required=False)
|
||||
status = serializers.ChoiceField(
|
||||
choices=("queued", "pending", "started", "running", "skipped"), required=False
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -743,6 +762,16 @@ class ParserRunResponseSerializer(serializers.Serializer):
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class SnapshotMetadataSerializer(serializers.Serializer):
|
||||
"""Provenance of the exact artifact published by this load."""
|
||||
|
||||
artifact_id = serializers.UUIDField(read_only=True)
|
||||
snapshot_date = serializers.DateField(read_only=True, allow_null=True)
|
||||
version = serializers.CharField(read_only=True, allow_blank=True)
|
||||
checksum_sha256 = serializers.CharField(read_only=True, allow_blank=True)
|
||||
published_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||||
|
||||
|
||||
class ParserLoadLogSerializer(serializers.ModelSerializer):
|
||||
"""
|
||||
Лог загрузки парсера.
|
||||
@@ -751,6 +780,9 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
|
||||
"""
|
||||
|
||||
source_display = serializers.CharField(source="get_source_display", read_only=True)
|
||||
source_label = serializers.CharField(source="get_source_display", read_only=True)
|
||||
snapshot = serializers.SerializerMethodField()
|
||||
meta = serializers.SerializerMethodField()
|
||||
organizations_count = serializers.SerializerMethodField()
|
||||
artifact_uid = serializers.SerializerMethodField()
|
||||
parsed_count = serializers.SerializerMethodField()
|
||||
@@ -765,6 +797,9 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
|
||||
"batch_id",
|
||||
"source",
|
||||
"source_display",
|
||||
"source_label",
|
||||
"snapshot",
|
||||
"meta",
|
||||
"records_count",
|
||||
"organizations_count",
|
||||
"artifact_uid",
|
||||
@@ -782,15 +817,7 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
|
||||
def get_organizations_count(self, obj) -> int:
|
||||
artifact = self._get_artifact(obj)
|
||||
if artifact is not None:
|
||||
return (
|
||||
artifact.staged_records.filter(
|
||||
disposition="published",
|
||||
organization__isnull=False,
|
||||
)
|
||||
.values("organization_id")
|
||||
.distinct()
|
||||
.count()
|
||||
)
|
||||
return artifact.published_organizations_count
|
||||
if obj.source == ParserLoadLog.Source.FNS_REPORTS:
|
||||
return (
|
||||
FinancialReport.objects.filter(load_batch=obj.batch_id)
|
||||
@@ -841,20 +868,29 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
|
||||
)
|
||||
return 0
|
||||
|
||||
@swagger_serializer_method(
|
||||
serializer_or_field=SnapshotRunMetadataSerializer(allow_null=True)
|
||||
)
|
||||
def get_meta(self, obj):
|
||||
artifact = self._get_artifact(obj)
|
||||
return (
|
||||
SnapshotRunMetadataSerializer(artifact.metadata).data
|
||||
if artifact is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@swagger_serializer_method(
|
||||
serializer_or_field=SnapshotMetadataSerializer(allow_null=True)
|
||||
)
|
||||
def get_snapshot(self, obj):
|
||||
metadata = snapshot_metadata(obj)
|
||||
return (
|
||||
SnapshotMetadataSerializer(metadata).data if metadata is not None else None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_artifact(obj) -> ParserSourceArtifact | None:
|
||||
if hasattr(obj, "_source_artifact_cache"):
|
||||
return obj._source_artifact_cache
|
||||
artifact = (
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source=obj.source,
|
||||
load_batch=obj.batch_id,
|
||||
)
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
)
|
||||
obj._source_artifact_cache = artifact
|
||||
return artifact
|
||||
return get_snapshot_artifact(obj)
|
||||
|
||||
def get_artifact_uid(self, obj) -> str | None:
|
||||
artifact = self._get_artifact(obj)
|
||||
@@ -945,16 +981,22 @@ class ParserLoadLogListSerializer(serializers.Serializer):
|
||||
|
||||
id = serializers.IntegerField(read_only=True)
|
||||
batch_id = serializers.IntegerField(read_only=True)
|
||||
source = serializers.CharField(read_only=True)
|
||||
source = serializers.ChoiceField(
|
||||
choices=[*ParserLoadLog.Source.values, *SOURCE_CARD_BY_SLUG], read_only=True
|
||||
)
|
||||
source_label = serializers.CharField(read_only=True, allow_null=True)
|
||||
records_count = serializers.IntegerField(read_only=True)
|
||||
organizations_count = serializers.IntegerField(read_only=True)
|
||||
artifact_uid = serializers.UUIDField(read_only=True, allow_null=True)
|
||||
meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True)
|
||||
snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True)
|
||||
parsed_count = serializers.IntegerField(read_only=True)
|
||||
published_count = serializers.IntegerField(read_only=True)
|
||||
quarantined_count = serializers.IntegerField(read_only=True)
|
||||
rejection_reasons = serializers.DictField(read_only=True)
|
||||
status = serializers.CharField(read_only=True)
|
||||
status = serializers.ChoiceField(
|
||||
choices=ParserLoadLog.Status.choices, read_only=True
|
||||
)
|
||||
status_label = serializers.CharField(read_only=True)
|
||||
error_message = serializers.CharField(read_only=True, allow_blank=True)
|
||||
created_at = serializers.DateTimeField(read_only=True)
|
||||
@@ -984,11 +1026,19 @@ class SourceCardRefreshParamSerializer(serializers.Serializer):
|
||||
class SourceCardLoadSerializer(serializers.Serializer):
|
||||
"""Сводка по последней загрузке источника."""
|
||||
|
||||
meta = SnapshotRunMetadataSerializer(read_only=True, allow_null=True)
|
||||
snapshot = SnapshotMetadataSerializer(read_only=True, allow_null=True)
|
||||
batch_id = serializers.IntegerField(read_only=True)
|
||||
source = serializers.CharField(read_only=True)
|
||||
source = serializers.ChoiceField(
|
||||
choices=ParserLoadLog.Source.choices, read_only=True
|
||||
)
|
||||
source_display = serializers.CharField(read_only=True)
|
||||
organizations_count = serializers.IntegerField(read_only=True)
|
||||
source_label = serializers.CharField(read_only=True)
|
||||
records_count = serializers.IntegerField(read_only=True)
|
||||
status = serializers.CharField(read_only=True)
|
||||
status = serializers.ChoiceField(
|
||||
choices=ParserLoadLog.Status.choices, read_only=True
|
||||
)
|
||||
error_message = serializers.CharField(read_only=True)
|
||||
created_at = serializers.DateTimeField(read_only=True)
|
||||
updated_at = serializers.DateTimeField(read_only=True)
|
||||
@@ -1002,7 +1052,7 @@ class SourceCardTaskSerializer(serializers.Serializer):
|
||||
status = serializers.CharField(read_only=True)
|
||||
progress = serializers.IntegerField(read_only=True)
|
||||
progress_message = serializers.CharField(read_only=True)
|
||||
started_at = serializers.DateTimeField(read_only=True)
|
||||
started_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||||
created_at = serializers.DateTimeField(read_only=True)
|
||||
meta = serializers.JSONField(read_only=True)
|
||||
|
||||
@@ -1046,6 +1096,11 @@ class SourceCardSerializer(serializers.Serializer):
|
||||
upload_url = serializers.CharField(read_only=True, allow_blank=True)
|
||||
refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True)
|
||||
|
||||
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
|
||||
source_items = SourceCardItemSerializer(many=True, read_only=True)
|
||||
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||||
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||||
|
||||
|
||||
class SourceTaskStatusSerializer(serializers.Serializer):
|
||||
"""Табличная строка статуса источника для экрана задач парсинга."""
|
||||
@@ -1065,12 +1120,7 @@ class SourceTaskStatusSerializer(serializers.Serializer):
|
||||
|
||||
|
||||
class SourceCardDetailSerializer(SourceCardSerializer):
|
||||
"""Detailed serializer for a single frontend source card."""
|
||||
|
||||
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
|
||||
source_items = SourceCardItemSerializer(many=True, read_only=True)
|
||||
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||||
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||||
"""List and detail deliberately share the complete card contract."""
|
||||
|
||||
|
||||
class SourceCardRefreshRequestSerializer(serializers.Serializer):
|
||||
@@ -1078,6 +1128,27 @@ class SourceCardRefreshRequestSerializer(serializers.Serializer):
|
||||
|
||||
params = serializers.DictField(required=False)
|
||||
|
||||
def to_internal_value(self, data):
|
||||
if isinstance(data, dict) and set(data) - set(self.fields):
|
||||
raise serializers.ValidationError({"detail": "Only params is supported."})
|
||||
return super().to_internal_value(data)
|
||||
|
||||
|
||||
class SroParserRunRequestSerializer(SourceCardRefreshRequestSerializer):
|
||||
mode = serializers.ChoiceField(
|
||||
choices=("incremental", "full"), required=False, default="incremental"
|
||||
)
|
||||
|
||||
def to_internal_value(self, data):
|
||||
if isinstance(data, dict) and data.get("mode", "incremental") not in (
|
||||
"incremental",
|
||||
"full",
|
||||
):
|
||||
raise APIValidationError(
|
||||
message="Допустимые режимы: incremental, full.", code="invalid_mode"
|
||||
)
|
||||
return super().to_internal_value(data)
|
||||
|
||||
|
||||
class SourceCardRefreshTaskSerializer(serializers.Serializer):
|
||||
"""Queued task info after refresh start."""
|
||||
@@ -1132,4 +1203,43 @@ class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
|
||||
|
||||
task_id = serializers.CharField(read_only=True, allow_null=True)
|
||||
task_ids = serializers.ListField(child=serializers.CharField(), read_only=True)
|
||||
status = serializers.CharField(read_only=True)
|
||||
status = serializers.ChoiceField(choices=("queued", "accepted"), read_only=True)
|
||||
|
||||
|
||||
class SourceRefreshConflictItemSerializer(serializers.Serializer):
|
||||
code = serializers.ChoiceField(
|
||||
choices=("refresh_already_running", "upstream_access_not_approved"),
|
||||
read_only=True,
|
||||
)
|
||||
message = serializers.CharField(read_only=True)
|
||||
|
||||
|
||||
class SourceRefreshConflictSerializer(serializers.Serializer):
|
||||
success = serializers.BooleanField(read_only=True)
|
||||
data = serializers.JSONField(read_only=True, allow_null=True)
|
||||
errors = SourceRefreshConflictItemSerializer(many=True, read_only=True)
|
||||
meta = serializers.DictField(read_only=True, allow_null=True)
|
||||
|
||||
|
||||
class ParserDashboardGroupsSerializer(serializers.Serializer):
|
||||
api = ParserSourceSerializer(many=True, read_only=True)
|
||||
uploads = ParserSourceSerializer(many=True, read_only=True)
|
||||
|
||||
|
||||
class ParserDashboardSerializer(serializers.Serializer):
|
||||
sources = ParserSourceSerializer(many=True, read_only=True)
|
||||
api_sources = ParserSourceSerializer(many=True, read_only=True)
|
||||
file_sources = ParserSourceSerializer(many=True, read_only=True)
|
||||
groups = ParserDashboardGroupsSerializer(read_only=True)
|
||||
schedules = ParserScheduleSerializer(many=True, read_only=True)
|
||||
jobs = BackgroundJobListSerializer(many=True, read_only=True)
|
||||
source_counts = serializers.DictField(
|
||||
child=serializers.IntegerField(), read_only=True
|
||||
)
|
||||
registry_data_coverage = serializers.DictField(read_only=True)
|
||||
registry_enrichment_analytics = serializers.DictField(read_only=True)
|
||||
load_logs = ParserLoadLogSerializer(many=True, read_only=True)
|
||||
|
||||
|
||||
class ParserDashboardResponseSerializer(FrontendApiResponseSerializer):
|
||||
data = ParserDashboardSerializer(read_only=True)
|
||||
|
||||
Reference in New Issue
Block a user