feat: add gosedo and media news sources
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 1m10s
CI/CD Pipeline / Build and Push Images (push) Successful in 19s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 24s

This commit is contained in:
2026-08-13 14:12:53 +02:00
parent 1c2dd985d4
commit a0eb0a5303
29 changed files with 3228 additions and 85 deletions

View File

@@ -19,6 +19,7 @@ from apps.parsers.models import (
InspectionRecord,
ManufacturerRecord,
ParserLoadLog,
ParserSourceArtifact,
ParsingSettings,
ProcurementRecord,
)
@@ -524,6 +525,8 @@ class ParserSourceSerializer(serializers.Serializer):
result_list_url = serializers.CharField()
result_detail_url = serializers.CharField()
upload_url = serializers.CharField(allow_blank=True)
supports_refresh = serializers.BooleanField()
admin_only = serializers.BooleanField()
class ParserRunRequestSerializer(serializers.Serializer):
@@ -729,6 +732,7 @@ class ParserRunResponseSerializer(serializers.Serializer):
"""Ответ API на запуск задачи."""
task_id = serializers.CharField()
task_ids = serializers.ListField(child=serializers.CharField())
source = serializers.CharField()
task_name = serializers.CharField()
@@ -747,6 +751,11 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
source_display = serializers.CharField(source="get_source_display", read_only=True)
organizations_count = serializers.SerializerMethodField()
artifact_uid = serializers.SerializerMethodField()
parsed_count = serializers.SerializerMethodField()
published_count = serializers.SerializerMethodField()
quarantined_count = serializers.SerializerMethodField()
rejection_reasons = serializers.SerializerMethodField()
class Meta:
model = ParserLoadLog
@@ -757,6 +766,11 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
"source_display",
"records_count",
"organizations_count",
"artifact_uid",
"parsed_count",
"published_count",
"quarantined_count",
"rejection_reasons",
"status",
"error_message",
"created_at",
@@ -765,6 +779,17 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
read_only_fields = fields
def get_organizations_count(self, obj) -> int:
artifact = self._get_artifact(obj)
if artifact is not None:
return (
artifact.staged_records.filter(
disposition="published",
organization__isnull=False,
)
.values("organization_id")
.distinct()
.count()
)
if obj.source == ParserLoadLog.Source.FNS_REPORTS:
return (
FinancialReport.objects.filter(load_batch=obj.batch_id)
@@ -815,6 +840,41 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
)
return 0
@staticmethod
def _get_artifact(obj) -> ParserSourceArtifact | None:
if hasattr(obj, "_source_artifact_cache"):
return obj._source_artifact_cache
artifact = (
ParserSourceArtifact.objects.filter(
source=obj.source,
load_batch=obj.batch_id,
)
.order_by("-created_at")
.first()
)
obj._source_artifact_cache = artifact
return artifact
def get_artifact_uid(self, obj) -> str | None:
artifact = self._get_artifact(obj)
return str(artifact.uid) if artifact is not None else None
def get_parsed_count(self, obj) -> int:
artifact = self._get_artifact(obj)
return artifact.parsed_count if artifact is not None else obj.records_count
def get_published_count(self, obj) -> int:
artifact = self._get_artifact(obj)
return artifact.published_count if artifact is not None else obj.records_count
def get_quarantined_count(self, obj) -> int:
artifact = self._get_artifact(obj)
return artifact.quarantined_count if artifact is not None else 0
def get_rejection_reasons(self, obj) -> dict:
artifact = self._get_artifact(obj)
return dict(artifact.rejection_reasons) if artifact is not None else {}
class GenericParserRecordSerializer(
CanonicalOrganizationEnrichmentMixin,
@@ -888,6 +948,11 @@ class ParserLoadLogListSerializer(serializers.Serializer):
source_label = serializers.CharField(read_only=True, allow_null=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
artifact_uid = serializers.UUIDField(read_only=True, allow_null=True)
parsed_count = serializers.IntegerField(read_only=True)
published_count = serializers.IntegerField(read_only=True)
quarantined_count = serializers.IntegerField(read_only=True)
rejection_reasons = serializers.DictField(read_only=True)
status = serializers.CharField(read_only=True)
status_label = serializers.CharField(read_only=True)
error_message = serializers.CharField(read_only=True, allow_blank=True)
@@ -953,6 +1018,7 @@ class SourceCardItemSerializer(serializers.Serializer):
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True)
upload_url = serializers.CharField(read_only=True, allow_blank=True)
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
@@ -975,6 +1041,8 @@ class SourceCardSerializer(serializers.Serializer):
error_message = serializers.CharField(read_only=True)
task_names = serializers.ListField(child=serializers.CharField(), read_only=True)
refresh_requires_params = serializers.BooleanField(read_only=True)
supports_refresh = serializers.BooleanField(read_only=True)
upload_url = serializers.CharField(read_only=True, allow_blank=True)
refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True)
@@ -1056,4 +1124,5 @@ class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
"""Минимальный ответ запуска обновления карточки по md."""
task_id = serializers.CharField(read_only=True, allow_null=True)
task_ids = serializers.ListField(child=serializers.CharField(), read_only=True)
status = serializers.CharField(read_only=True)