feat: import gosedo and media news sources

tests/apps/exchange/test_api.py

@@ -20,6 +20,7 @@ from apps.external_data.models import (
     ArbitrationCase,
     BankruptcyProcedure,
     DefenseUnreliableSupplier,
+    ElectronicDocumentExchangeEntry,
     FinancialReport,
     FinancialReportLine,
     IndustrialCertificate,
@@ -27,6 +28,7 @@ from apps.external_data.models import (
     InformationSecurityRegistryEntry,
     LaborVacancy,
     ManufacturerRegistryEntry,
+    MediaMention,
     ProsecutorCheck,
     PublicProcurement,
 )
@@ -58,6 +60,18 @@ def build_exchange_archive(
     schema_version: int = ExchangePackageImportService.SUPPORTED_SCHEMA_VERSION,
 ) -> SimpleUploadedFile:
     """Build encrypted exchange archive compatible with import service."""
+    provided_data = data or {}
+    normalized_data = {
+        section: provided_data.get(section, [])
+        for section in ExchangePackageImportService.SECTION_KEYS
+    }
+    normalized_data.update(
+        {
+            section: rows
+            for section, rows in provided_data.items()
+            if section not in normalized_data
+        }
+    )
     payload = {
         "format": ExchangePackageImportService.PAYLOAD_FORMAT,
         "schema_version": schema_version,
@@ -66,9 +80,9 @@ def build_exchange_archive(
             "source_system": "mostovik-dev",
             "produced_at": "2026-04-07T12:00:00+00:00",
             "schema_version": schema_version,
-            "sections": list((data or {}).keys()),
+            "sections": list(ExchangePackageImportService.SECTION_KEYS),
         },
-        "data": data or {},
+        "data": normalized_data,
     }
     payload_bytes = json.dumps(
         payload,
@@ -369,6 +383,41 @@ def build_exchange_payload() -> dict[str, list[dict[str, object]]]:
                 "source_url": "https://trudvsem.ru/vacancy/001",
             }
         ],
+        "electronic_document_exchange": [
+            {
+                "organization_inn": "7707083893",
+                "external_id": "gosedo:001",
+                "record_type": "participant",
+                "title": "АО Альфа",
+                "status": "active",
+                "participant_status": "active",
+                "attestation_status": "attested",
+                "registration_type": "ogrn",
+                "registration_number": "1027700132195",
+                "source_version": "633",
+                "source_published_at": "2026-08-10",
+                "source_row_class": "pt-on",
+                "responsible_phones": [],
+                "responsible_emails": [],
+                "extra_fields": {},
+                "load_batch": 8,
+            }
+        ],
+        "media_mentions": [
+            {
+                "organization_inn": "7707083893",
+                "external_id": "a" * 64,
+                "news_source": "Тестовое СМИ",
+                "title": "АО Альфа расширяет производство",
+                "sentiment": "positive",
+                "published_at": "2026-07-01",
+                "source_url": "https://example.test/news/1",
+                "excerpt_lines": ["СМИ", "Заголовок", "Лид", "Продолжение"],
+                "full_text": "СМИ\nЗаголовок\nЛид\nПродолжение\nПолный текст",
+                "okpo": "12345678",
+                "load_batch": 9,
+            }
+        ],
     }
 
 
@@ -481,6 +530,8 @@ class ExchangePackageApiTest(APITestCase):
         self.assertEqual(DefenseUnreliableSupplier.objects.count(), 1)
         self.assertEqual(InformationSecurityRegistryEntry.objects.count(), 1)
         self.assertEqual(LaborVacancy.objects.count(), 1)
+        self.assertEqual(ElectronicDocumentExchangeEntry.objects.count(), 1)
+        self.assertEqual(MediaMention.objects.count(), 1)
         self.assertEqual(
             response.data["result"]["bankruptcy_procedures"]["created"],
             1,
@@ -499,6 +550,11 @@ class ExchangePackageApiTest(APITestCase):
             1,
         )
         self.assertEqual(response.data["result"]["labor_vacancies"]["created"], 1)
+        self.assertEqual(
+            response.data["result"]["electronic_document_exchange"]["created"],
+            1,
+        )
+        self.assertEqual(response.data["result"]["media_mentions"]["created"], 1)
 
         organization = Organization.objects.get(inn="7707083893")
         self.assertEqual(organization.name, "АО Альфа Обновленная")
@@ -634,7 +690,7 @@ class ExchangePackageApiTest(APITestCase):
     def test_upload_rejects_legacy_schema_version(self):
         archive = build_exchange_archive(
             package_id="pkg-legacy-schema-version",
-            schema_version=2,
+            schema_version=3,
             data=build_exchange_payload(),
         )
 
@@ -646,7 +702,7 @@ class ExchangePackageApiTest(APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn("schema_version 3", response.data["file"][0])
+        self.assertIn("schema_version 4", response.data["file"][0])
         self.assertEqual(Organization.objects.count(), 0)
 
     def test_upload_rejects_legacy_registry_memberships_section(self):
@@ -697,7 +753,7 @@ class ExchangePackageApiTest(APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn("schema_version 3", response.data["file"][0])
+        self.assertIn("schema_version 4", response.data["file"][0])
         self.assertEqual(Organization.objects.count(), 0)
 
     def test_upload_rejects_external_rows_for_organization_absent_from_package(self):
@@ -769,6 +825,61 @@ class ExchangePackageApiTest(APITestCase):
         duplicate_import = ExchangePackageImport.objects.order_by("-created_at").first()
         self.assertIsNotNone(duplicate_import.duplicate_of)
 
+    def test_v4_gosedo_is_scoped_replacement_and_media_is_upsert_only(self):
+        first_archive = build_exchange_archive(
+            package_id="pkg-v4-first",
+            data=build_exchange_payload(),
+        )
+        first_response = self.client.post(
+            self.url,
+            {"file": first_archive},
+            format="multipart",
+            HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN,
+        )
+        self.assertEqual(first_response.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(ElectronicDocumentExchangeEntry.objects.count(), 1)
+        self.assertEqual(MediaMention.objects.count(), 1)
+        outside_scope = Organization.objects.create(
+            inn="7707083999",
+            name="АО Вне пакета",
+        )
+        ElectronicDocumentExchangeEntry.objects.create(
+            organization=outside_scope,
+            external_id="outside-scope",
+            record_type=ElectronicDocumentExchangeEntry.RecordType.PARTICIPANT,
+            title=outside_scope.name,
+            source_version="632",
+            source_published_at="2026-08-01",
+        )
+
+        second_payload = build_exchange_payload()
+        second_payload["electronic_document_exchange"] = []
+        second_payload["media_mentions"] = []
+        second_archive = build_exchange_archive(
+            package_id="pkg-v4-second",
+            data=second_payload,
+        )
+        second_response = self.client.post(
+            self.url,
+            {"file": second_archive},
+            format="multipart",
+            HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN,
+        )
+
+        self.assertEqual(second_response.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(
+            second_response.data["result"]["electronic_document_exchange"]["deleted"],
+            1,
+        )
+        self.assertEqual(ElectronicDocumentExchangeEntry.objects.count(), 1)
+        self.assertTrue(
+            ElectronicDocumentExchangeEntry.objects.filter(
+                organization=outside_scope,
+                external_id="outside-scope",
+            ).exists()
+        )
+        self.assertEqual(MediaMention.objects.count(), 1)
+
     def test_upload_replaces_stale_financial_report_lines(self):
         first_payload = build_exchange_payload()
         first_archive = build_exchange_archive(

tests/apps/external_data/test_api.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from datetime import date
 
+from apps.external_data.models import ElectronicDocumentExchangeEntry, MediaMention
 from django.test import override_settings
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -212,3 +213,47 @@ class ExternalDataApiTest(APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data["count"], 1)
         self.assertEqual(response.data["results"][0]["lines"][0]["line_code"], "1600")
+
+    def test_gosedo_and_media_endpoints_filter_and_separate_full_text(self):
+        ElectronicDocumentExchangeEntry.objects.create(
+            organization=self.organization,
+            external_id="gosedo-1",
+            record_type="participant",
+            title="АО Тест",
+            status="active",
+            attestation_status="attested",
+            source_version="633",
+            source_published_at=date(2026, 8, 10),
+        )
+        mention = MediaMention.objects.create(
+            organization=self.organization,
+            external_id="a" * 64,
+            news_source="Тестовое СМИ",
+            title="Заголовок",
+            sentiment="positive",
+            published_at=date(2026, 7, 1),
+            excerpt_lines=["1", "2", "3", "4"],
+            full_text="1\n2\n3\n4\n5",
+        )
+
+        gosedo_response = self.client.get(
+            f"/api/v1/electronic-document-exchange/?organization={self.organization.id}"
+            "&record_type=participant&attestation_status=attested&source_version=633"
+            "&source_published_at_from=2026-08-01&source_published_at_to=2026-08-31"
+        )
+        media_list_response = self.client.get(
+            f"/api/v1/media-mentions/?organization={self.organization.id}"
+            "&source=Тестовое СМИ&sentiment=positive"
+        )
+        media_detail_response = self.client.get(f"/api/v1/media-mentions/{mention.id}/")
+
+        self.assertEqual(gosedo_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(gosedo_response.data["count"], 1)
+        self.assertEqual(media_list_response.status_code, status.HTTP_200_OK)
+        self.assertNotIn("full_text", media_list_response.data["results"][0])
+        self.assertEqual(
+            media_list_response.data["results"][0]["excerpt_lines"],
+            ["1", "2", "3", "4"],
+        )
+        self.assertEqual(media_detail_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(media_detail_response.data["full_text"], "1\n2\n3\n4\n5")

tests/apps/external_data/test_export_tasks.py

@@ -36,7 +36,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase):
         result = refresh_source_record_export_artifacts()
 
         self.assertEqual(result["status"], "success")
-        self.assertEqual(result["artifacts_count"], 25)
+        self.assertEqual(result["artifacts_count"], 31)
         self.assertEqual(result["export_year"], timezone.localdate().year)
         self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))
 

tests/apps/external_data/test_source_record_export.py

@@ -7,6 +7,7 @@ from io import BytesIO, StringIO
 from tempfile import TemporaryDirectory
 from unittest.mock import patch
 
+from apps.external_data.models import MediaMention
 from apps.external_data.source_record_export import (
     ORGANIZATION_EXPORT_FIELDS,
     SOURCE_GROUP_EXPORT_SPECS,
@@ -105,8 +106,8 @@ class SourceRecordExportApiTest(APITestCase):
 
         generation = build_source_record_export_artifacts()
 
-        self.assertEqual(generation.artifacts_count, 25)
-        self.assertEqual(generation.files_count, 25)
+        self.assertEqual(generation.artifacts_count, 31)
+        self.assertEqual(generation.files_count, 31)
         self.assertEqual(generation.records_count, 6)
         self.assertEqual(generation.export_year, current_date.year)
         expected_prefix = [*ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS]
@@ -193,6 +194,38 @@ class SourceRecordExportApiTest(APITestCase):
             ],
         )
 
+    def test_media_mentions_export_keeps_all_history_and_full_text(self):
+        organization = OrganizationFactory.create(okpo="00123456")
+        mention = MediaMention.objects.create(
+            organization=organization,
+            external_id="a" * 64,
+            news_source="Архивное СМИ",
+            title="Старая публикация",
+            sentiment=MediaMention.Sentiment.POSITIVE,
+            published_at="2023-02-15",
+            excerpt_lines=["Строка 1", "Строка 2"],
+            full_text="Строка 1\nСтрока 2\nПолная история",
+            okpo=organization.okpo,
+        )
+
+        generation = build_source_record_export_artifacts(
+            now=datetime(2026, 8, 4, 6, 0, tzinfo=UTC)
+        )
+
+        media_path = next(
+            artifact.path
+            for artifact in generation.artifacts
+            if artifact.source_group == "media_mentions"
+            and artifact.file_format == "json"
+        )
+        rows = json.loads(media_path.read_text(encoding="utf-8"))
+        exported = next(row for row in rows if row["uid"] == str(mention.id))
+        self.assertEqual(exported["record_date"], "2023-02-15")
+        self.assertEqual(
+            exported["payload.full_text"],
+            "Строка 1\nСтрока 2\nПолная история",
+        )
+
     def test_generation_keeps_provider_records_but_hides_provider_mentions(self):
         organization = OrganizationFactory.create(okpo="11223344")
         excluded_record = BankruptcyProcedureFactory.create(
@@ -257,9 +290,9 @@ class SourceRecordExportApiTest(APITestCase):
         call_command("build_source_record_exports", stdout=command_output)
 
         generation = load_current_source_record_export_generation()
-        self.assertEqual(generation.artifacts_count, 25)
+        self.assertEqual(generation.artifacts_count, 31)
         self.assertEqual(generation.export_year, timezone.localdate().year)
-        self.assertIn('"artifacts_count": 25', command_output.getvalue())
+        self.assertIn('"artifacts_count": 31', command_output.getvalue())
 
     def test_generation_contains_only_records_from_its_calendar_year(self):
         export_year = 2026