fix: align source exports with mostovik

docs/source-record-export-matrix-ru.md

@@ -36,10 +36,12 @@ JavaScript. Ticket не попадает в URL и после первого з
 Итого формируется 25 логических артефактов. Финансовые показатели всегда
 выгружаются в JSON с вложенным массивом `financial_lines`. Промышленная группа
 объединяет три таблицы, а поле `record_type` различает тип строки. Все строки
-содержат реквизиты организации, включая ОКПО.
+используют тот же контракт, что и Mostovik: реквизиты организации, включая ОКПО,
+общие поля записи источника и специфичные поля в `payload.*`.
 
-Записи технического внешнего поставщика в публичные файлы не включаются; в БД
-они сохраняются для работы интеграции и дедупликации.
+Все записи включаются в публичные файлы, а техническое наименование внешнего
+поставщика нейтрализуется. Исходные значения в БД сохраняются для работы
+интеграции и дедупликации.
 
 Физических XLSX-файлов может быть больше: по умолчанию один файл содержит не
 более 100 000 строк данных и получает суффикс `-part-001`, `-part-002` и далее.

src/apps/external_data/source_record_export.py

@@ -54,7 +54,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000
 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60
 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "external-data:source-record-exports:ticket"
 SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}")
-EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile(
+EXPORT_PROVIDER_URL_PATTERN = re.compile(
+    r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*",
+    flags=re.IGNORECASE,
+)
+EXPORT_PROVIDER_NAME_PATTERN = re.compile(
     r"(?<![A-Za-zА-Яа-яЁё0-9])(?:checko(?:\.ru)?|чекало|чекко|чеко)"
     r"(?![A-Za-zА-Яа-яЁё0-9])",
     flags=re.IGNORECASE,
@@ -66,12 +70,19 @@ ORGANIZATION_EXPORT_FIELDS = [
     "ОГРН",
     "КПП",
     "ОКПО",
-    "organization",
 ]
 SOURCE_RECORD_EXPORT_FIELDS = [
     "uid",
     "source_group",
+    "source",
     "record_type",
+    "external_id",
+    "title",
+    "record_date",
+    "amount",
+    "status",
+    "url",
+    "load_batch",
     "created_at",
     "updated_at",
 ]
@@ -93,6 +104,18 @@ class SourceModelExportSpec:
     model: Any
     record_type: str
     fields: tuple[str, ...]
+    source: str
+    source_field: str | None = None
+    external_id_field: str | None = None
+    title_field: str | None = None
+    record_date_field: str | None = None
+    amount_field: str | None = None
+    status_field: str | None = None
+    url_field: str | None = None
+    load_batch_field: str | None = None
+    title_prefix: str = ""
+    payload_aliases: tuple[tuple[str, str], ...] = ()
+    payload_export_fields: tuple[str, ...] | None = None
     prefetch_related: tuple[str, ...] = ()
 
 
@@ -123,6 +146,12 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "source",
                     "error_message",
                 ),
+                source=FINANCIAL_SOURCE_GROUP,
+                source_field="source",
+                external_id_field="external_id",
+                title_field="file_name",
+                status_field="status",
+                load_batch_field="load_batch",
                 prefetch_related=("lines",),
             ),
         ),
@@ -144,6 +173,12 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "execution_end_date",
                     "purchase_name",
                 ),
+                source="government_procurements",
+                external_id_field="purchase_number",
+                title_field="purchase_name",
+                record_date_field="contract_date",
+                amount_field="contract_amount",
+                status_field="status",
             ),
         ),
     ),
@@ -161,6 +196,9 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "tnved_code",
                     "registry_number",
                 ),
+                source="industrial_production",
+                external_id_field="registry_number",
+                title_field="product_name",
             ),
             SourceModelExportSpec(
                 model=IndustrialCertificate,
@@ -173,11 +211,20 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "organisation_name",
                     "ogrn",
                 ),
+                source="industrial_production",
+                external_id_field="certificate_number",
+                title_field="organisation_name",
+                record_date_field="issue_date",
+                url_field="certificate_file_url",
+                payload_aliases=(("certificate_file_url", "url"),),
             ),
             SourceModelExportSpec(
                 model=ManufacturerRegistryEntry,
                 record_type="manufacturer_registry_entry",
                 fields=("full_legal_name", "inn", "ogrn", "address"),
+                source="industrial_production",
+                external_id_field="ogrn",
+                title_field="full_legal_name",
             ),
         ),
     ),
@@ -196,6 +243,11 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "start_date",
                     "status",
                 ),
+                source="planned_inspections",
+                external_id_field="registration_number",
+                title_field="control_authority",
+                record_date_field="start_date",
+                status_field="status",
             ),
         ),
     ),
@@ -214,6 +266,26 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "status",
                     "source_url",
                 ),
+                source="bankruptcy",
+                external_id_field="external_id",
+                title_field="message_type",
+                record_date_field="message_date",
+                status_field="status",
+                url_field="source_url",
+                payload_aliases=(("source_url", "url"),),
+                payload_export_fields=(
+                    "inn",
+                    "url",
+                    "ogrn",
+                    "source",
+                    "messages",
+                    "load_batch",
+                    "case_number",
+                    "message_date",
+                    "message_type",
+                    "messages_count",
+                    "organisation_name",
+                ),
             ),
         ),
     ),
@@ -234,6 +306,14 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "status",
                     "source_url",
                 ),
+                source="defense_suppliers",
+                source_field="registry_source",
+                external_id_field="external_id",
+                title_field="supplier_name",
+                record_date_field="included_at",
+                status_field="status",
+                url_field="source_url",
+                payload_aliases=(("source_url", "url"),),
             ),
         ),
     ),
@@ -251,6 +331,26 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "status",
                     "decision_date",
                 ),
+                source="arbitration",
+                external_id_field="case_number",
+                title_field="case_number",
+                record_date_field="decision_date",
+                status_field="status",
+                title_prefix="Дело ",
+                payload_aliases=(("court_name", "court"), ("party_role", "role")),
+                payload_export_fields=(
+                    "inn",
+                    "url",
+                    "ogrn",
+                    "role",
+                    "court",
+                    "source",
+                    "status",
+                    "load_batch",
+                    "case_number",
+                    "claim_amount",
+                    "organisation_name",
+                ),
             ),
         ),
     ),
@@ -269,6 +369,11 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "issued_at",
                     "expires_at",
                 ),
+                source="security_registries",
+                external_id_field="external_id",
+                title_field="registry_name",
+                record_date_field="issued_at",
+                status_field="presence_status",
             ),
         ),
     ),
@@ -288,6 +393,15 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "salary_amount",
                     "source_url",
                 ),
+                source="vacancies",
+                source_field="vacancy_source",
+                external_id_field="external_id",
+                title_field="title",
+                record_date_field="published_at",
+                amount_field="salary_amount",
+                status_field="status",
+                url_field="source_url",
+                payload_aliases=(("source_url", "url"),),
             ),
         ),
     ),
@@ -717,17 +831,19 @@ def _iter_source_model_records(
 
 
 def _source_group_headers(source_spec: SourceGroupExportSpec) -> list[str]:
-    model_fields: list[str] = []
+    payload_fields: list[str] = []
     for model_spec in source_spec.models:
-        for field_name in model_spec.fields:
-            if field_name not in model_fields:
-                model_fields.append(field_name)
+        for field_name in _payload_export_field_names(model_spec):
+            payload_field_name = f"payload.{field_name}"
+            if payload_field_name not in payload_fields:
+                payload_fields.append(payload_field_name)
+    payload_fields.sort()
     if source_spec.source_group == FINANCIAL_SOURCE_GROUP:
-        model_fields.append("financial_lines")
+        payload_fields.append("financial_lines")
     return [
         *ORGANIZATION_EXPORT_FIELDS,
         *SOURCE_RECORD_EXPORT_FIELDS,
-        *model_fields,
+        *payload_fields,
     ]
 
 
@@ -742,8 +858,6 @@ def _spool_source_group_rows(
         is_first_row = True
         for model_spec in source_spec.models:
             for record in _iter_source_model_records(model_spec):
-                if _is_excluded_export_provider_record(record, model_spec=model_spec):
-                    continue
                 row = _build_record_row(
                     record,
                     source_spec=source_spec,
@@ -899,13 +1013,28 @@ def _build_record_row(
         "ОГРН": organization.ogrn,
         "КПП": organization.kpp,
         "ОКПО": organization.okpo,
-        "organization": organization.id,
         "uid": record.id,
         "source_group": source_spec.source_group,
+        "source": _source_value(record, model_spec=model_spec),
         "record_type": model_spec.record_type,
+        "external_id": _record_field_value(record, model_spec.external_id_field),
+        "title": _title_value(record, model_spec=model_spec),
+        "record_date": _record_field_value(record, model_spec.record_date_field),
+        "amount": _record_field_value(record, model_spec.amount_field),
+        "status": _record_field_value(record, model_spec.status_field),
+        "url": _record_field_value(record, model_spec.url_field),
+        "load_batch": _record_field_value(record, model_spec.load_batch_field),
         "created_at": record.created_at,
         "updated_at": record.updated_at,
-        **{field_name: getattr(record, field_name) for field_name in model_spec.fields},
+        **{
+            f"payload.{field_name}": _payload_field_value(
+                record,
+                organization=organization,
+                model_spec=model_spec,
+                field_name=field_name,
+            )
+            for field_name in _payload_export_field_names(model_spec)
+        },
     }
     if source_spec.source_group == FINANCIAL_SOURCE_GROUP:
         row["financial_lines"] = [
@@ -915,33 +1044,78 @@ def _build_record_row(
             }
             for line in record.lines.all()
         ]
-    return {key: _serialize_json_value(value) for key, value in row.items()}
+    public_row = _sanitize_export_provider_value(row)
+    return {key: _serialize_json_value(value) for key, value in public_row.items()}
 
 
-def _is_excluded_export_provider_record(
+def _source_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any:
+    source_value = _record_field_value(record, model_spec.source_field)
+    return source_value or model_spec.source
+
+
+def _title_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any:
+    title_value = _record_field_value(record, model_spec.title_field)
+    if not title_value or not model_spec.title_prefix:
+        return title_value
+    return f"{model_spec.title_prefix}{title_value}"
+
+
+def _payload_field_name(model_spec: SourceModelExportSpec, field_name: str) -> str:
+    return dict(model_spec.payload_aliases).get(field_name, field_name)
+
+
+def _payload_export_field_names(model_spec: SourceModelExportSpec) -> tuple[str, ...]:
+    if model_spec.payload_export_fields is not None:
+        return model_spec.payload_export_fields
+
+    field_names = ["inn", "ogrn", "organisation_name", "source"]
+    for field_name in model_spec.fields:
+        payload_field_name = _payload_field_name(model_spec, field_name)
+        if payload_field_name not in field_names:
+            field_names.append(payload_field_name)
+    return tuple(field_names)
+
+
+def _payload_field_value(
     record: Any,
     *,
+    organization: Any,
     model_spec: SourceModelExportSpec,
-) -> bool:
-    """Return whether a provider-backed record must stay out of public files."""
-    return any(
-        _contains_excluded_export_provider(getattr(record, field_name))
-        for field_name in model_spec.fields
-    )
+    field_name: str,
+) -> Any:
+    if field_name == "inn":
+        return organization.inn
+    if field_name == "ogrn":
+        return organization.ogrn
+    if field_name == "organisation_name":
+        return organization.full_name or organization.short_name or organization.name
+    if field_name == "source":
+        return _source_value(record, model_spec=model_spec)
+
+    source_field_by_alias = {
+        alias: source_field for source_field, alias in model_spec.payload_aliases
+    }
+    source_field = source_field_by_alias.get(field_name, field_name)
+    return getattr(record, source_field, None)
 
 
-def _contains_excluded_export_provider(value: object) -> bool:
+def _record_field_value(record: Any, field_name: str | None) -> Any:
+    return getattr(record, field_name) if field_name else None
+
+
+def _sanitize_export_provider_value(value: Any) -> Any:
+    """Remove provider wording from files without changing stored source data."""
     if isinstance(value, str):
-        return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None
+        sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value)
+        return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized)
     if isinstance(value, dict):
-        return any(
-            _contains_excluded_export_provider(key)
-            or _contains_excluded_export_provider(item)
+        return {
+            _sanitize_export_provider_value(key): _sanitize_export_provider_value(item)
             for key, item in value.items()
-        )
+        }
     if isinstance(value, list | tuple):
-        return any(_contains_excluded_export_provider(item) for item in value)
-    return False
+        return [_sanitize_export_provider_value(item) for item in value]
+    return value
 
 
 def _serialize_flat_value(value: Any) -> str | int | float | bool:

tests/apps/external_data/test_source_record_export.py

@@ -6,6 +6,10 @@ from io import BytesIO, StringIO
 from tempfile import TemporaryDirectory
 
 from apps.external_data.source_record_export import (
+    ORGANIZATION_EXPORT_FIELDS,
+    SOURCE_GROUP_EXPORT_SPECS,
+    SOURCE_RECORD_EXPORT_FIELDS,
+    _source_group_headers,
     build_source_record_export_artifacts,
     load_current_source_record_export_generation,
 )
@@ -16,6 +20,7 @@ from rest_framework import status
 from rest_framework.test import APITestCase
 
 from tests.apps.external_data.factories import (
+    ArbitrationCaseFactory,
     BankruptcyProcedureFactory,
     FinancialReportFactory,
     FinancialReportLineFactory,
@@ -74,6 +79,7 @@ class SourceRecordExportApiTest(APITestCase):
         IndustrialCertificateFactory.create(organization=organization)
         ManufacturerRegistryEntryFactory.create(organization=organization)
         ProsecutorCheckFactory.create(organization=organization)
+        arbitration_case = ArbitrationCaseFactory.create(organization=organization)
         report = FinancialReportFactory.create(organization=organization)
         FinancialReportLineFactory.create(report=report, line_code="1600")
 
@@ -81,7 +87,13 @@ class SourceRecordExportApiTest(APITestCase):
 
         self.assertEqual(generation.artifacts_count, 25)
         self.assertEqual(generation.files_count, 25)
-        self.assertEqual(generation.records_count, 5)
+        self.assertEqual(generation.records_count, 6)
+        expected_prefix = [*ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS]
+        for source_spec in SOURCE_GROUP_EXPORT_SPECS.values():
+            self.assertEqual(
+                _source_group_headers(source_spec)[: len(expected_prefix)],
+                expected_prefix,
+            )
         industrial_path = next(
             artifact.path
             for artifact in generation.artifacts
@@ -98,6 +110,12 @@ class SourceRecordExportApiTest(APITestCase):
             },
         )
         self.assertEqual({row["ОКПО"] for row in industrial_rows}, {"12345678"})
+        self.assertEqual(
+            list(industrial_rows[0])[: len(expected_prefix)],
+            expected_prefix,
+        )
+        self.assertNotIn("organization", industrial_rows[0])
+        self.assertIn("payload.product_name", industrial_rows[0])
 
         financial_path = next(
             artifact.path
@@ -107,7 +125,46 @@ class SourceRecordExportApiTest(APITestCase):
         financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
         self.assertEqual(financial_rows[0]["financial_lines"][0]["line_code"], "1600")
 
-    def test_generation_excludes_provider_records_and_keeps_okpo(self):
+        arbitration_path = next(
+            artifact.path
+            for artifact in generation.artifacts
+            if artifact.source_group == "arbitration" and artifact.file_format == "json"
+        )
+        arbitration_row = json.loads(arbitration_path.read_text(encoding="utf-8"))[0]
+        self.assertEqual(arbitration_row["source"], "arbitration")
+        self.assertEqual(
+            arbitration_row["external_id"], arbitration_row["payload.case_number"]
+        )
+        self.assertEqual(
+            arbitration_row["title"],
+            f"Дело {arbitration_row['payload.case_number']}",
+        )
+        self.assertEqual(
+            arbitration_row["record_date"], arbitration_case.decision_date.isoformat()
+        )
+        self.assertEqual(arbitration_row["status"], arbitration_row["payload.status"])
+        self.assertEqual(arbitration_row["payload.role"], "defendant")
+        self.assertIn("payload.court", arbitration_row)
+        self.assertEqual(arbitration_row["payload.source"], "arbitration")
+        self.assertEqual(arbitration_row["payload.inn"], organization.inn)
+        self.assertEqual(
+            [key for key in arbitration_row if key.startswith("payload.")],
+            [
+                "payload.inn",
+                "payload.url",
+                "payload.ogrn",
+                "payload.role",
+                "payload.court",
+                "payload.source",
+                "payload.status",
+                "payload.load_batch",
+                "payload.case_number",
+                "payload.claim_amount",
+                "payload.organisation_name",
+            ],
+        )
+
+    def test_generation_keeps_provider_records_but_hides_provider_mentions(self):
         organization = OrganizationFactory.create(okpo="11223344")
         excluded_record = BankruptcyProcedureFactory.create(
             organization=organization,
@@ -122,7 +179,7 @@ class SourceRecordExportApiTest(APITestCase):
 
         generation = build_source_record_export_artifacts()
 
-        self.assertEqual(generation.records_count, 1)
+        self.assertEqual(generation.records_count, 2)
         artifacts = [
             artifact
             for artifact in generation.artifacts
@@ -134,14 +191,36 @@ class SourceRecordExportApiTest(APITestCase):
         for artifact in artifacts:
             if artifact.file_format == "json":
                 exported_values = json.loads(artifact.path.read_text(encoding="utf-8"))
+                self.assertEqual(
+                    [key for key in exported_values[0] if key.startswith("payload.")],
+                    [
+                        "payload.inn",
+                        "payload.url",
+                        "payload.ogrn",
+                        "payload.source",
+                        "payload.messages",
+                        "payload.load_batch",
+                        "payload.case_number",
+                        "payload.message_date",
+                        "payload.message_type",
+                        "payload.messages_count",
+                        "payload.organisation_name",
+                    ],
+                )
             elif artifact.file_format == "csv":
                 exported_values = artifact.path.read_text(encoding="utf-8-sig")
             else:
                 workbook = load_workbook(artifact.path, read_only=True)
                 exported_values = list(workbook["data"].iter_rows(values_only=True))
-            self.assertNotIn(str(excluded_record.id), str(exported_values))
+            serialized_values = str(exported_values)
+            self.assertIn(str(excluded_record.id), serialized_values)
             self.assertIn(str(included_record.id), str(exported_values))
             self.assertIn("11223344", str(exported_values))
+            self.assertNotRegex(
+                serialized_values,
+                r"(?i)(?<![a-zа-яё0-9])(?:checko(?:\.ru)?|чеко|чекало|чекко)"
+                r"(?![a-zа-яё0-9])",
+            )
 
     def test_management_command_bootstraps_first_generation(self):
         command_output = StringIO()
@@ -190,7 +269,7 @@ class SourceRecordExportApiTest(APITestCase):
             rows = list(workbook["data"].iter_rows(values_only=True))
             self.assertEqual(
                 rows[0][:6],
-                ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО", "organization"),
+                ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО", "uid"),
             )
             self.assertEqual(rows[1][4], "87654321")