fix: exclude provider records from source exports

docs/source-record-export-matrix-ru.md

@@ -41,6 +41,11 @@ XLSX или JSON заново. Он упаковывает файлы после
 больше из-за разбиения крупных XLSX. Если финансовые показатели выбраны вместе
 с другим форматом, в ZIP для них всё равно включается JSON.
 
+Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`,
+`КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого
+`payload`. Записи технического внешнего поставщика в публичные файлы не
+включаются; в БД они сохраняются для работы интеграции и дедупликации.
+
 ## Ночная генерация
 
 Celery Beat запускает

src/organizations/source_record_export.py

@@ -43,6 +43,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000
 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60
 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket"
 SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}")
+EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile(
+    r"(?<![A-Za-zА-Яа-яЁё0-9])(?:checko(?:\.ru)?|чекало|чекко|чеко)"
+    r"(?![A-Za-zА-Яа-яЁё0-9])",
+    flags=re.IGNORECASE,
+)
 
 SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = {
     SourceGroup.FINANCIAL_INDICATORS.value: "financial-indicators",
@@ -56,7 +61,7 @@ SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = {
     SourceGroup.VACANCIES.value: "labor-vacancies",
 }
 
-ORGANIZATION_EXPORT_FIELDS = ["Наименование", "ИНН", "ОГРН", "КПП"]
+ORGANIZATION_EXPORT_FIELDS = ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"]
 SOURCE_RECORD_EXPORT_FIELDS = [
     "uid",
     "source_group",
@@ -530,6 +535,8 @@ def _spool_source_group_rows(
             source_group=source_group,
             include_financial_lines=include_financial_lines,
         ):
+            if _is_excluded_export_provider_record(record):
+                continue
             row = _build_record_row(
                 record,
                 include_financial_lines=include_financial_lines,
@@ -696,6 +703,7 @@ def _build_record_row(
         "ИНН": organization.inn,
         "ОГРН": organization.ogrn,
         "КПП": organization.kpp,
+        "ОКПО": organization.okpo,
         "uid": str(record.uid),
         "source_group": record.extension.source_group,
         "source": record.source,
@@ -724,6 +732,32 @@ def _build_record_row(
     return serialized_row
 
 
+def _is_excluded_export_provider_record(record: OrganizationSourceRecord) -> bool:
+    """Return whether a provider-backed record must stay out of public files."""
+    values = (
+        record.source,
+        record.external_id,
+        record.title,
+        record.url,
+        record.payload,
+    )
+    return any(_contains_excluded_export_provider(value) for value in values)
+
+
+def _contains_excluded_export_provider(value: object) -> bool:
+    if isinstance(value, str):
+        return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None
+    if isinstance(value, dict):
+        return any(
+            _contains_excluded_export_provider(key)
+            or _contains_excluded_export_provider(item)
+            for key, item in value.items()
+        )
+    if isinstance(value, list | tuple):
+        return any(_contains_excluded_export_provider(item) for item in value)
+    return False
+
+
 def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]:
     if not isinstance(value, dict):
         return {prefix: value} if value not in (None, "") else {}

tests/apps/organizations/test_source_record_export.py

@@ -99,6 +99,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
             inn="7707083810",
             ogrn="1027700132010",
             kpp="770701001",
+            okpo="12345678",
         )
         inspection_extension = PlannedInspectionExtension.objects.create(
             organization=organization,
@@ -176,16 +177,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
             worksheet = workbook["data"]
             rows = list(worksheet.iter_rows(values_only=True))
             self.assertEqual(
-                rows[0][:4],
+                rows[0][:5],
-                ("Наименование", "ИНН", "ОГРН", "КПП"),
+                ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"),
             )
             self.assertEqual(
-                rows[1][:4],
+                rows[1][:5],
                 (
                     'Общество с ограниченной ответственностью "Экспорт"',
                     "7707083810",
                     "1027700132010",
                     "770701001",
+                    "12345678",
                 ),
             )
             self.assertIn("payload.risk.score", rows[0])
@@ -255,6 +257,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
             inn="7707083811",
             ogrn="1027700132011",
             kpp="770701002",
+            okpo="87654321",
         )
         extension = PlannedInspectionExtension.objects.create(
             organization=organization,
@@ -287,9 +290,19 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
             csv_text = csv_bytes.decode("utf-8-sig")
             csv_rows = list(csv.reader(StringIO(csv_text)))
 
-        self.assertEqual(csv_rows[0][:4], ["Наименование", "ИНН", "ОГРН", "КПП"])
         self.assertEqual(
-            csv_rows[1][:4], ['ООО "CSV"', "7707083811", "1027700132011", "770701002"]
+            csv_rows[0][:5],
+            ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"],
+        )
+        self.assertEqual(
+            csv_rows[1][:5],
+            [
+                'ООО "CSV"',
+                "7707083811",
+                "1027700132011",
+                "770701002",
+                "87654321",
+            ],
         )
         self.assertIn("payload.nested.value", csv_rows[0])
 
@@ -348,8 +361,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
             2,
         )
         self.assertEqual(
-            next(second_workbook["data"].iter_rows(values_only=True))[:4],
+            next(second_workbook["data"].iter_rows(values_only=True))[:5],
-            ("Наименование", "ИНН", "ОГРН", "КПП"),
+            ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"),
         )
 
         selected_artifacts = [
@@ -376,6 +389,88 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
         self.assertEqual(package.files_count, 2)
         self.assertFalse((Path(self.export_directory.name) / "tmp").exists())
 
+    def test_prepared_files_include_okpo_and_exclude_provider_records(self):
+        organization = Organization.objects.create(
+            name='ООО "Публичная выгрузка"',
+            inn="7707083888",
+            ogrn="1027700132088",
+            kpp="770701008",
+            okpo="11223344",
+        )
+        extension = PlannedInspectionExtension.objects.create(
+            organization=organization,
+            title="Плановые проверки Генпрокуратуры России",
+        )
+        source_record = OrganizationSourceRecord.objects.create(
+            extension=extension,
+            record_type="inspection",
+            source="checko",
+            external_id="checko-inspection:123",
+            title="Запись Checko",
+            url="https://api.checko.ru/v2/inspections/123",
+            payload={
+                "provider": "Checko",
+                "provider_alias": "Чеко",
+                "provider_url": "https://checko.ru/company/123",
+                "checkout_marker": "checkout_sha",
+            },
+        )
+        included_record = OrganizationSourceRecord.objects.create(
+            extension=extension,
+            record_type="inspection",
+            source="official-registry",
+            external_id="inspection:456",
+            title="Официальная запись",
+            payload={"checkout_marker": "checkout_sha"},
+        )
+
+        with TemporaryDirectory() as temporary_directory:
+            export_directory = Path(temporary_directory)
+            spool_path = export_directory / "rows.json"
+            headers, records_count = _spool_source_group_rows(
+                source_group=SourceGroup.PLANNED_INSPECTIONS.value,
+                output_path=spool_path,
+            )
+
+            self.assertEqual(
+                headers[:5],
+                ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"],
+            )
+            self.assertEqual(records_count, 1)
+            spooled_rows = json.loads(spool_path.read_text(encoding="utf-8"))
+            self.assertEqual(spooled_rows[0]["ОКПО"], "11223344")
+            self.assertEqual(spooled_rows[0]["uid"], str(included_record.uid))
+            self.assertEqual(spooled_rows[0]["payload.checkout_marker"], "checkout_sha")
+
+            for file_format in ("json", "csv", "xlsx"):
+                artifact_path = export_directory / f"artifact.{file_format}"
+                _render_source_group_artifact(
+                    row_spool_path=spool_path,
+                    output_path=artifact_path,
+                    headers=headers,
+                    file_format=file_format,
+                    records_count=records_count,
+                )
+                if file_format == "json":
+                    exported_values = json.loads(
+                        artifact_path.read_text(encoding="utf-8")
+                    )
+                elif file_format == "csv":
+                    with artifact_path.open(
+                        encoding="utf-8-sig",
+                        newline="",
+                    ) as csv_file:
+                        exported_values = list(csv.reader(csv_file))
+                else:
+                    workbook = load_workbook(artifact_path, read_only=True)
+                    exported_values = list(workbook["data"].iter_rows(values_only=True))
+                self.assertNotIn(str(source_record.uid), str(exported_values))
+                self.assertIn(str(included_record.uid), str(exported_values))
+
+        source_record.refresh_from_db()
+        self.assertEqual(source_record.source, "checko")
+        self.assertEqual(source_record.payload["provider"], "Checko")
+
     def test_nightly_export_clears_model_ordering_to_avoid_multi_million_row_sort(self):
         queryset = _source_group_queryset(SourceGroup.GOVERNMENT_PROCUREMENTS.value)