From de5c1aa4cc0a16e0d7d3e0d01dc78c7f060ed04d Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 4 Aug 2026 09:07:08 +0200 Subject: [PATCH] fix: exclude provider records from source exports --- docs/source-record-export-matrix-ru.md | 5 + src/organizations/source_record_export.py | 36 +++++- .../test_source_record_export.py | 109 ++++++++++++++++-- 3 files changed, 142 insertions(+), 8 deletions(-) diff --git a/docs/source-record-export-matrix-ru.md b/docs/source-record-export-matrix-ru.md index 33492d7..4e7f104 100644 --- a/docs/source-record-export-matrix-ru.md +++ b/docs/source-record-export-matrix-ru.md @@ -41,6 +41,11 @@ XLSX или JSON заново. Он упаковывает файлы после больше из-за разбиения крупных XLSX. Если финансовые показатели выбраны вместе с другим форматом, в ZIP для них всё равно включается JSON. +Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`, +`КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого +`payload`. Записи технического внешнего поставщика в публичные файлы не +включаются; в БД они сохраняются для работы интеграции и дедупликации. + ## Ночная генерация Celery Beat запускает diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index 05ed310..dbf209f 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -43,6 +43,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket" SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}") +EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile( + r"(? bool: + """Return whether a provider-backed record must stay out of public files.""" + values = ( + record.source, + record.external_id, + record.title, + record.url, + record.payload, + ) + return any(_contains_excluded_export_provider(value) for value in values) + + +def _contains_excluded_export_provider(value: object) -> bool: + if isinstance(value, str): + return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None + if isinstance(value, dict): + return any( + _contains_excluded_export_provider(key) + or _contains_excluded_export_provider(item) + for key, item in value.items() + ) + if isinstance(value, list | tuple): + return any(_contains_excluded_export_provider(item) for item in value) + return False + + def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]: if not isinstance(value, dict): return {prefix: value} if value not in (None, "") else {} diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index 1f48174..096142b 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -99,6 +99,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): inn="7707083810", ogrn="1027700132010", kpp="770701001", + okpo="12345678", ) inspection_extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -176,16 +177,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): worksheet = workbook["data"] rows = list(worksheet.iter_rows(values_only=True)) self.assertEqual( - rows[0][:4], - ("Наименование", "ИНН", "ОГРН", "КПП"), + rows[0][:5], + ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"), ) self.assertEqual( - rows[1][:4], + rows[1][:5], ( 'Общество с ограниченной ответственностью "Экспорт"', "7707083810", "1027700132010", "770701001", + "12345678", ), ) self.assertIn("payload.risk.score", rows[0]) @@ -255,6 +257,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): inn="7707083811", ogrn="1027700132011", kpp="770701002", + okpo="87654321", ) extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -287,9 +290,19 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): csv_text = csv_bytes.decode("utf-8-sig") csv_rows = list(csv.reader(StringIO(csv_text))) - self.assertEqual(csv_rows[0][:4], ["Наименование", "ИНН", "ОГРН", "КПП"]) self.assertEqual( - csv_rows[1][:4], ['ООО "CSV"', "7707083811", "1027700132011", "770701002"] + csv_rows[0][:5], + ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"], + ) + self.assertEqual( + csv_rows[1][:5], + [ + 'ООО "CSV"', + "7707083811", + "1027700132011", + "770701002", + "87654321", + ], ) self.assertIn("payload.nested.value", csv_rows[0]) @@ -348,8 +361,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): 2, ) self.assertEqual( - next(second_workbook["data"].iter_rows(values_only=True))[:4], - ("Наименование", "ИНН", "ОГРН", "КПП"), + next(second_workbook["data"].iter_rows(values_only=True))[:5], + ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"), ) selected_artifacts = [ @@ -376,6 +389,88 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): self.assertEqual(package.files_count, 2) self.assertFalse((Path(self.export_directory.name) / "tmp").exists()) + def test_prepared_files_include_okpo_and_exclude_provider_records(self): + organization = Organization.objects.create( + name='ООО "Публичная выгрузка"', + inn="7707083888", + ogrn="1027700132088", + kpp="770701008", + okpo="11223344", + ) + extension = PlannedInspectionExtension.objects.create( + organization=organization, + title="Плановые проверки Генпрокуратуры России", + ) + source_record = OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="checko", + external_id="checko-inspection:123", + title="Запись Checko", + url="https://api.checko.ru/v2/inspections/123", + payload={ + "provider": "Checko", + "provider_alias": "Чеко", + "provider_url": "https://checko.ru/company/123", + "checkout_marker": "checkout_sha", + }, + ) + included_record = OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="official-registry", + external_id="inspection:456", + title="Официальная запись", + payload={"checkout_marker": "checkout_sha"}, + ) + + with TemporaryDirectory() as temporary_directory: + export_directory = Path(temporary_directory) + spool_path = export_directory / "rows.json" + headers, records_count = _spool_source_group_rows( + source_group=SourceGroup.PLANNED_INSPECTIONS.value, + output_path=spool_path, + ) + + self.assertEqual( + headers[:5], + ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"], + ) + self.assertEqual(records_count, 1) + spooled_rows = json.loads(spool_path.read_text(encoding="utf-8")) + self.assertEqual(spooled_rows[0]["ОКПО"], "11223344") + self.assertEqual(spooled_rows[0]["uid"], str(included_record.uid)) + self.assertEqual(spooled_rows[0]["payload.checkout_marker"], "checkout_sha") + + for file_format in ("json", "csv", "xlsx"): + artifact_path = export_directory / f"artifact.{file_format}" + _render_source_group_artifact( + row_spool_path=spool_path, + output_path=artifact_path, + headers=headers, + file_format=file_format, + records_count=records_count, + ) + if file_format == "json": + exported_values = json.loads( + artifact_path.read_text(encoding="utf-8") + ) + elif file_format == "csv": + with artifact_path.open( + encoding="utf-8-sig", + newline="", + ) as csv_file: + exported_values = list(csv.reader(csv_file)) + else: + workbook = load_workbook(artifact_path, read_only=True) + exported_values = list(workbook["data"].iter_rows(values_only=True)) + self.assertNotIn(str(source_record.uid), str(exported_values)) + self.assertIn(str(included_record.uid), str(exported_values)) + + source_record.refresh_from_db() + self.assertEqual(source_record.source, "checko") + self.assertEqual(source_record.payload["provider"], "Checko") + def test_nightly_export_clears_model_ordering_to_avoid_multi_million_row_sort(self): queryset = _source_group_queryset(SourceGroup.GOVERNMENT_PROCUREMENTS.value)