diff --git a/docs/source-record-export-matrix-ru.md b/docs/source-record-export-matrix-ru.md index 4e7f104..e95f84c 100644 --- a/docs/source-record-export-matrix-ru.md +++ b/docs/source-record-export-matrix-ru.md @@ -43,8 +43,9 @@ XLSX или JSON заново. Он упаковывает файлы после Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`, `КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого -`payload`. Записи технического внешнего поставщика в публичные файлы не -включаются; в БД они сохраняются для работы интеграции и дедупликации. +`payload`. Все записи включаются в публичные файлы, а техническое наименование +внешнего поставщика нейтрализуется. Исходные значения в БД сохраняются для +работы интеграции и дедупликации. ## Ночная генерация diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index dbf209f..2807510 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -43,7 +43,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket" SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}") -EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile( +EXPORT_PROVIDER_URL_PATTERN = re.compile( + r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*", + flags=re.IGNORECASE, +) +EXPORT_PROVIDER_NAME_PATTERN = re.compile( r"(? bool: - """Return whether a provider-backed record must stay out of public files.""" - values = ( - record.source, - record.external_id, - record.title, - record.url, - record.payload, - ) - return any(_contains_excluded_export_provider(value) for value in values) - - -def _contains_excluded_export_provider(value: object) -> bool: +def _sanitize_export_provider_value(value: Any) -> Any: + """Remove provider wording from files without changing stored source data.""" if isinstance(value, str): - return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None + sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value) + return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized) if isinstance(value, dict): - return any( - _contains_excluded_export_provider(key) - or _contains_excluded_export_provider(item) + return { + _sanitize_export_provider_value(key): _sanitize_export_provider_value(item) for key, item in value.items() - ) + } if isinstance(value, list | tuple): - return any(_contains_excluded_export_provider(item) for item in value) - return False + return [_sanitize_export_provider_value(item) for item in value] + return value def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]: diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index 096142b..abecdb4 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -389,7 +389,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): self.assertEqual(package.files_count, 2) self.assertFalse((Path(self.export_directory.name) / "tmp").exists()) - def test_prepared_files_include_okpo_and_exclude_provider_records(self): + def test_prepared_files_include_all_records_and_hide_provider_mentions(self): organization = Organization.objects.create( name='ООО "Публичная выгрузка"', inn="7707083888", @@ -436,11 +436,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): headers[:5], ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"], ) - self.assertEqual(records_count, 1) + self.assertEqual(records_count, 2) spooled_rows = json.loads(spool_path.read_text(encoding="utf-8")) - self.assertEqual(spooled_rows[0]["ОКПО"], "11223344") - self.assertEqual(spooled_rows[0]["uid"], str(included_record.uid)) - self.assertEqual(spooled_rows[0]["payload.checkout_marker"], "checkout_sha") + rows_by_uid = {row["uid"]: row for row in spooled_rows} + self.assertEqual( + set(rows_by_uid), {str(source_record.uid), str(included_record.uid)} + ) + provider_row = rows_by_uid[str(source_record.uid)] + self.assertEqual(provider_row["ОКПО"], "11223344") + self.assertEqual(provider_row["source"], "external-source") + self.assertEqual(provider_row["payload.provider"], "external-source") + self.assertEqual(provider_row["payload.checkout_marker"], "checkout_sha") for file_format in ("json", "csv", "xlsx"): artifact_path = export_directory / f"artifact.{file_format}" @@ -464,8 +470,14 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): else: workbook = load_workbook(artifact_path, read_only=True) exported_values = list(workbook["data"].iter_rows(values_only=True)) - self.assertNotIn(str(source_record.uid), str(exported_values)) + serialized_values = str(exported_values) + self.assertIn(str(source_record.uid), serialized_values) self.assertIn(str(included_record.uid), str(exported_values)) + self.assertNotRegex( + serialized_values, + r"(?i)(?