fix: preserve source export record counts

docs/source-record-export-matrix-ru.md

@@ -43,8 +43,9 @@ XLSX или JSON заново. Он упаковывает файлы после
 
 Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`,
 `КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого
-`payload`. Записи технического внешнего поставщика в публичные файлы не
+`payload`. Все записи включаются в публичные файлы, а техническое наименование
-включаются; в БД они сохраняются для работы интеграции и дедупликации.
+внешнего поставщика нейтрализуется. Исходные значения в БД сохраняются для
+работы интеграции и дедупликации.
 
 ## Ночная генерация
 

src/organizations/source_record_export.py

@@ -43,7 +43,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000
 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60
 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket"
 SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}")
-EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile(
+EXPORT_PROVIDER_URL_PATTERN = re.compile(
+    r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*",
+    flags=re.IGNORECASE,
+)
+EXPORT_PROVIDER_NAME_PATTERN = re.compile(
     r"(?<![A-Za-zА-Яа-яЁё0-9])(?:checko(?:\.ru)?|чекало|чекко|чеко)"
     r"(?![A-Za-zА-Яа-яЁё0-9])",
     flags=re.IGNORECASE,
@@ -535,8 +539,6 @@ def _spool_source_group_rows(
             source_group=source_group,
             include_financial_lines=include_financial_lines,
         ):
-            if _is_excluded_export_provider_record(record):
-                continue
             row = _build_record_row(
                 record,
                 include_financial_lines=include_financial_lines,
@@ -719,7 +721,10 @@ def _build_record_row(
         "updated_at": record.updated_at,
         **_flatten_payload(record.payload),
     }
-    serialized_row = {key: _serialize_json_value(value) for key, value in row.items()}
+    public_row = _sanitize_export_provider_value(row)
+    serialized_row = {
+        key: _serialize_json_value(value) for key, value in public_row.items()
+    }
 
     if include_financial_lines:
         serialized_row["financial_lines"] = [
@@ -732,30 +737,19 @@ def _build_record_row(
     return serialized_row
 
 
-def _is_excluded_export_provider_record(record: OrganizationSourceRecord) -> bool:
+def _sanitize_export_provider_value(value: Any) -> Any:
-    """Return whether a provider-backed record must stay out of public files."""
+    """Remove provider wording from files without changing stored source data."""
-    values = (
-        record.source,
-        record.external_id,
-        record.title,
-        record.url,
-        record.payload,
-    )
-    return any(_contains_excluded_export_provider(value) for value in values)
-
-
-def _contains_excluded_export_provider(value: object) -> bool:
     if isinstance(value, str):
-        return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None
+        sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value)
+        return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized)
     if isinstance(value, dict):
-        return any(
+        return {
-            _contains_excluded_export_provider(key)
+            _sanitize_export_provider_value(key): _sanitize_export_provider_value(item)
-            or _contains_excluded_export_provider(item)
             for key, item in value.items()
-        )
+        }
     if isinstance(value, list | tuple):
-        return any(_contains_excluded_export_provider(item) for item in value)
+        return [_sanitize_export_provider_value(item) for item in value]
-    return False
+    return value
 
 
 def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]:

tests/apps/organizations/test_source_record_export.py

@@ -389,7 +389,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
         self.assertEqual(package.files_count, 2)
         self.assertFalse((Path(self.export_directory.name) / "tmp").exists())
 
-    def test_prepared_files_include_okpo_and_exclude_provider_records(self):
+    def test_prepared_files_include_all_records_and_hide_provider_mentions(self):
         organization = Organization.objects.create(
             name='ООО "Публичная выгрузка"',
             inn="7707083888",
@@ -436,11 +436,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
                 headers[:5],
                 ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"],
             )
-            self.assertEqual(records_count, 1)
+            self.assertEqual(records_count, 2)
             spooled_rows = json.loads(spool_path.read_text(encoding="utf-8"))
-            self.assertEqual(spooled_rows[0]["ОКПО"], "11223344")
+            rows_by_uid = {row["uid"]: row for row in spooled_rows}
-            self.assertEqual(spooled_rows[0]["uid"], str(included_record.uid))
+            self.assertEqual(
-            self.assertEqual(spooled_rows[0]["payload.checkout_marker"], "checkout_sha")
+                set(rows_by_uid), {str(source_record.uid), str(included_record.uid)}
+            )
+            provider_row = rows_by_uid[str(source_record.uid)]
+            self.assertEqual(provider_row["ОКПО"], "11223344")
+            self.assertEqual(provider_row["source"], "external-source")
+            self.assertEqual(provider_row["payload.provider"], "external-source")
+            self.assertEqual(provider_row["payload.checkout_marker"], "checkout_sha")
 
             for file_format in ("json", "csv", "xlsx"):
                 artifact_path = export_directory / f"artifact.{file_format}"
@@ -464,8 +470,14 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
                 else:
                     workbook = load_workbook(artifact_path, read_only=True)
                     exported_values = list(workbook["data"].iter_rows(values_only=True))
-                self.assertNotIn(str(source_record.uid), str(exported_values))
+                serialized_values = str(exported_values)
+                self.assertIn(str(source_record.uid), serialized_values)
                 self.assertIn(str(included_record.uid), str(exported_values))
+                self.assertNotRegex(
+                    serialized_values,
+                    r"(?i)(?<![a-zа-яё0-9])(?:checko(?:\.ru)?|чеко|чекало|чекко)"
+                    r"(?![a-zа-яё0-9])",
+                )
 
         source_record.refresh_from_db()
         self.assertEqual(source_record.source, "checko")