fix: align source exports with mostovik

tests/apps/external_data/test_source_record_export.py

@@ -6,6 +6,10 @@ from io import BytesIO, StringIO
 from tempfile import TemporaryDirectory
 
 from apps.external_data.source_record_export import (
+    ORGANIZATION_EXPORT_FIELDS,
+    SOURCE_GROUP_EXPORT_SPECS,
+    SOURCE_RECORD_EXPORT_FIELDS,
+    _source_group_headers,
     build_source_record_export_artifacts,
     load_current_source_record_export_generation,
 )
@@ -16,6 +20,7 @@ from rest_framework import status
 from rest_framework.test import APITestCase
 
 from tests.apps.external_data.factories import (
+    ArbitrationCaseFactory,
     BankruptcyProcedureFactory,
     FinancialReportFactory,
     FinancialReportLineFactory,
@@ -74,6 +79,7 @@ class SourceRecordExportApiTest(APITestCase):
         IndustrialCertificateFactory.create(organization=organization)
         ManufacturerRegistryEntryFactory.create(organization=organization)
         ProsecutorCheckFactory.create(organization=organization)
+        arbitration_case = ArbitrationCaseFactory.create(organization=organization)
         report = FinancialReportFactory.create(organization=organization)
         FinancialReportLineFactory.create(report=report, line_code="1600")
 
@@ -81,7 +87,13 @@ class SourceRecordExportApiTest(APITestCase):
 
         self.assertEqual(generation.artifacts_count, 25)
         self.assertEqual(generation.files_count, 25)
-        self.assertEqual(generation.records_count, 5)
+        self.assertEqual(generation.records_count, 6)
+        expected_prefix = [*ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS]
+        for source_spec in SOURCE_GROUP_EXPORT_SPECS.values():
+            self.assertEqual(
+                _source_group_headers(source_spec)[: len(expected_prefix)],
+                expected_prefix,
+            )
         industrial_path = next(
             artifact.path
             for artifact in generation.artifacts
@@ -98,6 +110,12 @@ class SourceRecordExportApiTest(APITestCase):
             },
         )
         self.assertEqual({row["ОКПО"] for row in industrial_rows}, {"12345678"})
+        self.assertEqual(
+            list(industrial_rows[0])[: len(expected_prefix)],
+            expected_prefix,
+        )
+        self.assertNotIn("organization", industrial_rows[0])
+        self.assertIn("payload.product_name", industrial_rows[0])
 
         financial_path = next(
             artifact.path
@@ -107,7 +125,46 @@ class SourceRecordExportApiTest(APITestCase):
         financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
         self.assertEqual(financial_rows[0]["financial_lines"][0]["line_code"], "1600")
 
-    def test_generation_excludes_provider_records_and_keeps_okpo(self):
+        arbitration_path = next(
+            artifact.path
+            for artifact in generation.artifacts
+            if artifact.source_group == "arbitration" and artifact.file_format == "json"
+        )
+        arbitration_row = json.loads(arbitration_path.read_text(encoding="utf-8"))[0]
+        self.assertEqual(arbitration_row["source"], "arbitration")
+        self.assertEqual(
+            arbitration_row["external_id"], arbitration_row["payload.case_number"]
+        )
+        self.assertEqual(
+            arbitration_row["title"],
+            f"Дело {arbitration_row['payload.case_number']}",
+        )
+        self.assertEqual(
+            arbitration_row["record_date"], arbitration_case.decision_date.isoformat()
+        )
+        self.assertEqual(arbitration_row["status"], arbitration_row["payload.status"])
+        self.assertEqual(arbitration_row["payload.role"], "defendant")
+        self.assertIn("payload.court", arbitration_row)
+        self.assertEqual(arbitration_row["payload.source"], "arbitration")
+        self.assertEqual(arbitration_row["payload.inn"], organization.inn)
+        self.assertEqual(
+            [key for key in arbitration_row if key.startswith("payload.")],
+            [
+                "payload.inn",
+                "payload.url",
+                "payload.ogrn",
+                "payload.role",
+                "payload.court",
+                "payload.source",
+                "payload.status",
+                "payload.load_batch",
+                "payload.case_number",
+                "payload.claim_amount",
+                "payload.organisation_name",
+            ],
+        )
+
+    def test_generation_keeps_provider_records_but_hides_provider_mentions(self):
         organization = OrganizationFactory.create(okpo="11223344")
         excluded_record = BankruptcyProcedureFactory.create(
             organization=organization,
@@ -122,7 +179,7 @@ class SourceRecordExportApiTest(APITestCase):
 
         generation = build_source_record_export_artifacts()
 
-        self.assertEqual(generation.records_count, 1)
+        self.assertEqual(generation.records_count, 2)
         artifacts = [
             artifact
             for artifact in generation.artifacts
@@ -134,14 +191,36 @@ class SourceRecordExportApiTest(APITestCase):
         for artifact in artifacts:
             if artifact.file_format == "json":
                 exported_values = json.loads(artifact.path.read_text(encoding="utf-8"))
+                self.assertEqual(
+                    [key for key in exported_values[0] if key.startswith("payload.")],
+                    [
+                        "payload.inn",
+                        "payload.url",
+                        "payload.ogrn",
+                        "payload.source",
+                        "payload.messages",
+                        "payload.load_batch",
+                        "payload.case_number",
+                        "payload.message_date",
+                        "payload.message_type",
+                        "payload.messages_count",
+                        "payload.organisation_name",
+                    ],
+                )
             elif artifact.file_format == "csv":
                 exported_values = artifact.path.read_text(encoding="utf-8-sig")
             else:
                 workbook = load_workbook(artifact.path, read_only=True)
                 exported_values = list(workbook["data"].iter_rows(values_only=True))
-            self.assertNotIn(str(excluded_record.id), str(exported_values))
+            serialized_values = str(exported_values)
+            self.assertIn(str(excluded_record.id), serialized_values)
             self.assertIn(str(included_record.id), str(exported_values))
             self.assertIn("11223344", str(exported_values))
+            self.assertNotRegex(
+                serialized_values,
+                r"(?i)(?<![a-zа-яё0-9])(?:checko(?:\.ru)?|чеко|чекало|чекко)"
+                r"(?![a-zа-яё0-9])",
+            )
 
     def test_management_command_bootstraps_first_generation(self):
         command_output = StringIO()
@@ -190,7 +269,7 @@ class SourceRecordExportApiTest(APITestCase):
             rows = list(workbook["data"].iter_rows(values_only=True))
             self.assertEqual(
                 rows[0][:6],
-                ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО", "organization"),
+                ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО", "uid"),
             )
             self.assertEqual(rows[1][4], "87654321")