fix: align source exports with mostovik

src/apps/external_data/source_record_export.py

@@ -54,7 +54,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000
 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60
 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "external-data:source-record-exports:ticket"
 SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}")
-EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile(
+EXPORT_PROVIDER_URL_PATTERN = re.compile(
+    r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*",
+    flags=re.IGNORECASE,
+)
+EXPORT_PROVIDER_NAME_PATTERN = re.compile(
     r"(?<![A-Za-zА-Яа-яЁё0-9])(?:checko(?:\.ru)?|чекало|чекко|чеко)"
     r"(?![A-Za-zА-Яа-яЁё0-9])",
     flags=re.IGNORECASE,
@@ -66,12 +70,19 @@ ORGANIZATION_EXPORT_FIELDS = [
     "ОГРН",
     "КПП",
     "ОКПО",
-    "organization",
 ]
 SOURCE_RECORD_EXPORT_FIELDS = [
     "uid",
     "source_group",
+    "source",
     "record_type",
+    "external_id",
+    "title",
+    "record_date",
+    "amount",
+    "status",
+    "url",
+    "load_batch",
     "created_at",
     "updated_at",
 ]
@@ -93,6 +104,18 @@ class SourceModelExportSpec:
     model: Any
     record_type: str
     fields: tuple[str, ...]
+    source: str
+    source_field: str | None = None
+    external_id_field: str | None = None
+    title_field: str | None = None
+    record_date_field: str | None = None
+    amount_field: str | None = None
+    status_field: str | None = None
+    url_field: str | None = None
+    load_batch_field: str | None = None
+    title_prefix: str = ""
+    payload_aliases: tuple[tuple[str, str], ...] = ()
+    payload_export_fields: tuple[str, ...] | None = None
     prefetch_related: tuple[str, ...] = ()
 
 
@@ -123,6 +146,12 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "source",
                     "error_message",
                 ),
+                source=FINANCIAL_SOURCE_GROUP,
+                source_field="source",
+                external_id_field="external_id",
+                title_field="file_name",
+                status_field="status",
+                load_batch_field="load_batch",
                 prefetch_related=("lines",),
             ),
         ),
@@ -144,6 +173,12 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "execution_end_date",
                     "purchase_name",
                 ),
+                source="government_procurements",
+                external_id_field="purchase_number",
+                title_field="purchase_name",
+                record_date_field="contract_date",
+                amount_field="contract_amount",
+                status_field="status",
             ),
         ),
     ),
@@ -161,6 +196,9 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "tnved_code",
                     "registry_number",
                 ),
+                source="industrial_production",
+                external_id_field="registry_number",
+                title_field="product_name",
             ),
             SourceModelExportSpec(
                 model=IndustrialCertificate,
@@ -173,11 +211,20 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "organisation_name",
                     "ogrn",
                 ),
+                source="industrial_production",
+                external_id_field="certificate_number",
+                title_field="organisation_name",
+                record_date_field="issue_date",
+                url_field="certificate_file_url",
+                payload_aliases=(("certificate_file_url", "url"),),
             ),
             SourceModelExportSpec(
                 model=ManufacturerRegistryEntry,
                 record_type="manufacturer_registry_entry",
                 fields=("full_legal_name", "inn", "ogrn", "address"),
+                source="industrial_production",
+                external_id_field="ogrn",
+                title_field="full_legal_name",
             ),
         ),
     ),
@@ -196,6 +243,11 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "start_date",
                     "status",
                 ),
+                source="planned_inspections",
+                external_id_field="registration_number",
+                title_field="control_authority",
+                record_date_field="start_date",
+                status_field="status",
             ),
         ),
     ),
@@ -214,6 +266,26 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "status",
                     "source_url",
                 ),
+                source="bankruptcy",
+                external_id_field="external_id",
+                title_field="message_type",
+                record_date_field="message_date",
+                status_field="status",
+                url_field="source_url",
+                payload_aliases=(("source_url", "url"),),
+                payload_export_fields=(
+                    "inn",
+                    "url",
+                    "ogrn",
+                    "source",
+                    "messages",
+                    "load_batch",
+                    "case_number",
+                    "message_date",
+                    "message_type",
+                    "messages_count",
+                    "organisation_name",
+                ),
             ),
         ),
     ),
@@ -234,6 +306,14 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "status",
                     "source_url",
                 ),
+                source="defense_suppliers",
+                source_field="registry_source",
+                external_id_field="external_id",
+                title_field="supplier_name",
+                record_date_field="included_at",
+                status_field="status",
+                url_field="source_url",
+                payload_aliases=(("source_url", "url"),),
             ),
         ),
     ),
@@ -251,6 +331,26 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "status",
                     "decision_date",
                 ),
+                source="arbitration",
+                external_id_field="case_number",
+                title_field="case_number",
+                record_date_field="decision_date",
+                status_field="status",
+                title_prefix="Дело ",
+                payload_aliases=(("court_name", "court"), ("party_role", "role")),
+                payload_export_fields=(
+                    "inn",
+                    "url",
+                    "ogrn",
+                    "role",
+                    "court",
+                    "source",
+                    "status",
+                    "load_batch",
+                    "case_number",
+                    "claim_amount",
+                    "organisation_name",
+                ),
             ),
         ),
     ),
@@ -269,6 +369,11 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "issued_at",
                     "expires_at",
                 ),
+                source="security_registries",
+                external_id_field="external_id",
+                title_field="registry_name",
+                record_date_field="issued_at",
+                status_field="presence_status",
             ),
         ),
     ),
@@ -288,6 +393,15 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
                     "salary_amount",
                     "source_url",
                 ),
+                source="vacancies",
+                source_field="vacancy_source",
+                external_id_field="external_id",
+                title_field="title",
+                record_date_field="published_at",
+                amount_field="salary_amount",
+                status_field="status",
+                url_field="source_url",
+                payload_aliases=(("source_url", "url"),),
             ),
         ),
     ),
@@ -717,17 +831,19 @@ def _iter_source_model_records(
 
 
 def _source_group_headers(source_spec: SourceGroupExportSpec) -> list[str]:
-    model_fields: list[str] = []
+    payload_fields: list[str] = []
     for model_spec in source_spec.models:
-        for field_name in model_spec.fields:
-            if field_name not in model_fields:
-                model_fields.append(field_name)
+        for field_name in _payload_export_field_names(model_spec):
+            payload_field_name = f"payload.{field_name}"
+            if payload_field_name not in payload_fields:
+                payload_fields.append(payload_field_name)
+    payload_fields.sort()
     if source_spec.source_group == FINANCIAL_SOURCE_GROUP:
-        model_fields.append("financial_lines")
+        payload_fields.append("financial_lines")
     return [
         *ORGANIZATION_EXPORT_FIELDS,
         *SOURCE_RECORD_EXPORT_FIELDS,
-        *model_fields,
+        *payload_fields,
     ]
 
 
@@ -742,8 +858,6 @@ def _spool_source_group_rows(
         is_first_row = True
         for model_spec in source_spec.models:
             for record in _iter_source_model_records(model_spec):
-                if _is_excluded_export_provider_record(record, model_spec=model_spec):
-                    continue
                 row = _build_record_row(
                     record,
                     source_spec=source_spec,
@@ -899,13 +1013,28 @@ def _build_record_row(
         "ОГРН": organization.ogrn,
         "КПП": organization.kpp,
         "ОКПО": organization.okpo,
-        "organization": organization.id,
         "uid": record.id,
         "source_group": source_spec.source_group,
+        "source": _source_value(record, model_spec=model_spec),
         "record_type": model_spec.record_type,
+        "external_id": _record_field_value(record, model_spec.external_id_field),
+        "title": _title_value(record, model_spec=model_spec),
+        "record_date": _record_field_value(record, model_spec.record_date_field),
+        "amount": _record_field_value(record, model_spec.amount_field),
+        "status": _record_field_value(record, model_spec.status_field),
+        "url": _record_field_value(record, model_spec.url_field),
+        "load_batch": _record_field_value(record, model_spec.load_batch_field),
         "created_at": record.created_at,
         "updated_at": record.updated_at,
-        **{field_name: getattr(record, field_name) for field_name in model_spec.fields},
+        **{
+            f"payload.{field_name}": _payload_field_value(
+                record,
+                organization=organization,
+                model_spec=model_spec,
+                field_name=field_name,
+            )
+            for field_name in _payload_export_field_names(model_spec)
+        },
     }
     if source_spec.source_group == FINANCIAL_SOURCE_GROUP:
         row["financial_lines"] = [
@@ -915,33 +1044,78 @@ def _build_record_row(
             }
             for line in record.lines.all()
         ]
-    return {key: _serialize_json_value(value) for key, value in row.items()}
+    public_row = _sanitize_export_provider_value(row)
+    return {key: _serialize_json_value(value) for key, value in public_row.items()}
 
 
-def _is_excluded_export_provider_record(
+def _source_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any:
+    source_value = _record_field_value(record, model_spec.source_field)
+    return source_value or model_spec.source
+
+
+def _title_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any:
+    title_value = _record_field_value(record, model_spec.title_field)
+    if not title_value or not model_spec.title_prefix:
+        return title_value
+    return f"{model_spec.title_prefix}{title_value}"
+
+
+def _payload_field_name(model_spec: SourceModelExportSpec, field_name: str) -> str:
+    return dict(model_spec.payload_aliases).get(field_name, field_name)
+
+
+def _payload_export_field_names(model_spec: SourceModelExportSpec) -> tuple[str, ...]:
+    if model_spec.payload_export_fields is not None:
+        return model_spec.payload_export_fields
+
+    field_names = ["inn", "ogrn", "organisation_name", "source"]
+    for field_name in model_spec.fields:
+        payload_field_name = _payload_field_name(model_spec, field_name)
+        if payload_field_name not in field_names:
+            field_names.append(payload_field_name)
+    return tuple(field_names)
+
+
+def _payload_field_value(
     record: Any,
     *,
+    organization: Any,
     model_spec: SourceModelExportSpec,
-) -> bool:
-    """Return whether a provider-backed record must stay out of public files."""
-    return any(
-        _contains_excluded_export_provider(getattr(record, field_name))
-        for field_name in model_spec.fields
-    )
+    field_name: str,
+) -> Any:
+    if field_name == "inn":
+        return organization.inn
+    if field_name == "ogrn":
+        return organization.ogrn
+    if field_name == "organisation_name":
+        return organization.full_name or organization.short_name or organization.name
+    if field_name == "source":
+        return _source_value(record, model_spec=model_spec)
+
+    source_field_by_alias = {
+        alias: source_field for source_field, alias in model_spec.payload_aliases
+    }
+    source_field = source_field_by_alias.get(field_name, field_name)
+    return getattr(record, source_field, None)
 
 
-def _contains_excluded_export_provider(value: object) -> bool:
+def _record_field_value(record: Any, field_name: str | None) -> Any:
+    return getattr(record, field_name) if field_name else None
+
+
+def _sanitize_export_provider_value(value: Any) -> Any:
+    """Remove provider wording from files without changing stored source data."""
     if isinstance(value, str):
-        return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None
+        sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value)
+        return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized)
     if isinstance(value, dict):
-        return any(
-            _contains_excluded_export_provider(key)
-            or _contains_excluded_export_provider(item)
+        return {
+            _sanitize_export_provider_value(key): _sanitize_export_provider_value(item)
             for key, item in value.items()
-        )
+        }
     if isinstance(value, list | tuple):
-        return any(_contains_excluded_export_provider(item) for item in value)
-    return False
+        return [_sanitize_export_provider_value(item) for item in value]
+    return value
 
 
 def _serialize_flat_value(value: Any) -> str | int | float | bool: