diff --git a/docs/source-record-export-matrix-ru.md b/docs/source-record-export-matrix-ru.md index 418d828..c36c2a8 100644 --- a/docs/source-record-export-matrix-ru.md +++ b/docs/source-record-export-matrix-ru.md @@ -36,10 +36,12 @@ JavaScript. Ticket не попадает в URL и после первого з Итого формируется 25 логических артефактов. Финансовые показатели всегда выгружаются в JSON с вложенным массивом `financial_lines`. Промышленная группа объединяет три таблицы, а поле `record_type` различает тип строки. Все строки -содержат реквизиты организации, включая ОКПО. +используют тот же контракт, что и Mostovik: реквизиты организации, включая ОКПО, +общие поля записи источника и специфичные поля в `payload.*`. -Записи технического внешнего поставщика в публичные файлы не включаются; в БД -они сохраняются для работы интеграции и дедупликации. +Все записи включаются в публичные файлы, а техническое наименование внешнего +поставщика нейтрализуется. Исходные значения в БД сохраняются для работы +интеграции и дедупликации. Физических XLSX-файлов может быть больше: по умолчанию один файл содержит не более 100 000 строк данных и получает суффикс `-part-001`, `-part-002` и далее. diff --git a/src/apps/external_data/source_record_export.py b/src/apps/external_data/source_record_export.py index 989a1e5..1f6c062 100644 --- a/src/apps/external_data/source_record_export.py +++ b/src/apps/external_data/source_record_export.py @@ -54,7 +54,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "external-data:source-record-exports:ticket" SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}") -EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile( +EXPORT_PROVIDER_URL_PATTERN = re.compile( + r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*", + flags=re.IGNORECASE, +) +EXPORT_PROVIDER_NAME_PATTERN = re.compile( r"(? list[str]: - model_fields: list[str] = [] + payload_fields: list[str] = [] for model_spec in source_spec.models: - for field_name in model_spec.fields: - if field_name not in model_fields: - model_fields.append(field_name) + for field_name in _payload_export_field_names(model_spec): + payload_field_name = f"payload.{field_name}" + if payload_field_name not in payload_fields: + payload_fields.append(payload_field_name) + payload_fields.sort() if source_spec.source_group == FINANCIAL_SOURCE_GROUP: - model_fields.append("financial_lines") + payload_fields.append("financial_lines") return [ *ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS, - *model_fields, + *payload_fields, ] @@ -742,8 +858,6 @@ def _spool_source_group_rows( is_first_row = True for model_spec in source_spec.models: for record in _iter_source_model_records(model_spec): - if _is_excluded_export_provider_record(record, model_spec=model_spec): - continue row = _build_record_row( record, source_spec=source_spec, @@ -899,13 +1013,28 @@ def _build_record_row( "ОГРН": organization.ogrn, "КПП": organization.kpp, "ОКПО": organization.okpo, - "organization": organization.id, "uid": record.id, "source_group": source_spec.source_group, + "source": _source_value(record, model_spec=model_spec), "record_type": model_spec.record_type, + "external_id": _record_field_value(record, model_spec.external_id_field), + "title": _title_value(record, model_spec=model_spec), + "record_date": _record_field_value(record, model_spec.record_date_field), + "amount": _record_field_value(record, model_spec.amount_field), + "status": _record_field_value(record, model_spec.status_field), + "url": _record_field_value(record, model_spec.url_field), + "load_batch": _record_field_value(record, model_spec.load_batch_field), "created_at": record.created_at, "updated_at": record.updated_at, - **{field_name: getattr(record, field_name) for field_name in model_spec.fields}, + **{ + f"payload.{field_name}": _payload_field_value( + record, + organization=organization, + model_spec=model_spec, + field_name=field_name, + ) + for field_name in _payload_export_field_names(model_spec) + }, } if source_spec.source_group == FINANCIAL_SOURCE_GROUP: row["financial_lines"] = [ @@ -915,33 +1044,78 @@ def _build_record_row( } for line in record.lines.all() ] - return {key: _serialize_json_value(value) for key, value in row.items()} + public_row = _sanitize_export_provider_value(row) + return {key: _serialize_json_value(value) for key, value in public_row.items()} -def _is_excluded_export_provider_record( +def _source_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any: + source_value = _record_field_value(record, model_spec.source_field) + return source_value or model_spec.source + + +def _title_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any: + title_value = _record_field_value(record, model_spec.title_field) + if not title_value or not model_spec.title_prefix: + return title_value + return f"{model_spec.title_prefix}{title_value}" + + +def _payload_field_name(model_spec: SourceModelExportSpec, field_name: str) -> str: + return dict(model_spec.payload_aliases).get(field_name, field_name) + + +def _payload_export_field_names(model_spec: SourceModelExportSpec) -> tuple[str, ...]: + if model_spec.payload_export_fields is not None: + return model_spec.payload_export_fields + + field_names = ["inn", "ogrn", "organisation_name", "source"] + for field_name in model_spec.fields: + payload_field_name = _payload_field_name(model_spec, field_name) + if payload_field_name not in field_names: + field_names.append(payload_field_name) + return tuple(field_names) + + +def _payload_field_value( record: Any, *, + organization: Any, model_spec: SourceModelExportSpec, -) -> bool: - """Return whether a provider-backed record must stay out of public files.""" - return any( - _contains_excluded_export_provider(getattr(record, field_name)) - for field_name in model_spec.fields - ) + field_name: str, +) -> Any: + if field_name == "inn": + return organization.inn + if field_name == "ogrn": + return organization.ogrn + if field_name == "organisation_name": + return organization.full_name or organization.short_name or organization.name + if field_name == "source": + return _source_value(record, model_spec=model_spec) + + source_field_by_alias = { + alias: source_field for source_field, alias in model_spec.payload_aliases + } + source_field = source_field_by_alias.get(field_name, field_name) + return getattr(record, source_field, None) -def _contains_excluded_export_provider(value: object) -> bool: +def _record_field_value(record: Any, field_name: str | None) -> Any: + return getattr(record, field_name) if field_name else None + + +def _sanitize_export_provider_value(value: Any) -> Any: + """Remove provider wording from files without changing stored source data.""" if isinstance(value, str): - return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None + sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value) + return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized) if isinstance(value, dict): - return any( - _contains_excluded_export_provider(key) - or _contains_excluded_export_provider(item) + return { + _sanitize_export_provider_value(key): _sanitize_export_provider_value(item) for key, item in value.items() - ) + } if isinstance(value, list | tuple): - return any(_contains_excluded_export_provider(item) for item in value) - return False + return [_sanitize_export_provider_value(item) for item in value] + return value def _serialize_flat_value(value: Any) -> str | int | float | bool: diff --git a/tests/apps/external_data/test_source_record_export.py b/tests/apps/external_data/test_source_record_export.py index 26aa0f6..e16aa2b 100644 --- a/tests/apps/external_data/test_source_record_export.py +++ b/tests/apps/external_data/test_source_record_export.py @@ -6,6 +6,10 @@ from io import BytesIO, StringIO from tempfile import TemporaryDirectory from apps.external_data.source_record_export import ( + ORGANIZATION_EXPORT_FIELDS, + SOURCE_GROUP_EXPORT_SPECS, + SOURCE_RECORD_EXPORT_FIELDS, + _source_group_headers, build_source_record_export_artifacts, load_current_source_record_export_generation, ) @@ -16,6 +20,7 @@ from rest_framework import status from rest_framework.test import APITestCase from tests.apps.external_data.factories import ( + ArbitrationCaseFactory, BankruptcyProcedureFactory, FinancialReportFactory, FinancialReportLineFactory, @@ -74,6 +79,7 @@ class SourceRecordExportApiTest(APITestCase): IndustrialCertificateFactory.create(organization=organization) ManufacturerRegistryEntryFactory.create(organization=organization) ProsecutorCheckFactory.create(organization=organization) + arbitration_case = ArbitrationCaseFactory.create(organization=organization) report = FinancialReportFactory.create(organization=organization) FinancialReportLineFactory.create(report=report, line_code="1600") @@ -81,7 +87,13 @@ class SourceRecordExportApiTest(APITestCase): self.assertEqual(generation.artifacts_count, 25) self.assertEqual(generation.files_count, 25) - self.assertEqual(generation.records_count, 5) + self.assertEqual(generation.records_count, 6) + expected_prefix = [*ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS] + for source_spec in SOURCE_GROUP_EXPORT_SPECS.values(): + self.assertEqual( + _source_group_headers(source_spec)[: len(expected_prefix)], + expected_prefix, + ) industrial_path = next( artifact.path for artifact in generation.artifacts @@ -98,6 +110,12 @@ class SourceRecordExportApiTest(APITestCase): }, ) self.assertEqual({row["ОКПО"] for row in industrial_rows}, {"12345678"}) + self.assertEqual( + list(industrial_rows[0])[: len(expected_prefix)], + expected_prefix, + ) + self.assertNotIn("organization", industrial_rows[0]) + self.assertIn("payload.product_name", industrial_rows[0]) financial_path = next( artifact.path @@ -107,7 +125,46 @@ class SourceRecordExportApiTest(APITestCase): financial_rows = json.loads(financial_path.read_text(encoding="utf-8")) self.assertEqual(financial_rows[0]["financial_lines"][0]["line_code"], "1600") - def test_generation_excludes_provider_records_and_keeps_okpo(self): + arbitration_path = next( + artifact.path + for artifact in generation.artifacts + if artifact.source_group == "arbitration" and artifact.file_format == "json" + ) + arbitration_row = json.loads(arbitration_path.read_text(encoding="utf-8"))[0] + self.assertEqual(arbitration_row["source"], "arbitration") + self.assertEqual( + arbitration_row["external_id"], arbitration_row["payload.case_number"] + ) + self.assertEqual( + arbitration_row["title"], + f"Дело {arbitration_row['payload.case_number']}", + ) + self.assertEqual( + arbitration_row["record_date"], arbitration_case.decision_date.isoformat() + ) + self.assertEqual(arbitration_row["status"], arbitration_row["payload.status"]) + self.assertEqual(arbitration_row["payload.role"], "defendant") + self.assertIn("payload.court", arbitration_row) + self.assertEqual(arbitration_row["payload.source"], "arbitration") + self.assertEqual(arbitration_row["payload.inn"], organization.inn) + self.assertEqual( + [key for key in arbitration_row if key.startswith("payload.")], + [ + "payload.inn", + "payload.url", + "payload.ogrn", + "payload.role", + "payload.court", + "payload.source", + "payload.status", + "payload.load_batch", + "payload.case_number", + "payload.claim_amount", + "payload.organisation_name", + ], + ) + + def test_generation_keeps_provider_records_but_hides_provider_mentions(self): organization = OrganizationFactory.create(okpo="11223344") excluded_record = BankruptcyProcedureFactory.create( organization=organization, @@ -122,7 +179,7 @@ class SourceRecordExportApiTest(APITestCase): generation = build_source_record_export_artifacts() - self.assertEqual(generation.records_count, 1) + self.assertEqual(generation.records_count, 2) artifacts = [ artifact for artifact in generation.artifacts @@ -134,14 +191,36 @@ class SourceRecordExportApiTest(APITestCase): for artifact in artifacts: if artifact.file_format == "json": exported_values = json.loads(artifact.path.read_text(encoding="utf-8")) + self.assertEqual( + [key for key in exported_values[0] if key.startswith("payload.")], + [ + "payload.inn", + "payload.url", + "payload.ogrn", + "payload.source", + "payload.messages", + "payload.load_batch", + "payload.case_number", + "payload.message_date", + "payload.message_type", + "payload.messages_count", + "payload.organisation_name", + ], + ) elif artifact.file_format == "csv": exported_values = artifact.path.read_text(encoding="utf-8-sig") else: workbook = load_workbook(artifact.path, read_only=True) exported_values = list(workbook["data"].iter_rows(values_only=True)) - self.assertNotIn(str(excluded_record.id), str(exported_values)) + serialized_values = str(exported_values) + self.assertIn(str(excluded_record.id), serialized_values) self.assertIn(str(included_record.id), str(exported_values)) self.assertIn("11223344", str(exported_values)) + self.assertNotRegex( + serialized_values, + r"(?i)(?