From 262193f1219e35aa60e687de8206dd445141be11 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 28 Sep 2026 11:27:46 +0200 Subject: [PATCH] fix(import): skip unknown organizations in integrated workbooks --- docs/integrated-structures-import.md | 74 +++++ src/apps/parsers/integrated_structures.py | 107 ++++++- src/apps/parsers/views.py | 5 +- ...est_integrated_structure_partial_import.py | 273 ++++++++++++++++++ 4 files changed, 441 insertions(+), 18 deletions(-) create mode 100644 docs/integrated-structures-import.md create mode 100644 tests/apps/parsers/test_integrated_structure_partial_import.py diff --git a/docs/integrated-structures-import.md b/docs/integrated-structures-import.md new file mode 100644 index 0000000..bb08479 --- /dev/null +++ b/docs/integrated-structures-import.md @@ -0,0 +1,74 @@ +# Частичная загрузка интегрированных структур + +`POST /api/v1/parsers/upload/integrated_structure_documents/` принимает multipart: +`workbook` (XLSX с листами `is_stages` и `is_stages_docs`) и необязательные +`attachments` (RTF). Требуются существующие права администратора загрузки. + +Backend сначала проверяет весь файл и вложения. ОКПО сопоставляются только с +организациями канонического справочника (`directory_imported_at` заполнен). +Неизвестные ОКПО пропускаются; организации автоматически не создаются. + +При наличии и известных, и неизвестных организаций HTTP-ответ остаётся 200, +`success=true`, `errors=null`. В `data` возвращаются: + +```json +{ + "status": "partial", + "records_count": 48, + "imported_organizations_count": 48, + "skipped_organizations_count": 2, + "skipped_organizations": [ + { + "okpo": "02742757", + "code": "unknown_canonical_okpo", + "message": "Организация не загружена: ОКПО отсутствует в каноническом реестре." + }, + { + "okpo": "10386058", + "code": "unknown_canonical_okpo", + "message": "Организация не загружена: ОКПО отсутствует в каноническом реестре." + } + ], + "warnings": [ + { + "code": "unknown_canonical_okpo", + "message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.", + "meta": {"okpos": ["02742757", "10386058"]} + } + ] +} +``` + +Это фрагмент ответа: также возвращаются существующие `artifact_id`, `task_id`, +`batch_id` и статистика workbook. Тот же результат доступен в BackgroundJob. +Фронтенду следует показать количество загруженных организаций и все пропуски +из `skipped_organizations`, а не трактовать `success=true` как отсутствие предупреждений. + +- Все ОКПО известны: `status=published`, список пропусков и warnings пусты. +- Все ОКПО неизвестны: HTTP 200, `status=skipped`, `reason=no_known_organizations`, + `records_count=0`, `artifact_id=null`, без `batch_id`; текущий snapshot не меняется. +- Идентичный последний опубликованный batch при неизменном сопоставлении с + реестром: `status=skipped`, `reason=identical_batch`; возвращается прежняя + статистика batch и его предупреждения, дублей нет. +- После пополнения канонического реестра тот же workbook можно загрузить повторно: + ранее пропущенные организации будут учтены. +- Два канонических совпадения одного ОКПО, повреждённый XLSX/RTF и нарушенные связи + остаются ошибками 422; известная часть при этих ошибках не публикуется. + +`structures_count`, `event_rows_count`, `document_link_rows_count`, +`unique_documents_count`, `events_without_documents_count` описывают исходный +workbook. `records_count`/`imported_organizations_count` описывают загруженную часть +batch. `snapshot_records_count` описывает весь текущий snapshot, включая +сохранённые старые записи. `imported_documents_count`, `attachments_count` и +`missing_attachments_count` относятся к загруженным организациям; +`submitted_attachments_count` — ко всем переданным и проверенным вложениям. +Полностью пропущенный файл возвращает только счётчики организаций и предупреждения. + +Частичная публикация атомарна и меняет данные только известных организаций из +файла. Прежние записи других организаций сохраняются. Документы и RTF, +связанные только с пропущенными организациями, не публикуются. Документ, +связанный также с известной организацией, включается в её данные. +Полная загрузка сохраняет прежнюю семантику замены snapshot. + +Хотфикс относится к Mostovik dev. Он не добавляет этот источник на main и не +меняет другие источники или форматы JSON-экспорта. diff --git a/src/apps/parsers/integrated_structures.py b/src/apps/parsers/integrated_structures.py index b43f13f..985536d 100644 --- a/src/apps/parsers/integrated_structures.py +++ b/src/apps/parsers/integrated_structures.py @@ -117,11 +117,55 @@ def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]: if organization.okpo in by_okpo: raise IntegratedWorkbookError("ambiguous_canonical_okpo") by_okpo[organization.okpo] = organization - if len(by_okpo) != len(parsed.structures): - raise IntegratedWorkbookError("unknown_canonical_okpo") return by_okpo +def _known_workbook( + parsed: ParsedWorkbook, organizations: dict[str, Organization] +) -> ParsedWorkbook: + structures = { + okpo: item for okpo, item in parsed.structures.items() if okpo in organizations + } + events = [event for item in structures.values() for event in item.events] + document_ids = {document_id for event in events for document_id in event.documents} + return ParsedWorkbook( + structures=structures, + documents={ + key: value for key, value in parsed.documents.items() if key in document_ids + }, + links_count=sum(len(event.documents) for event in events), + ) + + +def _resolution_metadata( + parsed: ParsedWorkbook, organizations: dict[str, Organization] +) -> dict: + missing = sorted(set(parsed.structures) - organizations.keys()) + message = "Организация не загружена: ОКПО отсутствует в каноническом реестре." + identity = "\n".join( + f"{okpo}:{organization.uid}" + for okpo, organization in sorted(organizations.items()) + ) + return { + "resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(), + "imported_organizations_count": len(organizations), + "skipped_organizations_count": len(missing), + "skipped_organizations": [ + {"okpo": okpo, "code": "unknown_canonical_okpo", "message": message} + for okpo in missing + ], + "warnings": [ + { + "code": "unknown_canonical_okpo", + "message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.", + "meta": {"okpos": missing}, + } + ] + if missing + else [], + } + + def _save_attachments( attachments: dict[str, ValidatedAttachment], ) -> dict[str, IntegratedStructureAttachment]: @@ -192,31 +236,50 @@ def import_integrated_structures( original_name: str, attachments: Iterable[BinaryIO], uploaded_by_id: int | None = None, -) -> tuple[ParserSourceArtifact, dict]: - """Validate all inputs, stage all structures, then publish one transaction.""" +) -> tuple[ParserSourceArtifact | None, dict]: + """Validate all inputs and atomically publish known organizations.""" workbook.seek(0) workbook_bytes = workbook.read(20 * 1024 * 1024 + 1) parsed = parse_integrated_workbook(BytesIO(workbook_bytes)) validated_attachments = _validated_attachments(attachments, parsed) composite_hash = _composite_hash(workbook_bytes, validated_attachments) + organizations = _resolve_organizations(parsed) + resolution = _resolution_metadata(parsed, organizations) + if not organizations: + return None, { + "status": "skipped", + "reason": "no_known_organizations", + "records_count": 0, + **resolution, + } existing = ( ParserSourceArtifact.objects.filter( source=PARSER_SOURCE, status=ParserSourceArtifact.Status.PUBLISHED, - metadata__composite_sha256=composite_hash, ) .order_by("-created_at") .first() ) - if existing is not None: + if ( + existing is not None + and existing.metadata.get("composite_sha256") == composite_hash + and existing.metadata.get("resolution_sha256") + == resolution["resolution_sha256"] + ): return existing, { "status": "skipped", "reason": "identical_batch", "batch_id": existing.load_batch, - "records_count": existing.published_count, + "records_count": existing.metadata["batch_published_records_count"], **existing.metadata, } - organizations = _resolve_organizations(parsed) + known = _known_workbook(parsed, organizations) + submitted_attachments_count = len(validated_attachments) + validated_attachments = { + key: value + for key, value in validated_attachments.items() + if key in known.documents + } load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS ) @@ -230,20 +293,23 @@ def import_integrated_structures( parsed_count=len(parsed.structures), metadata={ "composite_sha256": composite_hash, + **resolution, "structures_count": parsed.counts[0], "event_rows_count": parsed.counts[1], "document_link_rows_count": parsed.counts[2], "unique_documents_count": parsed.counts[3], "events_without_documents_count": parsed.counts[4], + "submitted_attachments_count": submitted_attachments_count, + "imported_documents_count": len(known.documents), "attachments_count": len(validated_attachments), - "missing_attachments_count": len(parsed.documents) + "missing_attachments_count": len(known.documents) - len(validated_attachments), }, ) try: save_artifact_file(artifact, BytesIO(workbook_bytes)) pending = [] - for index, (okpo, structure) in enumerate(parsed.structures.items(), 1): + for index, (okpo, structure) in enumerate(known.structures.items(), 1): organization = organizations[okpo] external_id = f"integrated-structure:{okpo}" document_ids = { @@ -284,20 +350,29 @@ def import_integrated_structures( artifact.save(update_fields=["status", "updated_at"]) def finalize(artifact, result): - _publish_relations(artifact, parsed, validated_attachments) + _publish_relations(artifact, known, validated_attachments) ParserLoadLogService.update( load_log, status=ParserLoadLog.Status.SUCCESS, - records_count=result.published, + records_count=len(organizations), ) - result = publish_snapshot( - artifact, record_source=RECORD_SOURCE, on_publish=finalize + publish_snapshot( + artifact, + record_source=RECORD_SOURCE, + on_publish=finalize, + scope_organization_ids=[ + organization.uid for organization in organizations.values() + ] + if resolution["skipped_organizations_count"] + else None, ) return artifact, { - "status": "published", + "status": "partial" + if resolution["skipped_organizations_count"] + else "published", "batch_id": batch_id, - "records_count": result.published, + "records_count": len(organizations), **artifact.metadata, } except Exception as exc: diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 8979efc..0b877dc 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -3171,9 +3171,10 @@ class ParserUploadView(APIView): except Exception as exc: job.fail(error=type(exc).__name__) raise - job.complete(result={**result, "artifact_id": str(artifact.uid)}) + artifact_id = str(artifact.uid) if artifact is not None else None + job.complete(result={**result, "artifact_id": artifact_id}) return api_response( - {**result, "artifact_id": str(artifact.uid), "task_id": job.task_id}, + {**result, "artifact_id": artifact_id, "task_id": job.task_id}, status_code=status.HTTP_200_OK, ) if not descriptor.supports_file_upload: diff --git a/tests/apps/parsers/test_integrated_structure_partial_import.py b/tests/apps/parsers/test_integrated_structure_partial_import.py new file mode 100644 index 0000000..9719b8d --- /dev/null +++ b/tests/apps/parsers/test_integrated_structure_partial_import.py @@ -0,0 +1,273 @@ +"""Partial publication of valid workbooks with organizations outside the directory.""" +from io import BytesIO +from unittest.mock import patch + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase, override_settings +from django.utils import timezone +from openpyxl import load_workbook +from organizations.models import Organization, OrganizationSourceRecord +from rest_framework.test import APIClient + +from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes +from tests.apps.user.factories import UserFactory + + +def multi_organization_workbook(*okpos): + combined = load_workbook(BytesIO(workbook_bytes(okpo=okpos[0]))) + for index, okpo in enumerate(okpos[1:], 1): + extra = load_workbook(BytesIO(workbook_bytes(okpo=okpo, doc_id=93 + index))) + for name in combined.sheetnames: + for row in extra[name].iter_rows(min_row=2, values_only=True): + combined[name].append(row) + extra.close() + stream = BytesIO() + combined.save(stream) + combined.close() + return stream.getvalue() + + +class IntegratedStructurePartialImportTest(TestCase): + def setUp(self): + from tempfile import TemporaryDirectory + + self.private = TemporaryDirectory() + self.addCleanup(self.private.cleanup) + self.settings = override_settings( + PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name, + MEDIA_ROOT=self.private.name + "/public", + ) + self.settings.enable() + self.addCleanup(self.settings.disable) + Organization.objects.create( + name="АО Тест", + inn="1234567890", + ogrn="1027700132195", + okpo="00084008", + directory_imported_at=timezone.now(), + ) + + def _upload_workbook(self, raw, attachments=()): + client = APIClient() + client.force_authenticate(UserFactory.create_superuser()) + return client.post( + "/api/v1/parsers/upload/integrated_structure_documents/", + { + "workbook": SimpleUploadedFile("test.xlsx", raw), + "attachments": list(attachments), + }, + format="multipart", + ) + + def test_partial_upload_reports_unknown_okpo_and_publishes_known_relations(self): + from apps.core.models import BackgroundJob + from apps.parsers.models import ( + IntegratedStructureDocument, + IntegratedStructureEvent, + ) + + raw = multi_organization_workbook("00084008", "02742757") + response = self._upload_workbook(raw) + self.assertEqual(response.status_code, 200) + data = response.data["data"] + self.assertEqual(data["status"], "partial") + self.assertEqual(data["records_count"], 1) + self.assertEqual(data["imported_organizations_count"], 1) + self.assertEqual(data["skipped_organizations_count"], 1) + self.assertEqual( + data["skipped_organizations"], + [ + { + "okpo": "02742757", + "code": "unknown_canonical_okpo", + "message": "Организация не загружена: ОКПО отсутствует в каноническом реестре.", + } + ], + ) + self.assertEqual(data["warnings"][0]["meta"]["okpos"], ["02742757"]) + self.assertEqual(IntegratedStructureEvent.objects.count(), 2) + self.assertEqual(IntegratedStructureDocument.objects.count(), 1) + self.assertEqual( + BackgroundJob.objects.get().result["skipped_organizations"], + data["skipped_organizations"], + ) + second = self._upload_workbook(raw).data["data"] + self.assertEqual(second["reason"], "identical_batch") + self.assertEqual(second["skipped_organizations"], data["skipped_organizations"]) + self.assertEqual(IntegratedStructureEvent.objects.count(), 2) + + def test_all_unknown_upload_preserves_existing_snapshot(self): + from apps.parsers.models import ParserSourceArtifact + + self.assertEqual(self._upload_workbook(workbook_bytes()).status_code, 200) + artifact_ids = list(ParserSourceArtifact.objects.values_list("uid", flat=True)) + response = self._upload_workbook(workbook_bytes(okpo="02742757")) + self.assertEqual(response.status_code, 200) + data = response.data["data"] + self.assertEqual(data["status"], "skipped") + self.assertEqual(data["reason"], "no_known_organizations") + self.assertEqual(data["records_count"], 0) + self.assertEqual(data["imported_organizations_count"], 0) + self.assertEqual(data["skipped_organizations"][0]["okpo"], "02742757") + self.assertIsNone(data["artifact_id"]) + self.assertEqual( + list(ParserSourceArtifact.objects.values_list("uid", flat=True)), + artifact_ids, + ) + self.assertEqual(OrganizationSourceRecord.objects.count(), 1) + + def test_partial_upload_preserves_previously_published_unknown_company(self): + other = Organization.objects.create( + name="Other", + inn="1234567891", + ogrn="1027700132196", + okpo="02742757", + directory_imported_at=timezone.now(), + ) + raw = multi_organization_workbook("00084008", "02742757") + self.assertEqual(self._upload_workbook(raw).status_code, 200) + record = OrganizationSourceRecord.objects.get(extension__organization=other) + event_ids = list(record.integrated_events.values_list("id", flat=True)) + other.directory_imported_at = None + other.save(update_fields=["directory_imported_at"]) + response = self._upload_workbook(raw) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["data"]["status"], "partial") + self.assertEqual(response.data["data"]["records_count"], 1) + self.assertEqual( + list(record.integrated_events.values_list("id", flat=True)), event_ids + ) + self.assertEqual(OrganizationSourceRecord.objects.count(), 2) + + def test_repeat_after_directory_update_imports_previously_skipped_company(self): + from apps.parsers.models import IntegratedStructureEvent + + raw = multi_organization_workbook("00084008", "02742757") + self.assertEqual(self._upload_workbook(raw).status_code, 200) + Organization.objects.create( + name="Other", + inn="1234567891", + ogrn="1027700132196", + okpo="02742757", + directory_imported_at=timezone.now(), + ) + response = self._upload_workbook(raw) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["data"]["status"], "published") + self.assertEqual(response.data["data"]["records_count"], 2) + self.assertEqual(response.data["data"]["skipped_organizations"], []) + self.assertEqual(IntegratedStructureEvent.objects.count(), 4) + + def test_reimport_older_workbook_is_not_skipped_after_another_publication(self): + first = workbook_bytes(doc_id=93) + self.assertEqual(self._upload_workbook(first).status_code, 200) + self.assertEqual( + self._upload_workbook(workbook_bytes(doc_id=94)).status_code, 200 + ) + response = self._upload_workbook(first) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["data"]["status"], "published") + record = OrganizationSourceRecord.objects.get() + self.assertEqual( + record.integrated_events.get(external_id="1137") + .document_links.get() + .document.external_id, + "93", + ) + + def test_ambiguous_canonical_okpo_still_rejects_entire_upload(self): + Organization.objects.create( + name="Ambiguous", + inn="1234567891", + ogrn="1027700132196", + okpo="00084008", + directory_imported_at=timezone.now(), + ) + response = self._upload_workbook( + multi_organization_workbook("00084008", "02742757") + ) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.data["errors"][0]["code"], "ambiguous_canonical_okpo") + self.assertEqual(OrganizationSourceRecord.objects.count(), 0) + + def test_attachments_for_unknown_organizations_are_not_published(self): + from apps.parsers.models import ( + IntegratedStructureAttachment, + IntegratedStructureDocument, + ) + + response = self._upload_workbook( + multi_organization_workbook("00084008", "02742757"), + [SimpleUploadedFile("000094.rtf", b"{\\rtf1\\ansi omitted}")], + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(IntegratedStructureAttachment.objects.count(), 0) + self.assertEqual( + list( + IntegratedStructureDocument.objects.values_list( + "external_id", flat=True + ) + ), + ["93"], + ) + + def test_partial_upload_retains_company_absent_from_current_workbook(self): + other = Organization.objects.create( + name="Existing", + inn="1234567891", + ogrn="1027700132196", + okpo="10386058", + directory_imported_at=timezone.now(), + ) + self.assertEqual( + self._upload_workbook( + multi_organization_workbook("00084008", "10386058") + ).status_code, + 200, + ) + previous = OrganizationSourceRecord.objects.get(extension__organization=other) + response = self._upload_workbook( + multi_organization_workbook("00084008", "02742757") + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["data"]["records_count"], 1) + self.assertTrue( + OrganizationSourceRecord.objects.filter(pk=previous.pk).exists() + ) + self.assertEqual(previous.integrated_events.count(), 2) + + def test_failed_partial_publication_keeps_previous_relations(self): + from apps.parsers.integrated_structures import import_integrated_structures + from apps.parsers.models import ( + IntegratedStructureDocument, + IntegratedStructureEvent, + ) + + import_integrated_structures( + workbook=BytesIO(workbook_bytes()), original_name="old.xlsx", attachments=[] + ) + previous_events = list( + IntegratedStructureEvent.objects.values_list("id", flat=True) + ) + with patch( + "apps.parsers.integrated_structures._publish_relations", + side_effect=RuntimeError("storage_failure"), + ), self.assertRaisesRegex(RuntimeError, "storage_failure"): + import_integrated_structures( + workbook=BytesIO(multi_organization_workbook("00084008", "02742757")), + original_name="partial.xlsx", + attachments=[], + ) + self.assertEqual( + list(IntegratedStructureEvent.objects.values_list("id", flat=True)), + previous_events, + ) + self.assertEqual(IntegratedStructureDocument.objects.count(), 1) + + def test_malformed_unknown_rows_still_reject_entire_workbook(self): + response = self._upload_workbook( + workbook_bytes(okpo="02742757", document_stage=9999) + ) + self.assertEqual(response.status_code, 422) + self.assertEqual(response.data["errors"][0]["code"], "unknown_stage") + self.assertEqual(OrganizationSourceRecord.objects.count(), 0)