"""Partial publication of valid workbooks with organizations outside the directory.""" from io import BytesIO from unittest.mock import patch from django.core.files.uploadedfile import SimpleUploadedFile from django.test import TestCase, override_settings from django.utils import timezone from openpyxl import load_workbook from organizations.models import Organization, OrganizationSourceRecord from rest_framework.test import APIClient from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes from tests.apps.user.factories import UserFactory def multi_organization_workbook(*okpos): combined = load_workbook(BytesIO(workbook_bytes(okpo=okpos[0]))) for index, okpo in enumerate(okpos[1:], 1): extra = load_workbook(BytesIO(workbook_bytes(okpo=okpo, doc_id=93 + index))) for name in combined.sheetnames: for row in extra[name].iter_rows(min_row=2, values_only=True): combined[name].append(row) extra.close() stream = BytesIO() combined.save(stream) combined.close() return stream.getvalue() class IntegratedStructurePartialImportTest(TestCase): def setUp(self): from tempfile import TemporaryDirectory self.private = TemporaryDirectory() self.addCleanup(self.private.cleanup) self.settings = override_settings( PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name, MEDIA_ROOT=self.private.name + "/public", ) self.settings.enable() self.addCleanup(self.settings.disable) Organization.objects.create( name="АО Тест", inn="1234567890", ogrn="1027700132195", okpo="00084008", directory_imported_at=timezone.now(), ) def _upload_workbook(self, raw, attachments=()): client = APIClient() client.force_authenticate(UserFactory.create_superuser()) return client.post( "/api/v1/parsers/upload/integrated_structure_documents/", { "workbook": SimpleUploadedFile("test.xlsx", raw), "attachments": list(attachments), }, format="multipart", ) def test_partial_upload_reports_unknown_okpo_and_publishes_known_relations(self): from apps.core.models import BackgroundJob from apps.parsers.models import ( IntegratedStructureDocument, IntegratedStructureEvent, ) raw = multi_organization_workbook("00084008", "02742757") response = self._upload_workbook(raw) self.assertEqual(response.status_code, 200) data = response.data["data"] self.assertEqual(data["status"], "partial") self.assertEqual(data["records_count"], 1) self.assertEqual(data["imported_organizations_count"], 1) self.assertEqual(data["skipped_organizations_count"], 1) self.assertEqual( data["skipped_organizations"], [ { "okpo": "02742757", "code": "unknown_canonical_okpo", "message": "Организация не загружена: ОКПО отсутствует в каноническом реестре.", } ], ) self.assertEqual(data["warnings"][0]["meta"]["okpos"], ["02742757"]) self.assertEqual(IntegratedStructureEvent.objects.count(), 2) self.assertEqual(IntegratedStructureDocument.objects.count(), 1) self.assertEqual( BackgroundJob.objects.get().result["skipped_organizations"], data["skipped_organizations"], ) second = self._upload_workbook(raw).data["data"] self.assertEqual(second["reason"], "identical_batch") self.assertEqual(second["skipped_organizations"], data["skipped_organizations"]) self.assertEqual(IntegratedStructureEvent.objects.count(), 2) def test_all_unknown_upload_preserves_existing_snapshot(self): from apps.parsers.models import ParserSourceArtifact self.assertEqual(self._upload_workbook(workbook_bytes()).status_code, 200) artifact_ids = list(ParserSourceArtifact.objects.values_list("uid", flat=True)) response = self._upload_workbook(workbook_bytes(okpo="02742757")) self.assertEqual(response.status_code, 200) data = response.data["data"] self.assertEqual(data["status"], "skipped") self.assertEqual(data["reason"], "no_known_organizations") self.assertEqual(data["records_count"], 0) self.assertEqual(data["imported_organizations_count"], 0) self.assertEqual(data["skipped_organizations"][0]["okpo"], "02742757") self.assertIsNone(data["artifact_id"]) self.assertEqual( list(ParserSourceArtifact.objects.values_list("uid", flat=True)), artifact_ids, ) self.assertEqual(OrganizationSourceRecord.objects.count(), 1) def test_partial_upload_preserves_previously_published_unknown_company(self): other = Organization.objects.create( name="Other", inn="1234567891", ogrn="1027700132196", okpo="02742757", directory_imported_at=timezone.now(), ) raw = multi_organization_workbook("00084008", "02742757") self.assertEqual(self._upload_workbook(raw).status_code, 200) record = OrganizationSourceRecord.objects.get(extension__organization=other) event_ids = list(record.integrated_events.values_list("id", flat=True)) other.directory_imported_at = None other.save(update_fields=["directory_imported_at"]) response = self._upload_workbook(raw) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["data"]["status"], "partial") self.assertEqual(response.data["data"]["records_count"], 1) self.assertEqual( list(record.integrated_events.values_list("id", flat=True)), event_ids ) self.assertEqual(OrganizationSourceRecord.objects.count(), 2) def test_repeat_after_directory_update_imports_previously_skipped_company(self): from apps.parsers.models import IntegratedStructureEvent raw = multi_organization_workbook("00084008", "02742757") self.assertEqual(self._upload_workbook(raw).status_code, 200) Organization.objects.create( name="Other", inn="1234567891", ogrn="1027700132196", okpo="02742757", directory_imported_at=timezone.now(), ) response = self._upload_workbook(raw) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["data"]["status"], "published") self.assertEqual(response.data["data"]["records_count"], 2) self.assertEqual(response.data["data"]["skipped_organizations"], []) self.assertEqual(IntegratedStructureEvent.objects.count(), 4) def test_reimport_older_workbook_is_not_skipped_after_another_publication(self): first = workbook_bytes(doc_id=93) self.assertEqual(self._upload_workbook(first).status_code, 200) self.assertEqual( self._upload_workbook(workbook_bytes(doc_id=94)).status_code, 200 ) response = self._upload_workbook(first) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["data"]["status"], "published") record = OrganizationSourceRecord.objects.get() self.assertEqual( record.integrated_events.get(external_id="1137") .document_links.get() .document.external_id, "93", ) def test_ambiguous_canonical_okpo_still_rejects_entire_upload(self): Organization.objects.create( name="Ambiguous", inn="1234567891", ogrn="1027700132196", okpo="00084008", directory_imported_at=timezone.now(), ) response = self._upload_workbook( multi_organization_workbook("00084008", "02742757") ) self.assertEqual(response.status_code, 422) self.assertEqual(response.data["errors"][0]["code"], "ambiguous_canonical_okpo") self.assertEqual(OrganizationSourceRecord.objects.count(), 0) def test_attachments_for_unknown_organizations_are_not_published(self): from apps.parsers.models import ( IntegratedStructureAttachment, IntegratedStructureDocument, ) response = self._upload_workbook( multi_organization_workbook("00084008", "02742757"), [SimpleUploadedFile("000094.rtf", b"{\\rtf1\\ansi omitted}")], ) self.assertEqual(response.status_code, 200) self.assertEqual(IntegratedStructureAttachment.objects.count(), 0) self.assertEqual( list( IntegratedStructureDocument.objects.values_list( "external_id", flat=True ) ), ["93"], ) def test_partial_upload_retains_company_absent_from_current_workbook(self): other = Organization.objects.create( name="Existing", inn="1234567891", ogrn="1027700132196", okpo="10386058", directory_imported_at=timezone.now(), ) self.assertEqual( self._upload_workbook( multi_organization_workbook("00084008", "10386058") ).status_code, 200, ) previous = OrganizationSourceRecord.objects.get(extension__organization=other) response = self._upload_workbook( multi_organization_workbook("00084008", "02742757") ) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["data"]["records_count"], 1) self.assertTrue( OrganizationSourceRecord.objects.filter(pk=previous.pk).exists() ) self.assertEqual(previous.integrated_events.count(), 2) def test_failed_partial_publication_keeps_previous_relations(self): from apps.parsers.integrated_structures import import_integrated_structures from apps.parsers.models import ( IntegratedStructureDocument, IntegratedStructureEvent, ) import_integrated_structures( workbook=BytesIO(workbook_bytes()), original_name="old.xlsx", attachments=[] ) previous_events = list( IntegratedStructureEvent.objects.values_list("id", flat=True) ) with patch( "apps.parsers.integrated_structures._publish_relations", side_effect=RuntimeError("storage_failure"), ), self.assertRaisesRegex(RuntimeError, "storage_failure"): import_integrated_structures( workbook=BytesIO(multi_organization_workbook("00084008", "02742757")), original_name="partial.xlsx", attachments=[], ) self.assertEqual( list(IntegratedStructureEvent.objects.values_list("id", flat=True)), previous_events, ) self.assertEqual(IntegratedStructureDocument.objects.count(), 1) def test_malformed_unknown_rows_still_reject_entire_workbook(self): response = self._upload_workbook( workbook_bytes(okpo="02742757", document_stage=9999) ) self.assertEqual(response.status_code, 422) self.assertEqual(response.data["errors"][0]["code"], "unknown_stage") self.assertEqual(OrganizationSourceRecord.objects.count(), 0)