from io import BytesIO from unittest.mock import patch from zipfile import ZIP_DEFLATED, ZipFile from apps.parsers.integrated_structure_workbook import ( IntegratedWorkbookError, parse_integrated_workbook, ) from django.core.files.uploadedfile import SimpleUploadedFile from django.test import SimpleTestCase, TestCase, override_settings from django.utils import timezone from openpyxl import Workbook from organizations.models import Organization, OrganizationSourceRecord from rest_framework.test import APIClient from tests.apps.user.factories import UserFactory def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137): workbook = Workbook() stages = workbook.active stages.title = "is_stages" stages.append(["okpo", "stage_id", "stage_date", "stage_prim"]) stages.append([okpo, 1137, "2026-09-01", "Первое"]) stages.append([okpo, 1245, "2026-09-02", None]) documents = workbook.create_sheet("is_stages_docs") documents.append( [ "okpo", "stage_id", "stage_date", "doc_id", "doc_name", "doc_type", "doc_n", "doc_date", "dsp", "have_text", "cons_local", "cons_web", ] ) documents.append( [ okpo, document_stage, "2026-09-01", doc_id, "Документ", "Приказ", "001", "2026-09-01", 0, 1, "local://not-for-browser", "https://online.consultant.ru/example", ] ) stream = BytesIO() workbook.save(stream) return stream.getvalue() def archive_bytes(raw, attachments=()): stream = BytesIO() with ZipFile(stream, "w", compression=ZIP_DEFLATED) as archive: archive.writestr("test.xlsx", raw) for attachment in attachments: attachment.seek(0) archive.writestr(attachment.name, attachment.read()) return stream.getvalue() class IntegratedWorkbookTest(SimpleTestCase): def test_parses_relations_and_preserves_zero_padded_okpo(self): result = parse_integrated_workbook(BytesIO(workbook_bytes())) self.assertEqual(result.counts, (1, 2, 1, 1, 1)) self.assertEqual(result.structures["00084008"].events[0].external_id, "1137") self.assertEqual(result.structures["00084008"].events[1].documents, []) self.assertEqual(result.documents["93"].number, "001") self.assertEqual( result.documents["93"].consultant_web_url, "https://online.consultant.ru/example", ) def test_rejects_numeric_okpo_because_leading_zeroes_are_unrecoverable(self): with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_okpo"): parse_integrated_workbook(BytesIO(workbook_bytes(okpo=84008))) def test_rejects_orphan_document_relation(self): with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"): parse_integrated_workbook( BytesIO(workbook_bytes(document_stage=9999)) ) def test_rejects_non_allowlisted_document_url(self): value = workbook_bytes() workbook = Workbook() # Use an ordinary generated fixture rather than modifying ZIP internals. from openpyxl import load_workbook workbook = load_workbook(BytesIO(value)) workbook["is_stages_docs"]["L2"] = "https://attacker.example/file" stream = BytesIO() workbook.save(stream) with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_consultant_url"): parse_integrated_workbook(BytesIO(stream.getvalue())) def test_rejects_duplicate_stage(self): from openpyxl import load_workbook workbook = load_workbook(BytesIO(workbook_bytes())) workbook["is_stages"].append(["00084008", 1137, "2026-09-01", "duplicate"]) stream = BytesIO() workbook.save(stream) with self.assertRaisesRegex(IntegratedWorkbookError, "duplicate_stage"): parse_integrated_workbook(BytesIO(stream.getvalue())) class IntegratedStructureImportTest(TestCase): def setUp(self): from tempfile import TemporaryDirectory self.private = TemporaryDirectory() self.addCleanup(self.private.cleanup) self.settings = override_settings( PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name, MEDIA_ROOT=self.private.name + "/public", ) self.settings.enable() self.addCleanup(self.settings.disable) Organization.objects.create( name="АО Тест", inn="1234567890", ogrn="1027700132195", okpo="00084008", directory_imported_at=timezone.now(), ) def test_atomic_publish_relations_attachment_and_idempotency(self): from apps.parsers.integrated_structures import import_integrated_structures from apps.parsers.models import ( IntegratedStructureAttachment, IntegratedStructureDocument, IntegratedStructureEvent, IntegratedStructureEventDocument, ) workbook = workbook_bytes() attachment = BytesIO(b"{\\rtf1\\ansi document}") attachment.name = "000093.rtf" first = import_integrated_structures( workbook=BytesIO(workbook), original_name="test.xlsx", attachments=[attachment], uploaded_by_id=None, ) second = import_integrated_structures( workbook=BytesIO(workbook), original_name="test.xlsx", attachments=[attachment], uploaded_by_id=None, ) self.assertEqual(first[0].uid, second[0].uid) self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1) self.assertEqual(IntegratedStructureEvent.objects.count(), 2) self.assertEqual(IntegratedStructureDocument.objects.count(), 1) self.assertEqual(IntegratedStructureEventDocument.objects.count(), 1) self.assertEqual(IntegratedStructureAttachment.objects.count(), 1) def test_text_document_id_keeps_leading_zeroes_for_attachment(self): from apps.parsers.integrated_structures import import_integrated_structures from apps.parsers.models import IntegratedStructureDocument attachment = BytesIO(b"{\\rtf1\\ansi document}") attachment.name = "000093.rtf" import_integrated_structures( workbook=BytesIO(workbook_bytes(doc_id="000093")), original_name="test.xlsx", attachments=[attachment], ) document = IntegratedStructureDocument.objects.get(external_id="000093") self.assertEqual(document.attachment_filename, "000093.rtf") self.assertIsNotNone(document.attachment) def test_invalid_batch_retains_last_snapshot(self): from apps.parsers.integrated_structures import import_integrated_structures valid = workbook_bytes() import_integrated_structures( workbook=BytesIO(valid), original_name="test.xlsx", attachments=[] ) with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"): import_integrated_structures( workbook=BytesIO(workbook_bytes(document_stage=9999)), original_name="bad.xlsx", attachments=[], ) self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1) def test_failed_publication_preserves_rejected_artifact_and_failed_log(self): from apps.parsers.integrated_structures import import_integrated_structures from apps.parsers.models import ParserLoadLog, ParserSourceArtifact with patch( "apps.parsers.integrated_structures._publish_relations", side_effect=RuntimeError("storage_failure"), ), self.assertRaisesRegex(RuntimeError, "storage_failure"): import_integrated_structures( workbook=BytesIO(workbook_bytes()), original_name="test.xlsx", attachments=[], ) artifact = ParserSourceArtifact.objects.get(source="integrated_structure_documents") load_log = ParserLoadLog.objects.get(source="integrated_structure_documents") self.assertEqual(artifact.status, ParserSourceArtifact.Status.REJECTED) self.assertEqual(load_log.status, ParserLoadLog.Status.FAILED) self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 0) def test_export_has_three_relational_sheets_without_binary(self): from tempfile import TemporaryDirectory from apps.parsers.integrated_structures import import_integrated_structures from openpyxl import load_workbook from organizations.source_record_export import ( build_source_record_export_artifacts, ) attachment = BytesIO(b"{\\rtf1\\ansi document}") attachment.name = "000093.rtf" import_integrated_structures( workbook=BytesIO(workbook_bytes()), original_name="test.xlsx", attachments=[attachment], ) with TemporaryDirectory() as directory: generation = build_source_record_export_artifacts(export_directory=directory) artifact = next( item for item in generation.artifacts if item.source_group == "integrated_structure_documents" and item.file_format == "xlsx" ) workbook = load_workbook(artifact.path, read_only=True) self.assertEqual(workbook.sheetnames, ["structures", "events", "event_documents"]) self.assertEqual(sum(1 for _ in workbook["structures"].iter_rows()), 2) self.assertEqual(sum(1 for _ in workbook["events"].iter_rows()), 3) self.assertEqual(sum(1 for _ in workbook["event_documents"].iter_rows()), 2) self.assertNotIn(b"{\\rtf1", artifact.path.read_bytes()) workbook.close() def test_detail_and_authorized_attachment_stream(self): from apps.parsers.integrated_structures import import_integrated_structures attachment = BytesIO(b"{\\rtf1\\ansi document}") attachment.name = "000093.rtf" import_integrated_structures( workbook=BytesIO(workbook_bytes()), original_name="test.xlsx", attachments=[attachment], ) record = OrganizationSourceRecord.objects.get(source="integrated_structures") path = f"/api/v2/organization-source-records/{record.uid}/" download = path + "documents/93/attachment/" client = APIClient() self.assertEqual(client.get(download).status_code, 401) with override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True): self.assertEqual(client.get(download).status_code, 401) client.force_authenticate(UserFactory.create_user()) detail = client.get(path) self.assertEqual(detail.status_code, 200) payload = detail.data["payload"] self.assertEqual(payload["structure"]["events_count"], 2) self.assertEqual(len(payload["events"]), 2) self.assertEqual(payload["events"][1]["documents"][0]["attachment"]["filename"], "000093.rtf") response = client.get(download) self.assertEqual(response.status_code, 200) self.assertEqual(b"".join(response.streaming_content), b"{\\rtf1\\ansi document}") def test_upload_requires_admin_and_reports_missing_rtf(self): user = UserFactory.create_user() admin = UserFactory.create_superuser() client = APIClient() url = "/api/v1/parsers/upload/integrated_structure_documents/" client.force_authenticate(user) response = client.post( url, {"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))}, format="multipart", ) self.assertEqual(response.status_code, 403) client.force_authenticate(admin) response = client.post( url, {"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))}, format="multipart", ) self.assertEqual(response.status_code, 200) record = OrganizationSourceRecord.objects.get(source="integrated_structures") client.force_authenticate(user) missing = client.get( f"/api/v2/organization-source-records/{record.uid}/documents/93/attachment/" ) self.assertEqual(missing.status_code, 409) self.assertEqual(missing.data["errors"][0]["code"], "attachment_unavailable") def test_unexpected_upload_failure_finishes_background_job(self): from apps.core.models import BackgroundJob, JobStatus client = APIClient() client.raise_request_exception = False client.force_authenticate(UserFactory.create_superuser()) with patch( "apps.parsers.integrated_structures.import_integrated_structures", side_effect=RuntimeError("storage_failure"), ): response = client.post( "/api/v1/parsers/upload/integrated_structure_documents/", {"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))}, format="multipart", ) self.assertEqual(response.status_code, 500) self.assertEqual(BackgroundJob.objects.get().status, JobStatus.FAILURE)