Files
mostovik-backend/tests/apps/parsers/test_integrated_structure_workbook.py
Aleksandr Meshchryakov 4e548fcc8e
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
feat: import integrated structures from a ZIP archive
2026-10-08 10:34:42 +02:00

294 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from io import BytesIO
from unittest.mock import patch
from zipfile import ZIP_DEFLATED, ZipFile
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
parse_integrated_workbook,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import SimpleTestCase, TestCase, override_settings
from django.utils import timezone
from openpyxl import Workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137):
workbook = Workbook()
stages = workbook.active
stages.title = "is_stages"
stages.append(["okpo", "stage_id", "stage_date", "stage_prim"])
stages.append([okpo, 1137, "2026-09-01", "Первое"])
stages.append([okpo, 1245, "2026-09-02", None])
documents = workbook.create_sheet("is_stages_docs")
documents.append(
[
"okpo", "stage_id", "stage_date", "doc_id", "doc_name",
"doc_type", "doc_n", "doc_date", "dsp", "have_text",
"cons_local", "cons_web",
]
)
documents.append(
[
okpo, document_stage, "2026-09-01", doc_id, "Документ",
"Приказ", "001", "2026-09-01", 0, 1, "local://not-for-browser",
"https://online.consultant.ru/example",
]
)
stream = BytesIO()
workbook.save(stream)
return stream.getvalue()
def archive_bytes(raw, attachments=()):
stream = BytesIO()
with ZipFile(stream, "w", compression=ZIP_DEFLATED) as archive:
archive.writestr("test.xlsx", raw)
for attachment in attachments:
attachment.seek(0)
archive.writestr(attachment.name, attachment.read())
return stream.getvalue()
class IntegratedWorkbookTest(SimpleTestCase):
def test_parses_relations_and_preserves_zero_padded_okpo(self):
result = parse_integrated_workbook(BytesIO(workbook_bytes()))
self.assertEqual(result.counts, (1, 2, 1, 1, 1))
self.assertEqual(result.structures["00084008"].events[0].external_id, "1137")
self.assertEqual(result.structures["00084008"].events[1].documents, [])
self.assertEqual(result.documents["93"].number, "001")
self.assertEqual(
result.documents["93"].consultant_web_url,
"https://online.consultant.ru/example",
)
def test_rejects_numeric_okpo_because_leading_zeroes_are_unrecoverable(self):
with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_okpo"):
parse_integrated_workbook(BytesIO(workbook_bytes(okpo=84008)))
def test_rejects_orphan_document_relation(self):
with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"):
parse_integrated_workbook(
BytesIO(workbook_bytes(document_stage=9999))
)
def test_rejects_non_allowlisted_document_url(self):
value = workbook_bytes()
workbook = Workbook()
# Use an ordinary generated fixture rather than modifying ZIP internals.
from openpyxl import load_workbook
workbook = load_workbook(BytesIO(value))
workbook["is_stages_docs"]["L2"] = "https://attacker.example/file"
stream = BytesIO()
workbook.save(stream)
with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_consultant_url"):
parse_integrated_workbook(BytesIO(stream.getvalue()))
def test_rejects_duplicate_stage(self):
from openpyxl import load_workbook
workbook = load_workbook(BytesIO(workbook_bytes()))
workbook["is_stages"].append(["00084008", 1137, "2026-09-01", "duplicate"])
stream = BytesIO()
workbook.save(stream)
with self.assertRaisesRegex(IntegratedWorkbookError, "duplicate_stage"):
parse_integrated_workbook(BytesIO(stream.getvalue()))
class IntegratedStructureImportTest(TestCase):
def setUp(self):
from tempfile import TemporaryDirectory
self.private = TemporaryDirectory()
self.addCleanup(self.private.cleanup)
self.settings = override_settings(
PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name,
MEDIA_ROOT=self.private.name + "/public",
)
self.settings.enable()
self.addCleanup(self.settings.disable)
Organization.objects.create(
name="АО Тест", inn="1234567890", ogrn="1027700132195",
okpo="00084008", directory_imported_at=timezone.now(),
)
def test_atomic_publish_relations_attachment_and_idempotency(self):
from apps.parsers.integrated_structures import import_integrated_structures
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
IntegratedStructureEvent,
IntegratedStructureEventDocument,
)
workbook = workbook_bytes()
attachment = BytesIO(b"{\\rtf1\\ansi document}")
attachment.name = "000093.rtf"
first = import_integrated_structures(
workbook=BytesIO(workbook), original_name="test.xlsx",
attachments=[attachment], uploaded_by_id=None,
)
second = import_integrated_structures(
workbook=BytesIO(workbook), original_name="test.xlsx",
attachments=[attachment], uploaded_by_id=None,
)
self.assertEqual(first[0].uid, second[0].uid)
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1)
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
self.assertEqual(IntegratedStructureEventDocument.objects.count(), 1)
self.assertEqual(IntegratedStructureAttachment.objects.count(), 1)
def test_text_document_id_keeps_leading_zeroes_for_attachment(self):
from apps.parsers.integrated_structures import import_integrated_structures
from apps.parsers.models import IntegratedStructureDocument
attachment = BytesIO(b"{\\rtf1\\ansi document}")
attachment.name = "000093.rtf"
import_integrated_structures(
workbook=BytesIO(workbook_bytes(doc_id="000093")),
original_name="test.xlsx",
attachments=[attachment],
)
document = IntegratedStructureDocument.objects.get(external_id="000093")
self.assertEqual(document.attachment_filename, "000093.rtf")
self.assertIsNotNone(document.attachment)
def test_invalid_batch_retains_last_snapshot(self):
from apps.parsers.integrated_structures import import_integrated_structures
valid = workbook_bytes()
import_integrated_structures(
workbook=BytesIO(valid), original_name="test.xlsx", attachments=[]
)
with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"):
import_integrated_structures(
workbook=BytesIO(workbook_bytes(document_stage=9999)),
original_name="bad.xlsx", attachments=[],
)
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1)
def test_failed_publication_preserves_rejected_artifact_and_failed_log(self):
from apps.parsers.integrated_structures import import_integrated_structures
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact
with patch(
"apps.parsers.integrated_structures._publish_relations",
side_effect=RuntimeError("storage_failure"),
), self.assertRaisesRegex(RuntimeError, "storage_failure"):
import_integrated_structures(
workbook=BytesIO(workbook_bytes()),
original_name="test.xlsx",
attachments=[],
)
artifact = ParserSourceArtifact.objects.get(source="integrated_structure_documents")
load_log = ParserLoadLog.objects.get(source="integrated_structure_documents")
self.assertEqual(artifact.status, ParserSourceArtifact.Status.REJECTED)
self.assertEqual(load_log.status, ParserLoadLog.Status.FAILED)
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 0)
def test_export_has_three_relational_sheets_without_binary(self):
from tempfile import TemporaryDirectory
from apps.parsers.integrated_structures import import_integrated_structures
from openpyxl import load_workbook
from organizations.source_record_export import (
build_source_record_export_artifacts,
)
attachment = BytesIO(b"{\\rtf1\\ansi document}")
attachment.name = "000093.rtf"
import_integrated_structures(
workbook=BytesIO(workbook_bytes()), original_name="test.xlsx",
attachments=[attachment],
)
with TemporaryDirectory() as directory:
generation = build_source_record_export_artifacts(export_directory=directory)
artifact = next(
item for item in generation.artifacts
if item.source_group == "integrated_structure_documents" and item.file_format == "xlsx"
)
workbook = load_workbook(artifact.path, read_only=True)
self.assertEqual(workbook.sheetnames, ["structures", "events", "event_documents"])
self.assertEqual(sum(1 for _ in workbook["structures"].iter_rows()), 2)
self.assertEqual(sum(1 for _ in workbook["events"].iter_rows()), 3)
self.assertEqual(sum(1 for _ in workbook["event_documents"].iter_rows()), 2)
self.assertNotIn(b"{\\rtf1", artifact.path.read_bytes())
workbook.close()
def test_detail_and_authorized_attachment_stream(self):
from apps.parsers.integrated_structures import import_integrated_structures
attachment = BytesIO(b"{\\rtf1\\ansi document}")
attachment.name = "000093.rtf"
import_integrated_structures(
workbook=BytesIO(workbook_bytes()), original_name="test.xlsx",
attachments=[attachment],
)
record = OrganizationSourceRecord.objects.get(source="integrated_structures")
path = f"/api/v2/organization-source-records/{record.uid}/"
download = path + "documents/93/attachment/"
client = APIClient()
self.assertEqual(client.get(download).status_code, 401)
with override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True):
self.assertEqual(client.get(download).status_code, 401)
client.force_authenticate(UserFactory.create_user())
detail = client.get(path)
self.assertEqual(detail.status_code, 200)
payload = detail.data["payload"]
self.assertEqual(payload["structure"]["events_count"], 2)
self.assertEqual(len(payload["events"]), 2)
self.assertEqual(payload["events"][1]["documents"][0]["attachment"]["filename"], "000093.rtf")
response = client.get(download)
self.assertEqual(response.status_code, 200)
self.assertEqual(b"".join(response.streaming_content), b"{\\rtf1\\ansi document}")
def test_upload_requires_admin_and_reports_missing_rtf(self):
user = UserFactory.create_user()
admin = UserFactory.create_superuser()
client = APIClient()
url = "/api/v1/parsers/upload/integrated_structure_documents/"
client.force_authenticate(user)
response = client.post(
url,
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 403)
client.force_authenticate(admin)
response = client.post(
url,
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 200)
record = OrganizationSourceRecord.objects.get(source="integrated_structures")
client.force_authenticate(user)
missing = client.get(
f"/api/v2/organization-source-records/{record.uid}/documents/93/attachment/"
)
self.assertEqual(missing.status_code, 409)
self.assertEqual(missing.data["errors"][0]["code"], "attachment_unavailable")
def test_unexpected_upload_failure_finishes_background_job(self):
from apps.core.models import BackgroundJob, JobStatus
client = APIClient()
client.raise_request_exception = False
client.force_authenticate(UserFactory.create_superuser())
with patch(
"apps.parsers.integrated_structures.import_integrated_structures",
side_effect=RuntimeError("storage_failure"),
):
response = client.post(
"/api/v1/parsers/upload/integrated_structure_documents/",
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 500)
self.assertEqual(BackgroundJob.objects.get().status, JobStatus.FAILURE)