Files
mostovik-backend/tests/apps/parsers/test_integrated_structure_partial_import.py
Aleksandr Meshchryakov 4e548fcc8e
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
feat: import integrated structures from a ZIP archive
2026-10-08 10:34:42 +02:00

276 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Partial publication of valid workbooks with organizations outside the directory."""
from io import BytesIO
from unittest.mock import patch
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase, override_settings
from django.utils import timezone
from openpyxl import load_workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.parsers.test_integrated_structure_workbook import (
archive_bytes,
workbook_bytes,
)
from tests.apps.user.factories import UserFactory
def multi_organization_workbook(*okpos):
combined = load_workbook(BytesIO(workbook_bytes(okpo=okpos[0])))
for index, okpo in enumerate(okpos[1:], 1):
extra = load_workbook(BytesIO(workbook_bytes(okpo=okpo, doc_id=93 + index)))
for name in combined.sheetnames:
for row in extra[name].iter_rows(min_row=2, values_only=True):
combined[name].append(row)
extra.close()
stream = BytesIO()
combined.save(stream)
combined.close()
return stream.getvalue()
class IntegratedStructurePartialImportTest(TestCase):
def setUp(self):
from tempfile import TemporaryDirectory
self.private = TemporaryDirectory()
self.addCleanup(self.private.cleanup)
self.settings = override_settings(
PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name,
MEDIA_ROOT=self.private.name + "/public",
)
self.settings.enable()
self.addCleanup(self.settings.disable)
Organization.objects.create(
name="АО Тест",
inn="1234567890",
ogrn="1027700132195",
okpo="00084008",
directory_imported_at=timezone.now(),
)
def _upload_workbook(self, raw, attachments=()):
client = APIClient()
client.force_authenticate(UserFactory.create_superuser())
return client.post(
"/api/v1/parsers/upload/integrated_structure_documents/",
{
"archive": SimpleUploadedFile("test.zip", archive_bytes(raw, attachments)),
},
format="multipart",
)
def test_partial_upload_reports_unknown_okpo_and_publishes_known_relations(self):
from apps.core.models import BackgroundJob
from apps.parsers.models import (
IntegratedStructureDocument,
IntegratedStructureEvent,
)
raw = multi_organization_workbook("00084008", "02742757")
response = self._upload_workbook(raw)
self.assertEqual(response.status_code, 200)
data = response.data["data"]
self.assertEqual(data["status"], "partial")
self.assertEqual(data["records_count"], 1)
self.assertEqual(data["imported_organizations_count"], 1)
self.assertEqual(data["skipped_organizations_count"], 1)
self.assertEqual(
data["skipped_organizations"],
[
{
"okpo": "02742757",
"code": "unknown_canonical_okpo",
"message": "Организация не загружена: ОКПО отсутствует в каноническом реестре.",
}
],
)
self.assertEqual(data["warnings"][0]["meta"]["okpos"], ["02742757"])
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
self.assertEqual(
BackgroundJob.objects.get().result["skipped_organizations"],
data["skipped_organizations"],
)
second = self._upload_workbook(raw).data["data"]
self.assertEqual(second["reason"], "identical_batch")
self.assertEqual(second["skipped_organizations"], data["skipped_organizations"])
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
def test_all_unknown_upload_preserves_existing_snapshot(self):
from apps.parsers.models import ParserSourceArtifact
self.assertEqual(self._upload_workbook(workbook_bytes()).status_code, 200)
artifact_ids = list(ParserSourceArtifact.objects.values_list("uid", flat=True))
response = self._upload_workbook(workbook_bytes(okpo="02742757"))
self.assertEqual(response.status_code, 200)
data = response.data["data"]
self.assertEqual(data["status"], "skipped")
self.assertEqual(data["reason"], "no_known_organizations")
self.assertEqual(data["records_count"], 0)
self.assertEqual(data["imported_organizations_count"], 0)
self.assertEqual(data["skipped_organizations"][0]["okpo"], "02742757")
self.assertIsNone(data["artifact_id"])
self.assertEqual(
list(ParserSourceArtifact.objects.values_list("uid", flat=True)),
artifact_ids,
)
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
def test_partial_upload_preserves_previously_published_unknown_company(self):
other = Organization.objects.create(
name="Other",
inn="1234567891",
ogrn="1027700132196",
okpo="02742757",
directory_imported_at=timezone.now(),
)
raw = multi_organization_workbook("00084008", "02742757")
self.assertEqual(self._upload_workbook(raw).status_code, 200)
record = OrganizationSourceRecord.objects.get(extension__organization=other)
event_ids = list(record.integrated_events.values_list("id", flat=True))
other.directory_imported_at = None
other.save(update_fields=["directory_imported_at"])
response = self._upload_workbook(raw)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["data"]["status"], "partial")
self.assertEqual(response.data["data"]["records_count"], 1)
self.assertEqual(
list(record.integrated_events.values_list("id", flat=True)), event_ids
)
self.assertEqual(OrganizationSourceRecord.objects.count(), 2)
def test_repeat_after_directory_update_imports_previously_skipped_company(self):
from apps.parsers.models import IntegratedStructureEvent
raw = multi_organization_workbook("00084008", "02742757")
self.assertEqual(self._upload_workbook(raw).status_code, 200)
Organization.objects.create(
name="Other",
inn="1234567891",
ogrn="1027700132196",
okpo="02742757",
directory_imported_at=timezone.now(),
)
response = self._upload_workbook(raw)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["data"]["status"], "published")
self.assertEqual(response.data["data"]["records_count"], 2)
self.assertEqual(response.data["data"]["skipped_organizations"], [])
self.assertEqual(IntegratedStructureEvent.objects.count(), 4)
def test_reimport_older_workbook_is_not_skipped_after_another_publication(self):
first = workbook_bytes(doc_id=93)
self.assertEqual(self._upload_workbook(first).status_code, 200)
self.assertEqual(
self._upload_workbook(workbook_bytes(doc_id=94)).status_code, 200
)
response = self._upload_workbook(first)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["data"]["status"], "published")
record = OrganizationSourceRecord.objects.get()
self.assertEqual(
record.integrated_events.get(external_id="1137")
.document_links.get()
.document.external_id,
"93",
)
def test_ambiguous_canonical_okpo_still_rejects_entire_upload(self):
Organization.objects.create(
name="Ambiguous",
inn="1234567891",
ogrn="1027700132196",
okpo="00084008",
directory_imported_at=timezone.now(),
)
response = self._upload_workbook(
multi_organization_workbook("00084008", "02742757")
)
self.assertEqual(response.status_code, 422)
self.assertEqual(response.data["errors"][0]["code"], "ambiguous_canonical_okpo")
self.assertEqual(OrganizationSourceRecord.objects.count(), 0)
def test_attachments_for_unknown_organizations_are_not_published(self):
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
)
response = self._upload_workbook(
multi_organization_workbook("00084008", "02742757"),
[SimpleUploadedFile("000094.rtf", b"{\\rtf1\\ansi omitted}")],
)
self.assertEqual(response.status_code, 200)
self.assertEqual(IntegratedStructureAttachment.objects.count(), 0)
self.assertEqual(
list(
IntegratedStructureDocument.objects.values_list(
"external_id", flat=True
)
),
["93"],
)
def test_partial_upload_retains_company_absent_from_current_workbook(self):
other = Organization.objects.create(
name="Existing",
inn="1234567891",
ogrn="1027700132196",
okpo="10386058",
directory_imported_at=timezone.now(),
)
self.assertEqual(
self._upload_workbook(
multi_organization_workbook("00084008", "10386058")
).status_code,
200,
)
previous = OrganizationSourceRecord.objects.get(extension__organization=other)
response = self._upload_workbook(
multi_organization_workbook("00084008", "02742757")
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data["data"]["records_count"], 1)
self.assertTrue(
OrganizationSourceRecord.objects.filter(pk=previous.pk).exists()
)
self.assertEqual(previous.integrated_events.count(), 2)
def test_failed_partial_publication_keeps_previous_relations(self):
from apps.parsers.integrated_structures import import_integrated_structures
from apps.parsers.models import (
IntegratedStructureDocument,
IntegratedStructureEvent,
)
import_integrated_structures(
workbook=BytesIO(workbook_bytes()), original_name="old.xlsx", attachments=[]
)
previous_events = list(
IntegratedStructureEvent.objects.values_list("id", flat=True)
)
with patch(
"apps.parsers.integrated_structures._publish_relations",
side_effect=RuntimeError("storage_failure"),
), self.assertRaisesRegex(RuntimeError, "storage_failure"):
import_integrated_structures(
workbook=BytesIO(multi_organization_workbook("00084008", "02742757")),
original_name="partial.xlsx",
attachments=[],
)
self.assertEqual(
list(IntegratedStructureEvent.objects.values_list("id", flat=True)),
previous_events,
)
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
def test_malformed_unknown_rows_still_reject_entire_workbook(self):
response = self._upload_workbook(
workbook_bytes(okpo="02742757", document_stage=9999)
)
self.assertEqual(response.status_code, 422)
self.assertEqual(response.data["errors"][0]["code"], "unknown_stage")
self.assertEqual(OrganizationSourceRecord.objects.count(), 0)