274 lines
11 KiB
Python
274 lines
11 KiB
Python
"""Partial publication of valid workbooks with organizations outside the directory."""
|
||
from io import BytesIO
|
||
from unittest.mock import patch
|
||
|
||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||
from django.test import TestCase, override_settings
|
||
from django.utils import timezone
|
||
from openpyxl import load_workbook
|
||
from organizations.models import Organization, OrganizationSourceRecord
|
||
from rest_framework.test import APIClient
|
||
|
||
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
|
||
from tests.apps.user.factories import UserFactory
|
||
|
||
|
||
def multi_organization_workbook(*okpos):
|
||
combined = load_workbook(BytesIO(workbook_bytes(okpo=okpos[0])))
|
||
for index, okpo in enumerate(okpos[1:], 1):
|
||
extra = load_workbook(BytesIO(workbook_bytes(okpo=okpo, doc_id=93 + index)))
|
||
for name in combined.sheetnames:
|
||
for row in extra[name].iter_rows(min_row=2, values_only=True):
|
||
combined[name].append(row)
|
||
extra.close()
|
||
stream = BytesIO()
|
||
combined.save(stream)
|
||
combined.close()
|
||
return stream.getvalue()
|
||
|
||
|
||
class IntegratedStructurePartialImportTest(TestCase):
|
||
def setUp(self):
|
||
from tempfile import TemporaryDirectory
|
||
|
||
self.private = TemporaryDirectory()
|
||
self.addCleanup(self.private.cleanup)
|
||
self.settings = override_settings(
|
||
PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name,
|
||
MEDIA_ROOT=self.private.name + "/public",
|
||
)
|
||
self.settings.enable()
|
||
self.addCleanup(self.settings.disable)
|
||
Organization.objects.create(
|
||
name="АО Тест",
|
||
inn="1234567890",
|
||
ogrn="1027700132195",
|
||
okpo="00084008",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
|
||
def _upload_workbook(self, raw, attachments=()):
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_superuser())
|
||
return client.post(
|
||
"/api/v1/parsers/upload/integrated_structure_documents/",
|
||
{
|
||
"workbook": SimpleUploadedFile("test.xlsx", raw),
|
||
"attachments": list(attachments),
|
||
},
|
||
format="multipart",
|
||
)
|
||
|
||
def test_partial_upload_reports_unknown_okpo_and_publishes_known_relations(self):
|
||
from apps.core.models import BackgroundJob
|
||
from apps.parsers.models import (
|
||
IntegratedStructureDocument,
|
||
IntegratedStructureEvent,
|
||
)
|
||
|
||
raw = multi_organization_workbook("00084008", "02742757")
|
||
response = self._upload_workbook(raw)
|
||
self.assertEqual(response.status_code, 200)
|
||
data = response.data["data"]
|
||
self.assertEqual(data["status"], "partial")
|
||
self.assertEqual(data["records_count"], 1)
|
||
self.assertEqual(data["imported_organizations_count"], 1)
|
||
self.assertEqual(data["skipped_organizations_count"], 1)
|
||
self.assertEqual(
|
||
data["skipped_organizations"],
|
||
[
|
||
{
|
||
"okpo": "02742757",
|
||
"code": "unknown_canonical_okpo",
|
||
"message": "Организация не загружена: ОКПО отсутствует в каноническом реестре.",
|
||
}
|
||
],
|
||
)
|
||
self.assertEqual(data["warnings"][0]["meta"]["okpos"], ["02742757"])
|
||
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
|
||
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
|
||
self.assertEqual(
|
||
BackgroundJob.objects.get().result["skipped_organizations"],
|
||
data["skipped_organizations"],
|
||
)
|
||
second = self._upload_workbook(raw).data["data"]
|
||
self.assertEqual(second["reason"], "identical_batch")
|
||
self.assertEqual(second["skipped_organizations"], data["skipped_organizations"])
|
||
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
|
||
|
||
def test_all_unknown_upload_preserves_existing_snapshot(self):
|
||
from apps.parsers.models import ParserSourceArtifact
|
||
|
||
self.assertEqual(self._upload_workbook(workbook_bytes()).status_code, 200)
|
||
artifact_ids = list(ParserSourceArtifact.objects.values_list("uid", flat=True))
|
||
response = self._upload_workbook(workbook_bytes(okpo="02742757"))
|
||
self.assertEqual(response.status_code, 200)
|
||
data = response.data["data"]
|
||
self.assertEqual(data["status"], "skipped")
|
||
self.assertEqual(data["reason"], "no_known_organizations")
|
||
self.assertEqual(data["records_count"], 0)
|
||
self.assertEqual(data["imported_organizations_count"], 0)
|
||
self.assertEqual(data["skipped_organizations"][0]["okpo"], "02742757")
|
||
self.assertIsNone(data["artifact_id"])
|
||
self.assertEqual(
|
||
list(ParserSourceArtifact.objects.values_list("uid", flat=True)),
|
||
artifact_ids,
|
||
)
|
||
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
|
||
|
||
def test_partial_upload_preserves_previously_published_unknown_company(self):
|
||
other = Organization.objects.create(
|
||
name="Other",
|
||
inn="1234567891",
|
||
ogrn="1027700132196",
|
||
okpo="02742757",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
raw = multi_organization_workbook("00084008", "02742757")
|
||
self.assertEqual(self._upload_workbook(raw).status_code, 200)
|
||
record = OrganizationSourceRecord.objects.get(extension__organization=other)
|
||
event_ids = list(record.integrated_events.values_list("id", flat=True))
|
||
other.directory_imported_at = None
|
||
other.save(update_fields=["directory_imported_at"])
|
||
response = self._upload_workbook(raw)
|
||
self.assertEqual(response.status_code, 200)
|
||
self.assertEqual(response.data["data"]["status"], "partial")
|
||
self.assertEqual(response.data["data"]["records_count"], 1)
|
||
self.assertEqual(
|
||
list(record.integrated_events.values_list("id", flat=True)), event_ids
|
||
)
|
||
self.assertEqual(OrganizationSourceRecord.objects.count(), 2)
|
||
|
||
def test_repeat_after_directory_update_imports_previously_skipped_company(self):
|
||
from apps.parsers.models import IntegratedStructureEvent
|
||
|
||
raw = multi_organization_workbook("00084008", "02742757")
|
||
self.assertEqual(self._upload_workbook(raw).status_code, 200)
|
||
Organization.objects.create(
|
||
name="Other",
|
||
inn="1234567891",
|
||
ogrn="1027700132196",
|
||
okpo="02742757",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
response = self._upload_workbook(raw)
|
||
self.assertEqual(response.status_code, 200)
|
||
self.assertEqual(response.data["data"]["status"], "published")
|
||
self.assertEqual(response.data["data"]["records_count"], 2)
|
||
self.assertEqual(response.data["data"]["skipped_organizations"], [])
|
||
self.assertEqual(IntegratedStructureEvent.objects.count(), 4)
|
||
|
||
def test_reimport_older_workbook_is_not_skipped_after_another_publication(self):
|
||
first = workbook_bytes(doc_id=93)
|
||
self.assertEqual(self._upload_workbook(first).status_code, 200)
|
||
self.assertEqual(
|
||
self._upload_workbook(workbook_bytes(doc_id=94)).status_code, 200
|
||
)
|
||
response = self._upload_workbook(first)
|
||
self.assertEqual(response.status_code, 200)
|
||
self.assertEqual(response.data["data"]["status"], "published")
|
||
record = OrganizationSourceRecord.objects.get()
|
||
self.assertEqual(
|
||
record.integrated_events.get(external_id="1137")
|
||
.document_links.get()
|
||
.document.external_id,
|
||
"93",
|
||
)
|
||
|
||
def test_ambiguous_canonical_okpo_still_rejects_entire_upload(self):
|
||
Organization.objects.create(
|
||
name="Ambiguous",
|
||
inn="1234567891",
|
||
ogrn="1027700132196",
|
||
okpo="00084008",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
response = self._upload_workbook(
|
||
multi_organization_workbook("00084008", "02742757")
|
||
)
|
||
self.assertEqual(response.status_code, 422)
|
||
self.assertEqual(response.data["errors"][0]["code"], "ambiguous_canonical_okpo")
|
||
self.assertEqual(OrganizationSourceRecord.objects.count(), 0)
|
||
|
||
def test_attachments_for_unknown_organizations_are_not_published(self):
|
||
from apps.parsers.models import (
|
||
IntegratedStructureAttachment,
|
||
IntegratedStructureDocument,
|
||
)
|
||
|
||
response = self._upload_workbook(
|
||
multi_organization_workbook("00084008", "02742757"),
|
||
[SimpleUploadedFile("000094.rtf", b"{\\rtf1\\ansi omitted}")],
|
||
)
|
||
self.assertEqual(response.status_code, 200)
|
||
self.assertEqual(IntegratedStructureAttachment.objects.count(), 0)
|
||
self.assertEqual(
|
||
list(
|
||
IntegratedStructureDocument.objects.values_list(
|
||
"external_id", flat=True
|
||
)
|
||
),
|
||
["93"],
|
||
)
|
||
|
||
def test_partial_upload_retains_company_absent_from_current_workbook(self):
|
||
other = Organization.objects.create(
|
||
name="Existing",
|
||
inn="1234567891",
|
||
ogrn="1027700132196",
|
||
okpo="10386058",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
self.assertEqual(
|
||
self._upload_workbook(
|
||
multi_organization_workbook("00084008", "10386058")
|
||
).status_code,
|
||
200,
|
||
)
|
||
previous = OrganizationSourceRecord.objects.get(extension__organization=other)
|
||
response = self._upload_workbook(
|
||
multi_organization_workbook("00084008", "02742757")
|
||
)
|
||
self.assertEqual(response.status_code, 200)
|
||
self.assertEqual(response.data["data"]["records_count"], 1)
|
||
self.assertTrue(
|
||
OrganizationSourceRecord.objects.filter(pk=previous.pk).exists()
|
||
)
|
||
self.assertEqual(previous.integrated_events.count(), 2)
|
||
|
||
def test_failed_partial_publication_keeps_previous_relations(self):
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
from apps.parsers.models import (
|
||
IntegratedStructureDocument,
|
||
IntegratedStructureEvent,
|
||
)
|
||
|
||
import_integrated_structures(
|
||
workbook=BytesIO(workbook_bytes()), original_name="old.xlsx", attachments=[]
|
||
)
|
||
previous_events = list(
|
||
IntegratedStructureEvent.objects.values_list("id", flat=True)
|
||
)
|
||
with patch(
|
||
"apps.parsers.integrated_structures._publish_relations",
|
||
side_effect=RuntimeError("storage_failure"),
|
||
), self.assertRaisesRegex(RuntimeError, "storage_failure"):
|
||
import_integrated_structures(
|
||
workbook=BytesIO(multi_organization_workbook("00084008", "02742757")),
|
||
original_name="partial.xlsx",
|
||
attachments=[],
|
||
)
|
||
self.assertEqual(
|
||
list(IntegratedStructureEvent.objects.values_list("id", flat=True)),
|
||
previous_events,
|
||
)
|
||
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
|
||
|
||
def test_malformed_unknown_rows_still_reject_entire_workbook(self):
|
||
response = self._upload_workbook(
|
||
workbook_bytes(okpo="02742757", document_stage=9999)
|
||
)
|
||
self.assertEqual(response.status_code, 422)
|
||
self.assertEqual(response.data["errors"][0]["code"], "unknown_stage")
|
||
self.assertEqual(OrganizationSourceRecord.objects.count(), 0)
|