Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
294 lines
13 KiB
Python
294 lines
13 KiB
Python
from io import BytesIO
|
||
from unittest.mock import patch
|
||
from zipfile import ZIP_DEFLATED, ZipFile
|
||
|
||
from apps.parsers.integrated_structure_workbook import (
|
||
IntegratedWorkbookError,
|
||
parse_integrated_workbook,
|
||
)
|
||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||
from django.test import SimpleTestCase, TestCase, override_settings
|
||
from django.utils import timezone
|
||
from openpyxl import Workbook
|
||
from organizations.models import Organization, OrganizationSourceRecord
|
||
from rest_framework.test import APIClient
|
||
|
||
from tests.apps.user.factories import UserFactory
|
||
|
||
|
||
def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137):
|
||
workbook = Workbook()
|
||
stages = workbook.active
|
||
stages.title = "is_stages"
|
||
stages.append(["okpo", "stage_id", "stage_date", "stage_prim"])
|
||
stages.append([okpo, 1137, "2026-09-01", "Первое"])
|
||
stages.append([okpo, 1245, "2026-09-02", None])
|
||
documents = workbook.create_sheet("is_stages_docs")
|
||
documents.append(
|
||
[
|
||
"okpo", "stage_id", "stage_date", "doc_id", "doc_name",
|
||
"doc_type", "doc_n", "doc_date", "dsp", "have_text",
|
||
"cons_local", "cons_web",
|
||
]
|
||
)
|
||
documents.append(
|
||
[
|
||
okpo, document_stage, "2026-09-01", doc_id, "Документ",
|
||
"Приказ", "001", "2026-09-01", 0, 1, "local://not-for-browser",
|
||
"https://online.consultant.ru/example",
|
||
]
|
||
)
|
||
stream = BytesIO()
|
||
workbook.save(stream)
|
||
return stream.getvalue()
|
||
|
||
|
||
def archive_bytes(raw, attachments=()):
|
||
stream = BytesIO()
|
||
with ZipFile(stream, "w", compression=ZIP_DEFLATED) as archive:
|
||
archive.writestr("test.xlsx", raw)
|
||
for attachment in attachments:
|
||
attachment.seek(0)
|
||
archive.writestr(attachment.name, attachment.read())
|
||
return stream.getvalue()
|
||
|
||
|
||
class IntegratedWorkbookTest(SimpleTestCase):
|
||
def test_parses_relations_and_preserves_zero_padded_okpo(self):
|
||
result = parse_integrated_workbook(BytesIO(workbook_bytes()))
|
||
self.assertEqual(result.counts, (1, 2, 1, 1, 1))
|
||
self.assertEqual(result.structures["00084008"].events[0].external_id, "1137")
|
||
self.assertEqual(result.structures["00084008"].events[1].documents, [])
|
||
self.assertEqual(result.documents["93"].number, "001")
|
||
self.assertEqual(
|
||
result.documents["93"].consultant_web_url,
|
||
"https://online.consultant.ru/example",
|
||
)
|
||
|
||
def test_rejects_numeric_okpo_because_leading_zeroes_are_unrecoverable(self):
|
||
with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_okpo"):
|
||
parse_integrated_workbook(BytesIO(workbook_bytes(okpo=84008)))
|
||
|
||
def test_rejects_orphan_document_relation(self):
|
||
with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"):
|
||
parse_integrated_workbook(
|
||
BytesIO(workbook_bytes(document_stage=9999))
|
||
)
|
||
|
||
def test_rejects_non_allowlisted_document_url(self):
|
||
value = workbook_bytes()
|
||
workbook = Workbook()
|
||
# Use an ordinary generated fixture rather than modifying ZIP internals.
|
||
from openpyxl import load_workbook
|
||
|
||
workbook = load_workbook(BytesIO(value))
|
||
workbook["is_stages_docs"]["L2"] = "https://attacker.example/file"
|
||
stream = BytesIO()
|
||
workbook.save(stream)
|
||
with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_consultant_url"):
|
||
parse_integrated_workbook(BytesIO(stream.getvalue()))
|
||
|
||
def test_rejects_duplicate_stage(self):
|
||
from openpyxl import load_workbook
|
||
|
||
workbook = load_workbook(BytesIO(workbook_bytes()))
|
||
workbook["is_stages"].append(["00084008", 1137, "2026-09-01", "duplicate"])
|
||
stream = BytesIO()
|
||
workbook.save(stream)
|
||
with self.assertRaisesRegex(IntegratedWorkbookError, "duplicate_stage"):
|
||
parse_integrated_workbook(BytesIO(stream.getvalue()))
|
||
|
||
|
||
class IntegratedStructureImportTest(TestCase):
|
||
def setUp(self):
|
||
from tempfile import TemporaryDirectory
|
||
|
||
self.private = TemporaryDirectory()
|
||
self.addCleanup(self.private.cleanup)
|
||
self.settings = override_settings(
|
||
PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name,
|
||
MEDIA_ROOT=self.private.name + "/public",
|
||
)
|
||
self.settings.enable()
|
||
self.addCleanup(self.settings.disable)
|
||
Organization.objects.create(
|
||
name="АО Тест", inn="1234567890", ogrn="1027700132195",
|
||
okpo="00084008", directory_imported_at=timezone.now(),
|
||
)
|
||
|
||
def test_atomic_publish_relations_attachment_and_idempotency(self):
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
from apps.parsers.models import (
|
||
IntegratedStructureAttachment,
|
||
IntegratedStructureDocument,
|
||
IntegratedStructureEvent,
|
||
IntegratedStructureEventDocument,
|
||
)
|
||
|
||
workbook = workbook_bytes()
|
||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||
attachment.name = "000093.rtf"
|
||
first = import_integrated_structures(
|
||
workbook=BytesIO(workbook), original_name="test.xlsx",
|
||
attachments=[attachment], uploaded_by_id=None,
|
||
)
|
||
second = import_integrated_structures(
|
||
workbook=BytesIO(workbook), original_name="test.xlsx",
|
||
attachments=[attachment], uploaded_by_id=None,
|
||
)
|
||
self.assertEqual(first[0].uid, second[0].uid)
|
||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1)
|
||
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
|
||
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
|
||
self.assertEqual(IntegratedStructureEventDocument.objects.count(), 1)
|
||
self.assertEqual(IntegratedStructureAttachment.objects.count(), 1)
|
||
|
||
def test_text_document_id_keeps_leading_zeroes_for_attachment(self):
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
from apps.parsers.models import IntegratedStructureDocument
|
||
|
||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||
attachment.name = "000093.rtf"
|
||
import_integrated_structures(
|
||
workbook=BytesIO(workbook_bytes(doc_id="000093")),
|
||
original_name="test.xlsx",
|
||
attachments=[attachment],
|
||
)
|
||
document = IntegratedStructureDocument.objects.get(external_id="000093")
|
||
self.assertEqual(document.attachment_filename, "000093.rtf")
|
||
self.assertIsNotNone(document.attachment)
|
||
|
||
def test_invalid_batch_retains_last_snapshot(self):
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
|
||
valid = workbook_bytes()
|
||
import_integrated_structures(
|
||
workbook=BytesIO(valid), original_name="test.xlsx", attachments=[]
|
||
)
|
||
with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"):
|
||
import_integrated_structures(
|
||
workbook=BytesIO(workbook_bytes(document_stage=9999)),
|
||
original_name="bad.xlsx", attachments=[],
|
||
)
|
||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1)
|
||
|
||
def test_failed_publication_preserves_rejected_artifact_and_failed_log(self):
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact
|
||
|
||
with patch(
|
||
"apps.parsers.integrated_structures._publish_relations",
|
||
side_effect=RuntimeError("storage_failure"),
|
||
), self.assertRaisesRegex(RuntimeError, "storage_failure"):
|
||
import_integrated_structures(
|
||
workbook=BytesIO(workbook_bytes()),
|
||
original_name="test.xlsx",
|
||
attachments=[],
|
||
)
|
||
artifact = ParserSourceArtifact.objects.get(source="integrated_structure_documents")
|
||
load_log = ParserLoadLog.objects.get(source="integrated_structure_documents")
|
||
self.assertEqual(artifact.status, ParserSourceArtifact.Status.REJECTED)
|
||
self.assertEqual(load_log.status, ParserLoadLog.Status.FAILED)
|
||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 0)
|
||
|
||
def test_export_has_three_relational_sheets_without_binary(self):
|
||
from tempfile import TemporaryDirectory
|
||
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
from openpyxl import load_workbook
|
||
from organizations.source_record_export import (
|
||
build_source_record_export_artifacts,
|
||
)
|
||
|
||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||
attachment.name = "000093.rtf"
|
||
import_integrated_structures(
|
||
workbook=BytesIO(workbook_bytes()), original_name="test.xlsx",
|
||
attachments=[attachment],
|
||
)
|
||
with TemporaryDirectory() as directory:
|
||
generation = build_source_record_export_artifacts(export_directory=directory)
|
||
artifact = next(
|
||
item for item in generation.artifacts
|
||
if item.source_group == "integrated_structure_documents" and item.file_format == "xlsx"
|
||
)
|
||
workbook = load_workbook(artifact.path, read_only=True)
|
||
self.assertEqual(workbook.sheetnames, ["structures", "events", "event_documents"])
|
||
self.assertEqual(sum(1 for _ in workbook["structures"].iter_rows()), 2)
|
||
self.assertEqual(sum(1 for _ in workbook["events"].iter_rows()), 3)
|
||
self.assertEqual(sum(1 for _ in workbook["event_documents"].iter_rows()), 2)
|
||
self.assertNotIn(b"{\\rtf1", artifact.path.read_bytes())
|
||
workbook.close()
|
||
|
||
def test_detail_and_authorized_attachment_stream(self):
|
||
from apps.parsers.integrated_structures import import_integrated_structures
|
||
|
||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||
attachment.name = "000093.rtf"
|
||
import_integrated_structures(
|
||
workbook=BytesIO(workbook_bytes()), original_name="test.xlsx",
|
||
attachments=[attachment],
|
||
)
|
||
record = OrganizationSourceRecord.objects.get(source="integrated_structures")
|
||
path = f"/api/v2/organization-source-records/{record.uid}/"
|
||
download = path + "documents/93/attachment/"
|
||
client = APIClient()
|
||
self.assertEqual(client.get(download).status_code, 401)
|
||
with override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True):
|
||
self.assertEqual(client.get(download).status_code, 401)
|
||
client.force_authenticate(UserFactory.create_user())
|
||
detail = client.get(path)
|
||
self.assertEqual(detail.status_code, 200)
|
||
payload = detail.data["payload"]
|
||
self.assertEqual(payload["structure"]["events_count"], 2)
|
||
self.assertEqual(len(payload["events"]), 2)
|
||
self.assertEqual(payload["events"][1]["documents"][0]["attachment"]["filename"], "000093.rtf")
|
||
response = client.get(download)
|
||
self.assertEqual(response.status_code, 200)
|
||
self.assertEqual(b"".join(response.streaming_content), b"{\\rtf1\\ansi document}")
|
||
|
||
def test_upload_requires_admin_and_reports_missing_rtf(self):
|
||
user = UserFactory.create_user()
|
||
admin = UserFactory.create_superuser()
|
||
client = APIClient()
|
||
url = "/api/v1/parsers/upload/integrated_structure_documents/"
|
||
client.force_authenticate(user)
|
||
response = client.post(
|
||
url,
|
||
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
|
||
format="multipart",
|
||
)
|
||
self.assertEqual(response.status_code, 403)
|
||
client.force_authenticate(admin)
|
||
response = client.post(
|
||
url,
|
||
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
|
||
format="multipart",
|
||
)
|
||
self.assertEqual(response.status_code, 200)
|
||
record = OrganizationSourceRecord.objects.get(source="integrated_structures")
|
||
client.force_authenticate(user)
|
||
missing = client.get(
|
||
f"/api/v2/organization-source-records/{record.uid}/documents/93/attachment/"
|
||
)
|
||
self.assertEqual(missing.status_code, 409)
|
||
self.assertEqual(missing.data["errors"][0]["code"], "attachment_unavailable")
|
||
|
||
def test_unexpected_upload_failure_finishes_background_job(self):
|
||
from apps.core.models import BackgroundJob, JobStatus
|
||
|
||
client = APIClient()
|
||
client.raise_request_exception = False
|
||
client.force_authenticate(UserFactory.create_superuser())
|
||
with patch(
|
||
"apps.parsers.integrated_structures.import_integrated_structures",
|
||
side_effect=RuntimeError("storage_failure"),
|
||
):
|
||
response = client.post(
|
||
"/api/v1/parsers/upload/integrated_structure_documents/",
|
||
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
|
||
format="multipart",
|
||
)
|
||
self.assertEqual(response.status_code, 500)
|
||
self.assertEqual(BackgroundJob.objects.get().status, JobStatus.FAILURE)
|