feat: import integrated structures from a ZIP archive
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-10-08 10:34:42 +02:00
parent 70dfc55f3d
commit 4e548fcc8e
8 changed files with 581 additions and 47 deletions

View File

@@ -0,0 +1,322 @@
"""ZIP upload contract, atomic import and rejection of unsafe archive members."""
import stat
import struct
from io import BytesIO
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo
import pytest
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
ParserSourceArtifact,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.utils import timezone
from openpyxl import load_workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
from tests.apps.user.factories import UserFactory
pytestmark = pytest.mark.django_db
UPLOAD_URL = "/api/v1/parsers/upload/integrated_structure_documents/"
RTF = b"{\\rtf1\\ansi document}"
def zip_bytes(entries, *, compression=ZIP_DEFLATED):
stream = BytesIO()
with ZipFile(stream, "w", compression=compression) as archive:
for name, data in entries:
archive.writestr(name, data)
return stream.getvalue()
@pytest.fixture
def archive_client(settings, tmp_path):
settings.PARSER_PRIVATE_ARTIFACT_ROOT = tmp_path / "private"
settings.MEDIA_ROOT = tmp_path / "public"
Organization.objects.create(
name="АО Тест",
inn="1234567890",
ogrn="1027700132195",
okpo="00084008",
directory_imported_at=timezone.now(),
)
client = APIClient()
client.force_authenticate(UserFactory.create_superuser())
return client
def upload_archive(client, data, *, name="structures.zip", extra=None):
return client.post(
UPLOAD_URL,
{"archive": SimpleUploadedFile(name, data), **(extra or {})},
format="multipart",
)
def test_zip_with_100_rtf_bypasses_multipart_limit_and_publishes_documents(
archive_client,
):
workbook = load_workbook(BytesIO(workbook_bytes()))
documents = workbook["is_stages_docs"]
row = list(next(documents.iter_rows(min_row=2, values_only=True)))
for document_id in range(94, 193):
row[3] = document_id
documents.append(row)
stream = BytesIO()
workbook.save(stream)
workbook.close()
archive = zip_bytes(
[("structures/Документы ИС.xlsx", stream.getvalue())]
+ [(f"structures/documents/{doc_id:06d}.rtf", RTF) for doc_id in range(93, 193)]
)
response = upload_archive(archive_client, archive)
assert response.status_code == 200
assert response.data["data"]["attachments_count"] == 100
assert response.data["data"]["missing_attachments_count"] == 0
assert IntegratedStructureDocument.objects.count() == 100
document = IntegratedStructureDocument.objects.get(external_id="93")
assert document.attachment_filename == "000093.rtf"
with document.attachment.file.open("rb") as stored:
assert stored.read() == RTF
assert BackgroundJob.objects.get().status == JobStatus.SUCCESS
repeated = upload_archive(archive_client, archive)
assert repeated.status_code == 200
assert repeated.data["data"]["reason"] == "identical_batch"
assert IntegratedStructureDocument.objects.count() == 100
def test_zip_without_rtf_preserves_missing_attachment_reporting(archive_client):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 200
assert response.data["data"]["missing_attachments_count"] == 1
assert IntegratedStructureDocument.objects.get().attachment_id is None
def test_macos_metadata_and_directories_do_not_become_attachments(archive_client):
response = upload_archive(
archive_client,
zip_bytes(
[
("structures/", b""),
("structures/test.xlsx", workbook_bytes()),
("structures/000093.rtf", RTF),
("structures/.DS_Store", b"metadata"),
("__MACOSX/structures/._000093.rtf", b"metadata"),
]
),
)
assert response.status_code == 200
assert response.data["data"]["attachments_count"] == 1
@pytest.mark.parametrize(
"member",
[
"../000093.rtf",
"/000093.rtf",
"files/../000093.rtf",
"C:/000093.rtf",
"files\\000093.rtf",
],
)
def test_unsafe_paths_reject_whole_archive_before_publication(archive_client, member):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), (member, RTF)])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
assert not OrganizationSourceRecord.objects.exists()
assert not ParserSourceArtifact.objects.exists()
assert not IntegratedStructureAttachment.objects.exists()
@pytest.mark.parametrize(
("extra_entries", "code"),
[
([("second.xlsx", b"xlsx")], "archive_workbook_count"),
([("unexpected.pdf", b"pdf")], "unsupported_archive_member"),
([("000093.rtf", b"not RTF")], "invalid_rtf_signature"),
([("999999.rtf", RTF)], "unknown_or_ambiguous_attachment_document"),
([("a/000093.rtf", RTF), ("b/000093.rtf", RTF)], "duplicate_archive_filename"),
([("93.rtf", RTF), ("000093.rtf", RTF)], "duplicate_attachment"),
],
)
def test_invalid_zip_or_document_set_has_no_partial_writes(
archive_client, extra_entries, code
):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), *extra_entries])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == code
assert not OrganizationSourceRecord.objects.exists()
assert not ParserSourceArtifact.objects.exists()
assert BackgroundJob.objects.get().status == JobStatus.FAILURE
def test_symbolic_link_is_rejected(archive_client):
link = ZipInfo("000093.rtf")
link.create_system = 3
link.external_attr = (stat.S_IFLNK | 0o777) << 16
response = upload_archive(
archive_client,
zip_bytes([("test.xlsx", workbook_bytes()), (link, b"/etc/passwd")]),
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
@pytest.mark.parametrize("data", [b"not a zip", zip_bytes([])])
def test_invalid_or_empty_zip_returns_validation_error(archive_client, data):
response = upload_archive(archive_client, data)
assert response.status_code == 422
assert not OrganizationSourceRecord.objects.exists()
def test_crc_failure_returns_validation_error_without_publication(archive_client):
raw = bytearray(
zip_bytes(
[("test.xlsx", workbook_bytes()), ("000093.rtf", RTF)],
compression=ZIP_STORED,
)
)
with ZipFile(BytesIO(raw)) as archive:
member = archive.getinfo("000093.rtf")
name_size, extra_size = struct.unpack_from("<HH", raw, member.header_offset + 26)
data_offset = member.header_offset + 30 + name_size + extra_size
raw[data_offset] ^= 1
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "invalid_archive"
assert not ParserSourceArtifact.objects.exists()
def test_old_loose_file_contract_is_rejected(archive_client):
response = archive_client.post(
UPLOAD_URL,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
format="multipart",
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
def test_extra_uploaded_files_are_rejected(archive_client):
response = upload_archive(
archive_client,
zip_bytes([("test.xlsx", workbook_bytes())]),
extra={"attachments": SimpleUploadedFile("000093.rtf", RTF)},
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
def test_zip_upload_preserves_admin_permission(archive_client):
archive_client.force_authenticate(UserFactory.create_user())
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 403
assert not BackgroundJob.objects.exists()
def test_many_loose_files_return_validation_error_instead_of_500(archive_client):
archive_client.raise_request_exception = False
response = archive_client.post(
UPLOAD_URL,
{
"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes()),
"attachments": [SimpleUploadedFile(f"{i}.rtf", RTF) for i in range(100)],
},
format="multipart",
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
assert not BackgroundJob.objects.exists()
def test_compressed_zip_is_bounded_by_expanded_size(archive_client, monkeypatch):
monkeypatch.setattr(
"apps.parsers.integrated_structure_archive.MAX_EXPANDED_BYTES", 100
)
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_expanded_size_exceeded"
assert not ParserSourceArtifact.objects.exists()
@pytest.mark.parametrize(
("member", "size", "code"),
[
("test.xlsx", 20 * 1024 * 1024 + 1, "workbook_too_large"),
("000093.rtf", 5 * 1024 * 1024 + 1, "attachment_too_large"),
],
)
def test_member_size_is_checked_before_expansion(archive_client, member, size, code):
entries = [(member, b"x" * size)]
if member.endswith(".rtf"):
entries.append(("test.xlsx", workbook_bytes()))
response = upload_archive(archive_client, zip_bytes(entries))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == code
assert not OrganizationSourceRecord.objects.exists()
def test_rtf_count_limit_is_preserved_inside_zip(archive_client):
archive = zip_bytes(
[("test.xlsx", workbook_bytes())] + [(f"{i}.rtf", RTF) for i in range(501)]
)
response = upload_archive(archive_client, archive)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "too_many_attachments"
def test_encrypted_zip_returns_validation_error(archive_client):
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
# Mark the member encrypted in both ZIP headers; no password is accepted.
struct.pack_into("<H", raw, 6, struct.unpack_from("<H", raw, 6)[0] | 1)
central = raw.index(b"PK\x01\x02")
struct.pack_into(
"<H", raw, central + 8, struct.unpack_from("<H", raw, central + 8)[0] | 1
)
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "encrypted_archive_member"
def test_invalid_utf8_zip_filename_returns_validation_error(archive_client):
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
central = raw.index(b"PK\x01\x02")
struct.pack_into("<H", raw, central + 8, 0x800)
raw[central + 46] = 0xFF
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "invalid_archive"
def test_zip_upload_size_limit_returns_413(archive_client, monkeypatch):
monkeypatch.setattr(
"apps.parsers.integrated_structure_archive.MAX_ARCHIVE_BYTES", 100
)
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 413
assert response.data["errors"][0]["code"] == "archive_too_large"
assert not BackgroundJob.objects.exists()
def test_many_zip_directory_entries_are_bounded(archive_client):
raw = zip_bytes([(f"folder-{i}/", b"") for i in range(1025)])
response = upload_archive(archive_client, raw)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "too_many_archive_entries"

View File

@@ -9,7 +9,10 @@ from openpyxl import load_workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
from tests.apps.parsers.test_integrated_structure_workbook import (
archive_bytes,
workbook_bytes,
)
from tests.apps.user.factories import UserFactory
@@ -53,8 +56,7 @@ class IntegratedStructurePartialImportTest(TestCase):
return client.post(
"/api/v1/parsers/upload/integrated_structure_documents/",
{
"workbook": SimpleUploadedFile("test.xlsx", raw),
"attachments": list(attachments),
"archive": SimpleUploadedFile("test.zip", archive_bytes(raw, attachments)),
},
format="multipart",
)

View File

@@ -1,5 +1,6 @@
from io import BytesIO
from unittest.mock import patch
from zipfile import ZIP_DEFLATED, ZipFile
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
@@ -42,6 +43,16 @@ def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137):
return stream.getvalue()
def archive_bytes(raw, attachments=()):
stream = BytesIO()
with ZipFile(stream, "w", compression=ZIP_DEFLATED) as archive:
archive.writestr("test.xlsx", raw)
for attachment in attachments:
attachment.seek(0)
archive.writestr(attachment.name, attachment.read())
return stream.getvalue()
class IntegratedWorkbookTest(SimpleTestCase):
def test_parses_relations_and_preserves_zero_padded_okpo(self):
result = parse_integrated_workbook(BytesIO(workbook_bytes()))
@@ -244,14 +255,14 @@ class IntegratedStructureImportTest(TestCase):
client.force_authenticate(user)
response = client.post(
url,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 403)
client.force_authenticate(admin)
response = client.post(
url,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 200)
@@ -275,7 +286,7 @@ class IntegratedStructureImportTest(TestCase):
):
response = client.post(
"/api/v1/parsers/upload/integrated_structure_documents/",
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 500)