Files
mostovik-backend/tests/apps/parsers/test_integrated_structure_archive.py
Aleksandr Meshchryakov 4e548fcc8e
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
feat: import integrated structures from a ZIP archive
2026-10-08 10:34:42 +02:00

323 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""ZIP upload contract, atomic import and rejection of unsafe archive members."""
import stat
import struct
from io import BytesIO
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo
import pytest
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
ParserSourceArtifact,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.utils import timezone
from openpyxl import load_workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
from tests.apps.user.factories import UserFactory
pytestmark = pytest.mark.django_db
UPLOAD_URL = "/api/v1/parsers/upload/integrated_structure_documents/"
RTF = b"{\\rtf1\\ansi document}"
def zip_bytes(entries, *, compression=ZIP_DEFLATED):
stream = BytesIO()
with ZipFile(stream, "w", compression=compression) as archive:
for name, data in entries:
archive.writestr(name, data)
return stream.getvalue()
@pytest.fixture
def archive_client(settings, tmp_path):
settings.PARSER_PRIVATE_ARTIFACT_ROOT = tmp_path / "private"
settings.MEDIA_ROOT = tmp_path / "public"
Organization.objects.create(
name="АО Тест",
inn="1234567890",
ogrn="1027700132195",
okpo="00084008",
directory_imported_at=timezone.now(),
)
client = APIClient()
client.force_authenticate(UserFactory.create_superuser())
return client
def upload_archive(client, data, *, name="structures.zip", extra=None):
return client.post(
UPLOAD_URL,
{"archive": SimpleUploadedFile(name, data), **(extra or {})},
format="multipart",
)
def test_zip_with_100_rtf_bypasses_multipart_limit_and_publishes_documents(
archive_client,
):
workbook = load_workbook(BytesIO(workbook_bytes()))
documents = workbook["is_stages_docs"]
row = list(next(documents.iter_rows(min_row=2, values_only=True)))
for document_id in range(94, 193):
row[3] = document_id
documents.append(row)
stream = BytesIO()
workbook.save(stream)
workbook.close()
archive = zip_bytes(
[("structures/Документы ИС.xlsx", stream.getvalue())]
+ [(f"structures/documents/{doc_id:06d}.rtf", RTF) for doc_id in range(93, 193)]
)
response = upload_archive(archive_client, archive)
assert response.status_code == 200
assert response.data["data"]["attachments_count"] == 100
assert response.data["data"]["missing_attachments_count"] == 0
assert IntegratedStructureDocument.objects.count() == 100
document = IntegratedStructureDocument.objects.get(external_id="93")
assert document.attachment_filename == "000093.rtf"
with document.attachment.file.open("rb") as stored:
assert stored.read() == RTF
assert BackgroundJob.objects.get().status == JobStatus.SUCCESS
repeated = upload_archive(archive_client, archive)
assert repeated.status_code == 200
assert repeated.data["data"]["reason"] == "identical_batch"
assert IntegratedStructureDocument.objects.count() == 100
def test_zip_without_rtf_preserves_missing_attachment_reporting(archive_client):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 200
assert response.data["data"]["missing_attachments_count"] == 1
assert IntegratedStructureDocument.objects.get().attachment_id is None
def test_macos_metadata_and_directories_do_not_become_attachments(archive_client):
response = upload_archive(
archive_client,
zip_bytes(
[
("structures/", b""),
("structures/test.xlsx", workbook_bytes()),
("structures/000093.rtf", RTF),
("structures/.DS_Store", b"metadata"),
("__MACOSX/structures/._000093.rtf", b"metadata"),
]
),
)
assert response.status_code == 200
assert response.data["data"]["attachments_count"] == 1
@pytest.mark.parametrize(
"member",
[
"../000093.rtf",
"/000093.rtf",
"files/../000093.rtf",
"C:/000093.rtf",
"files\\000093.rtf",
],
)
def test_unsafe_paths_reject_whole_archive_before_publication(archive_client, member):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), (member, RTF)])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
assert not OrganizationSourceRecord.objects.exists()
assert not ParserSourceArtifact.objects.exists()
assert not IntegratedStructureAttachment.objects.exists()
@pytest.mark.parametrize(
("extra_entries", "code"),
[
([("second.xlsx", b"xlsx")], "archive_workbook_count"),
([("unexpected.pdf", b"pdf")], "unsupported_archive_member"),
([("000093.rtf", b"not RTF")], "invalid_rtf_signature"),
([("999999.rtf", RTF)], "unknown_or_ambiguous_attachment_document"),
([("a/000093.rtf", RTF), ("b/000093.rtf", RTF)], "duplicate_archive_filename"),
([("93.rtf", RTF), ("000093.rtf", RTF)], "duplicate_attachment"),
],
)
def test_invalid_zip_or_document_set_has_no_partial_writes(
archive_client, extra_entries, code
):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), *extra_entries])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == code
assert not OrganizationSourceRecord.objects.exists()
assert not ParserSourceArtifact.objects.exists()
assert BackgroundJob.objects.get().status == JobStatus.FAILURE
def test_symbolic_link_is_rejected(archive_client):
link = ZipInfo("000093.rtf")
link.create_system = 3
link.external_attr = (stat.S_IFLNK | 0o777) << 16
response = upload_archive(
archive_client,
zip_bytes([("test.xlsx", workbook_bytes()), (link, b"/etc/passwd")]),
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
@pytest.mark.parametrize("data", [b"not a zip", zip_bytes([])])
def test_invalid_or_empty_zip_returns_validation_error(archive_client, data):
response = upload_archive(archive_client, data)
assert response.status_code == 422
assert not OrganizationSourceRecord.objects.exists()
def test_crc_failure_returns_validation_error_without_publication(archive_client):
raw = bytearray(
zip_bytes(
[("test.xlsx", workbook_bytes()), ("000093.rtf", RTF)],
compression=ZIP_STORED,
)
)
with ZipFile(BytesIO(raw)) as archive:
member = archive.getinfo("000093.rtf")
name_size, extra_size = struct.unpack_from("<HH", raw, member.header_offset + 26)
data_offset = member.header_offset + 30 + name_size + extra_size
raw[data_offset] ^= 1
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "invalid_archive"
assert not ParserSourceArtifact.objects.exists()
def test_old_loose_file_contract_is_rejected(archive_client):
response = archive_client.post(
UPLOAD_URL,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
format="multipart",
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
def test_extra_uploaded_files_are_rejected(archive_client):
response = upload_archive(
archive_client,
zip_bytes([("test.xlsx", workbook_bytes())]),
extra={"attachments": SimpleUploadedFile("000093.rtf", RTF)},
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
def test_zip_upload_preserves_admin_permission(archive_client):
archive_client.force_authenticate(UserFactory.create_user())
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 403
assert not BackgroundJob.objects.exists()
def test_many_loose_files_return_validation_error_instead_of_500(archive_client):
archive_client.raise_request_exception = False
response = archive_client.post(
UPLOAD_URL,
{
"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes()),
"attachments": [SimpleUploadedFile(f"{i}.rtf", RTF) for i in range(100)],
},
format="multipart",
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
assert not BackgroundJob.objects.exists()
def test_compressed_zip_is_bounded_by_expanded_size(archive_client, monkeypatch):
monkeypatch.setattr(
"apps.parsers.integrated_structure_archive.MAX_EXPANDED_BYTES", 100
)
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_expanded_size_exceeded"
assert not ParserSourceArtifact.objects.exists()
@pytest.mark.parametrize(
("member", "size", "code"),
[
("test.xlsx", 20 * 1024 * 1024 + 1, "workbook_too_large"),
("000093.rtf", 5 * 1024 * 1024 + 1, "attachment_too_large"),
],
)
def test_member_size_is_checked_before_expansion(archive_client, member, size, code):
entries = [(member, b"x" * size)]
if member.endswith(".rtf"):
entries.append(("test.xlsx", workbook_bytes()))
response = upload_archive(archive_client, zip_bytes(entries))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == code
assert not OrganizationSourceRecord.objects.exists()
def test_rtf_count_limit_is_preserved_inside_zip(archive_client):
archive = zip_bytes(
[("test.xlsx", workbook_bytes())] + [(f"{i}.rtf", RTF) for i in range(501)]
)
response = upload_archive(archive_client, archive)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "too_many_attachments"
def test_encrypted_zip_returns_validation_error(archive_client):
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
# Mark the member encrypted in both ZIP headers; no password is accepted.
struct.pack_into("<H", raw, 6, struct.unpack_from("<H", raw, 6)[0] | 1)
central = raw.index(b"PK\x01\x02")
struct.pack_into(
"<H", raw, central + 8, struct.unpack_from("<H", raw, central + 8)[0] | 1
)
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "encrypted_archive_member"
def test_invalid_utf8_zip_filename_returns_validation_error(archive_client):
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
central = raw.index(b"PK\x01\x02")
struct.pack_into("<H", raw, central + 8, 0x800)
raw[central + 46] = 0xFF
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "invalid_archive"
def test_zip_upload_size_limit_returns_413(archive_client, monkeypatch):
monkeypatch.setattr(
"apps.parsers.integrated_structure_archive.MAX_ARCHIVE_BYTES", 100
)
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 413
assert response.data["errors"][0]["code"] == "archive_too_large"
assert not BackgroundJob.objects.exists()
def test_many_zip_directory_entries_are_bounded(archive_client):
raw = zip_bytes([(f"folder-{i}/", b"") for i in range(1025)])
response = upload_archive(archive_client, raw)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "too_many_archive_entries"