feat: import integrated structures from a ZIP archive
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
322
tests/apps/parsers/test_integrated_structure_archive.py
Normal file
322
tests/apps/parsers/test_integrated_structure_archive.py
Normal file
@@ -0,0 +1,322 @@
|
||||
"""ZIP upload contract, atomic import and rejection of unsafe archive members."""
|
||||
|
||||
import stat
|
||||
import struct
|
||||
from io import BytesIO
|
||||
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo
|
||||
|
||||
import pytest
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.parsers.models import (
|
||||
IntegratedStructureAttachment,
|
||||
IntegratedStructureDocument,
|
||||
ParserSourceArtifact,
|
||||
)
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.utils import timezone
|
||||
from openpyxl import load_workbook
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
UPLOAD_URL = "/api/v1/parsers/upload/integrated_structure_documents/"
|
||||
RTF = b"{\\rtf1\\ansi document}"
|
||||
|
||||
|
||||
def zip_bytes(entries, *, compression=ZIP_DEFLATED):
|
||||
stream = BytesIO()
|
||||
with ZipFile(stream, "w", compression=compression) as archive:
|
||||
for name, data in entries:
|
||||
archive.writestr(name, data)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def archive_client(settings, tmp_path):
|
||||
settings.PARSER_PRIVATE_ARTIFACT_ROOT = tmp_path / "private"
|
||||
settings.MEDIA_ROOT = tmp_path / "public"
|
||||
Organization.objects.create(
|
||||
name="АО Тест",
|
||||
inn="1234567890",
|
||||
ogrn="1027700132195",
|
||||
okpo="00084008",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
client = APIClient()
|
||||
client.force_authenticate(UserFactory.create_superuser())
|
||||
return client
|
||||
|
||||
|
||||
def upload_archive(client, data, *, name="structures.zip", extra=None):
|
||||
return client.post(
|
||||
UPLOAD_URL,
|
||||
{"archive": SimpleUploadedFile(name, data), **(extra or {})},
|
||||
format="multipart",
|
||||
)
|
||||
|
||||
|
||||
def test_zip_with_100_rtf_bypasses_multipart_limit_and_publishes_documents(
|
||||
archive_client,
|
||||
):
|
||||
workbook = load_workbook(BytesIO(workbook_bytes()))
|
||||
documents = workbook["is_stages_docs"]
|
||||
row = list(next(documents.iter_rows(min_row=2, values_only=True)))
|
||||
for document_id in range(94, 193):
|
||||
row[3] = document_id
|
||||
documents.append(row)
|
||||
stream = BytesIO()
|
||||
workbook.save(stream)
|
||||
workbook.close()
|
||||
archive = zip_bytes(
|
||||
[("structures/Документы ИС.xlsx", stream.getvalue())]
|
||||
+ [(f"structures/documents/{doc_id:06d}.rtf", RTF) for doc_id in range(93, 193)]
|
||||
)
|
||||
response = upload_archive(archive_client, archive)
|
||||
assert response.status_code == 200
|
||||
assert response.data["data"]["attachments_count"] == 100
|
||||
assert response.data["data"]["missing_attachments_count"] == 0
|
||||
assert IntegratedStructureDocument.objects.count() == 100
|
||||
document = IntegratedStructureDocument.objects.get(external_id="93")
|
||||
assert document.attachment_filename == "000093.rtf"
|
||||
with document.attachment.file.open("rb") as stored:
|
||||
assert stored.read() == RTF
|
||||
assert BackgroundJob.objects.get().status == JobStatus.SUCCESS
|
||||
repeated = upload_archive(archive_client, archive)
|
||||
assert repeated.status_code == 200
|
||||
assert repeated.data["data"]["reason"] == "identical_batch"
|
||||
assert IntegratedStructureDocument.objects.count() == 100
|
||||
|
||||
|
||||
def test_zip_without_rtf_preserves_missing_attachment_reporting(archive_client):
|
||||
response = upload_archive(
|
||||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.data["data"]["missing_attachments_count"] == 1
|
||||
assert IntegratedStructureDocument.objects.get().attachment_id is None
|
||||
|
||||
|
||||
def test_macos_metadata_and_directories_do_not_become_attachments(archive_client):
|
||||
response = upload_archive(
|
||||
archive_client,
|
||||
zip_bytes(
|
||||
[
|
||||
("structures/", b""),
|
||||
("structures/test.xlsx", workbook_bytes()),
|
||||
("structures/000093.rtf", RTF),
|
||||
("structures/.DS_Store", b"metadata"),
|
||||
("__MACOSX/structures/._000093.rtf", b"metadata"),
|
||||
]
|
||||
),
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.data["data"]["attachments_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"member",
|
||||
[
|
||||
"../000093.rtf",
|
||||
"/000093.rtf",
|
||||
"files/../000093.rtf",
|
||||
"C:/000093.rtf",
|
||||
"files\\000093.rtf",
|
||||
],
|
||||
)
|
||||
def test_unsafe_paths_reject_whole_archive_before_publication(archive_client, member):
|
||||
response = upload_archive(
|
||||
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), (member, RTF)])
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
|
||||
assert not OrganizationSourceRecord.objects.exists()
|
||||
assert not ParserSourceArtifact.objects.exists()
|
||||
assert not IntegratedStructureAttachment.objects.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_entries", "code"),
|
||||
[
|
||||
([("second.xlsx", b"xlsx")], "archive_workbook_count"),
|
||||
([("unexpected.pdf", b"pdf")], "unsupported_archive_member"),
|
||||
([("000093.rtf", b"not RTF")], "invalid_rtf_signature"),
|
||||
([("999999.rtf", RTF)], "unknown_or_ambiguous_attachment_document"),
|
||||
([("a/000093.rtf", RTF), ("b/000093.rtf", RTF)], "duplicate_archive_filename"),
|
||||
([("93.rtf", RTF), ("000093.rtf", RTF)], "duplicate_attachment"),
|
||||
],
|
||||
)
|
||||
def test_invalid_zip_or_document_set_has_no_partial_writes(
|
||||
archive_client, extra_entries, code
|
||||
):
|
||||
response = upload_archive(
|
||||
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), *extra_entries])
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == code
|
||||
assert not OrganizationSourceRecord.objects.exists()
|
||||
assert not ParserSourceArtifact.objects.exists()
|
||||
assert BackgroundJob.objects.get().status == JobStatus.FAILURE
|
||||
|
||||
|
||||
def test_symbolic_link_is_rejected(archive_client):
|
||||
link = ZipInfo("000093.rtf")
|
||||
link.create_system = 3
|
||||
link.external_attr = (stat.S_IFLNK | 0o777) << 16
|
||||
response = upload_archive(
|
||||
archive_client,
|
||||
zip_bytes([("test.xlsx", workbook_bytes()), (link, b"/etc/passwd")]),
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data", [b"not a zip", zip_bytes([])])
|
||||
def test_invalid_or_empty_zip_returns_validation_error(archive_client, data):
|
||||
response = upload_archive(archive_client, data)
|
||||
assert response.status_code == 422
|
||||
assert not OrganizationSourceRecord.objects.exists()
|
||||
|
||||
|
||||
def test_crc_failure_returns_validation_error_without_publication(archive_client):
|
||||
raw = bytearray(
|
||||
zip_bytes(
|
||||
[("test.xlsx", workbook_bytes()), ("000093.rtf", RTF)],
|
||||
compression=ZIP_STORED,
|
||||
)
|
||||
)
|
||||
with ZipFile(BytesIO(raw)) as archive:
|
||||
member = archive.getinfo("000093.rtf")
|
||||
name_size, extra_size = struct.unpack_from("<HH", raw, member.header_offset + 26)
|
||||
data_offset = member.header_offset + 30 + name_size + extra_size
|
||||
raw[data_offset] ^= 1
|
||||
response = upload_archive(archive_client, bytes(raw))
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "invalid_archive"
|
||||
assert not ParserSourceArtifact.objects.exists()
|
||||
|
||||
|
||||
def test_old_loose_file_contract_is_rejected(archive_client):
|
||||
response = archive_client.post(
|
||||
UPLOAD_URL,
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
format="multipart",
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "archive_required"
|
||||
|
||||
|
||||
def test_extra_uploaded_files_are_rejected(archive_client):
|
||||
response = upload_archive(
|
||||
archive_client,
|
||||
zip_bytes([("test.xlsx", workbook_bytes())]),
|
||||
extra={"attachments": SimpleUploadedFile("000093.rtf", RTF)},
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "archive_required"
|
||||
|
||||
|
||||
def test_zip_upload_preserves_admin_permission(archive_client):
|
||||
archive_client.force_authenticate(UserFactory.create_user())
|
||||
response = upload_archive(
|
||||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||||
)
|
||||
assert response.status_code == 403
|
||||
assert not BackgroundJob.objects.exists()
|
||||
|
||||
|
||||
def test_many_loose_files_return_validation_error_instead_of_500(archive_client):
|
||||
archive_client.raise_request_exception = False
|
||||
response = archive_client.post(
|
||||
UPLOAD_URL,
|
||||
{
|
||||
"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes()),
|
||||
"attachments": [SimpleUploadedFile(f"{i}.rtf", RTF) for i in range(100)],
|
||||
},
|
||||
format="multipart",
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "archive_required"
|
||||
assert not BackgroundJob.objects.exists()
|
||||
|
||||
|
||||
def test_compressed_zip_is_bounded_by_expanded_size(archive_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"apps.parsers.integrated_structure_archive.MAX_EXPANDED_BYTES", 100
|
||||
)
|
||||
response = upload_archive(
|
||||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "archive_expanded_size_exceeded"
|
||||
assert not ParserSourceArtifact.objects.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("member", "size", "code"),
|
||||
[
|
||||
("test.xlsx", 20 * 1024 * 1024 + 1, "workbook_too_large"),
|
||||
("000093.rtf", 5 * 1024 * 1024 + 1, "attachment_too_large"),
|
||||
],
|
||||
)
|
||||
def test_member_size_is_checked_before_expansion(archive_client, member, size, code):
|
||||
entries = [(member, b"x" * size)]
|
||||
if member.endswith(".rtf"):
|
||||
entries.append(("test.xlsx", workbook_bytes()))
|
||||
response = upload_archive(archive_client, zip_bytes(entries))
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == code
|
||||
assert not OrganizationSourceRecord.objects.exists()
|
||||
|
||||
|
||||
def test_rtf_count_limit_is_preserved_inside_zip(archive_client):
|
||||
archive = zip_bytes(
|
||||
[("test.xlsx", workbook_bytes())] + [(f"{i}.rtf", RTF) for i in range(501)]
|
||||
)
|
||||
response = upload_archive(archive_client, archive)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "too_many_attachments"
|
||||
|
||||
|
||||
def test_encrypted_zip_returns_validation_error(archive_client):
|
||||
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
|
||||
# Mark the member encrypted in both ZIP headers; no password is accepted.
|
||||
struct.pack_into("<H", raw, 6, struct.unpack_from("<H", raw, 6)[0] | 1)
|
||||
central = raw.index(b"PK\x01\x02")
|
||||
struct.pack_into(
|
||||
"<H", raw, central + 8, struct.unpack_from("<H", raw, central + 8)[0] | 1
|
||||
)
|
||||
response = upload_archive(archive_client, bytes(raw))
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "encrypted_archive_member"
|
||||
|
||||
|
||||
def test_invalid_utf8_zip_filename_returns_validation_error(archive_client):
|
||||
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
|
||||
central = raw.index(b"PK\x01\x02")
|
||||
struct.pack_into("<H", raw, central + 8, 0x800)
|
||||
raw[central + 46] = 0xFF
|
||||
response = upload_archive(archive_client, bytes(raw))
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "invalid_archive"
|
||||
|
||||
|
||||
def test_zip_upload_size_limit_returns_413(archive_client, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"apps.parsers.integrated_structure_archive.MAX_ARCHIVE_BYTES", 100
|
||||
)
|
||||
response = upload_archive(
|
||||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||||
)
|
||||
assert response.status_code == 413
|
||||
assert response.data["errors"][0]["code"] == "archive_too_large"
|
||||
assert not BackgroundJob.objects.exists()
|
||||
|
||||
|
||||
def test_many_zip_directory_entries_are_bounded(archive_client):
|
||||
raw = zip_bytes([(f"folder-{i}/", b"") for i in range(1025)])
|
||||
response = upload_archive(archive_client, raw)
|
||||
assert response.status_code == 422
|
||||
assert response.data["errors"][0]["code"] == "too_many_archive_entries"
|
||||
@@ -9,7 +9,10 @@ from openpyxl import load_workbook
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
|
||||
from tests.apps.parsers.test_integrated_structure_workbook import (
|
||||
archive_bytes,
|
||||
workbook_bytes,
|
||||
)
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
|
||||
@@ -53,8 +56,7 @@ class IntegratedStructurePartialImportTest(TestCase):
|
||||
return client.post(
|
||||
"/api/v1/parsers/upload/integrated_structure_documents/",
|
||||
{
|
||||
"workbook": SimpleUploadedFile("test.xlsx", raw),
|
||||
"attachments": list(attachments),
|
||||
"archive": SimpleUploadedFile("test.zip", archive_bytes(raw, attachments)),
|
||||
},
|
||||
format="multipart",
|
||||
)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from io import BytesIO
|
||||
from unittest.mock import patch
|
||||
from zipfile import ZIP_DEFLATED, ZipFile
|
||||
|
||||
from apps.parsers.integrated_structure_workbook import (
|
||||
IntegratedWorkbookError,
|
||||
@@ -42,6 +43,16 @@ def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137):
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def archive_bytes(raw, attachments=()):
|
||||
stream = BytesIO()
|
||||
with ZipFile(stream, "w", compression=ZIP_DEFLATED) as archive:
|
||||
archive.writestr("test.xlsx", raw)
|
||||
for attachment in attachments:
|
||||
attachment.seek(0)
|
||||
archive.writestr(attachment.name, attachment.read())
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
class IntegratedWorkbookTest(SimpleTestCase):
|
||||
def test_parses_relations_and_preserves_zero_padded_okpo(self):
|
||||
result = parse_integrated_workbook(BytesIO(workbook_bytes()))
|
||||
@@ -244,14 +255,14 @@ class IntegratedStructureImportTest(TestCase):
|
||||
client.force_authenticate(user)
|
||||
response = client.post(
|
||||
url,
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
client.force_authenticate(admin)
|
||||
response = client.post(
|
||||
url,
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
@@ -275,7 +286,7 @@ class IntegratedStructureImportTest(TestCase):
|
||||
):
|
||||
response = client.post(
|
||||
"/api/v1/parsers/upload/integrated_structure_documents/",
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(response.status_code, 500)
|
||||
|
||||
Reference in New Issue
Block a user