Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
323 lines
12 KiB
Python
323 lines
12 KiB
Python
"""ZIP upload contract, atomic import and rejection of unsafe archive members."""
|
||
|
||
import stat
|
||
import struct
|
||
from io import BytesIO
|
||
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo
|
||
|
||
import pytest
|
||
from apps.core.models import BackgroundJob, JobStatus
|
||
from apps.parsers.models import (
|
||
IntegratedStructureAttachment,
|
||
IntegratedStructureDocument,
|
||
ParserSourceArtifact,
|
||
)
|
||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||
from django.utils import timezone
|
||
from openpyxl import load_workbook
|
||
from organizations.models import Organization, OrganizationSourceRecord
|
||
from rest_framework.test import APIClient
|
||
|
||
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
|
||
from tests.apps.user.factories import UserFactory
|
||
|
||
pytestmark = pytest.mark.django_db
|
||
UPLOAD_URL = "/api/v1/parsers/upload/integrated_structure_documents/"
|
||
RTF = b"{\\rtf1\\ansi document}"
|
||
|
||
|
||
def zip_bytes(entries, *, compression=ZIP_DEFLATED):
|
||
stream = BytesIO()
|
||
with ZipFile(stream, "w", compression=compression) as archive:
|
||
for name, data in entries:
|
||
archive.writestr(name, data)
|
||
return stream.getvalue()
|
||
|
||
|
||
@pytest.fixture
|
||
def archive_client(settings, tmp_path):
|
||
settings.PARSER_PRIVATE_ARTIFACT_ROOT = tmp_path / "private"
|
||
settings.MEDIA_ROOT = tmp_path / "public"
|
||
Organization.objects.create(
|
||
name="АО Тест",
|
||
inn="1234567890",
|
||
ogrn="1027700132195",
|
||
okpo="00084008",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_superuser())
|
||
return client
|
||
|
||
|
||
def upload_archive(client, data, *, name="structures.zip", extra=None):
|
||
return client.post(
|
||
UPLOAD_URL,
|
||
{"archive": SimpleUploadedFile(name, data), **(extra or {})},
|
||
format="multipart",
|
||
)
|
||
|
||
|
||
def test_zip_with_100_rtf_bypasses_multipart_limit_and_publishes_documents(
|
||
archive_client,
|
||
):
|
||
workbook = load_workbook(BytesIO(workbook_bytes()))
|
||
documents = workbook["is_stages_docs"]
|
||
row = list(next(documents.iter_rows(min_row=2, values_only=True)))
|
||
for document_id in range(94, 193):
|
||
row[3] = document_id
|
||
documents.append(row)
|
||
stream = BytesIO()
|
||
workbook.save(stream)
|
||
workbook.close()
|
||
archive = zip_bytes(
|
||
[("structures/Документы ИС.xlsx", stream.getvalue())]
|
||
+ [(f"structures/documents/{doc_id:06d}.rtf", RTF) for doc_id in range(93, 193)]
|
||
)
|
||
response = upload_archive(archive_client, archive)
|
||
assert response.status_code == 200
|
||
assert response.data["data"]["attachments_count"] == 100
|
||
assert response.data["data"]["missing_attachments_count"] == 0
|
||
assert IntegratedStructureDocument.objects.count() == 100
|
||
document = IntegratedStructureDocument.objects.get(external_id="93")
|
||
assert document.attachment_filename == "000093.rtf"
|
||
with document.attachment.file.open("rb") as stored:
|
||
assert stored.read() == RTF
|
||
assert BackgroundJob.objects.get().status == JobStatus.SUCCESS
|
||
repeated = upload_archive(archive_client, archive)
|
||
assert repeated.status_code == 200
|
||
assert repeated.data["data"]["reason"] == "identical_batch"
|
||
assert IntegratedStructureDocument.objects.count() == 100
|
||
|
||
|
||
def test_zip_without_rtf_preserves_missing_attachment_reporting(archive_client):
|
||
response = upload_archive(
|
||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.data["data"]["missing_attachments_count"] == 1
|
||
assert IntegratedStructureDocument.objects.get().attachment_id is None
|
||
|
||
|
||
def test_macos_metadata_and_directories_do_not_become_attachments(archive_client):
|
||
response = upload_archive(
|
||
archive_client,
|
||
zip_bytes(
|
||
[
|
||
("structures/", b""),
|
||
("structures/test.xlsx", workbook_bytes()),
|
||
("structures/000093.rtf", RTF),
|
||
("structures/.DS_Store", b"metadata"),
|
||
("__MACOSX/structures/._000093.rtf", b"metadata"),
|
||
]
|
||
),
|
||
)
|
||
assert response.status_code == 200
|
||
assert response.data["data"]["attachments_count"] == 1
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"member",
|
||
[
|
||
"../000093.rtf",
|
||
"/000093.rtf",
|
||
"files/../000093.rtf",
|
||
"C:/000093.rtf",
|
||
"files\\000093.rtf",
|
||
],
|
||
)
|
||
def test_unsafe_paths_reject_whole_archive_before_publication(archive_client, member):
|
||
response = upload_archive(
|
||
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), (member, RTF)])
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
|
||
assert not OrganizationSourceRecord.objects.exists()
|
||
assert not ParserSourceArtifact.objects.exists()
|
||
assert not IntegratedStructureAttachment.objects.exists()
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("extra_entries", "code"),
|
||
[
|
||
([("second.xlsx", b"xlsx")], "archive_workbook_count"),
|
||
([("unexpected.pdf", b"pdf")], "unsupported_archive_member"),
|
||
([("000093.rtf", b"not RTF")], "invalid_rtf_signature"),
|
||
([("999999.rtf", RTF)], "unknown_or_ambiguous_attachment_document"),
|
||
([("a/000093.rtf", RTF), ("b/000093.rtf", RTF)], "duplicate_archive_filename"),
|
||
([("93.rtf", RTF), ("000093.rtf", RTF)], "duplicate_attachment"),
|
||
],
|
||
)
|
||
def test_invalid_zip_or_document_set_has_no_partial_writes(
|
||
archive_client, extra_entries, code
|
||
):
|
||
response = upload_archive(
|
||
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), *extra_entries])
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == code
|
||
assert not OrganizationSourceRecord.objects.exists()
|
||
assert not ParserSourceArtifact.objects.exists()
|
||
assert BackgroundJob.objects.get().status == JobStatus.FAILURE
|
||
|
||
|
||
def test_symbolic_link_is_rejected(archive_client):
|
||
link = ZipInfo("000093.rtf")
|
||
link.create_system = 3
|
||
link.external_attr = (stat.S_IFLNK | 0o777) << 16
|
||
response = upload_archive(
|
||
archive_client,
|
||
zip_bytes([("test.xlsx", workbook_bytes()), (link, b"/etc/passwd")]),
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
|
||
|
||
|
||
@pytest.mark.parametrize("data", [b"not a zip", zip_bytes([])])
|
||
def test_invalid_or_empty_zip_returns_validation_error(archive_client, data):
|
||
response = upload_archive(archive_client, data)
|
||
assert response.status_code == 422
|
||
assert not OrganizationSourceRecord.objects.exists()
|
||
|
||
|
||
def test_crc_failure_returns_validation_error_without_publication(archive_client):
|
||
raw = bytearray(
|
||
zip_bytes(
|
||
[("test.xlsx", workbook_bytes()), ("000093.rtf", RTF)],
|
||
compression=ZIP_STORED,
|
||
)
|
||
)
|
||
with ZipFile(BytesIO(raw)) as archive:
|
||
member = archive.getinfo("000093.rtf")
|
||
name_size, extra_size = struct.unpack_from("<HH", raw, member.header_offset + 26)
|
||
data_offset = member.header_offset + 30 + name_size + extra_size
|
||
raw[data_offset] ^= 1
|
||
response = upload_archive(archive_client, bytes(raw))
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "invalid_archive"
|
||
assert not ParserSourceArtifact.objects.exists()
|
||
|
||
|
||
def test_old_loose_file_contract_is_rejected(archive_client):
|
||
response = archive_client.post(
|
||
UPLOAD_URL,
|
||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||
format="multipart",
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "archive_required"
|
||
|
||
|
||
def test_extra_uploaded_files_are_rejected(archive_client):
|
||
response = upload_archive(
|
||
archive_client,
|
||
zip_bytes([("test.xlsx", workbook_bytes())]),
|
||
extra={"attachments": SimpleUploadedFile("000093.rtf", RTF)},
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "archive_required"
|
||
|
||
|
||
def test_zip_upload_preserves_admin_permission(archive_client):
|
||
archive_client.force_authenticate(UserFactory.create_user())
|
||
response = upload_archive(
|
||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||
)
|
||
assert response.status_code == 403
|
||
assert not BackgroundJob.objects.exists()
|
||
|
||
|
||
def test_many_loose_files_return_validation_error_instead_of_500(archive_client):
|
||
archive_client.raise_request_exception = False
|
||
response = archive_client.post(
|
||
UPLOAD_URL,
|
||
{
|
||
"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes()),
|
||
"attachments": [SimpleUploadedFile(f"{i}.rtf", RTF) for i in range(100)],
|
||
},
|
||
format="multipart",
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "archive_required"
|
||
assert not BackgroundJob.objects.exists()
|
||
|
||
|
||
def test_compressed_zip_is_bounded_by_expanded_size(archive_client, monkeypatch):
|
||
monkeypatch.setattr(
|
||
"apps.parsers.integrated_structure_archive.MAX_EXPANDED_BYTES", 100
|
||
)
|
||
response = upload_archive(
|
||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||
)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "archive_expanded_size_exceeded"
|
||
assert not ParserSourceArtifact.objects.exists()
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("member", "size", "code"),
|
||
[
|
||
("test.xlsx", 20 * 1024 * 1024 + 1, "workbook_too_large"),
|
||
("000093.rtf", 5 * 1024 * 1024 + 1, "attachment_too_large"),
|
||
],
|
||
)
|
||
def test_member_size_is_checked_before_expansion(archive_client, member, size, code):
|
||
entries = [(member, b"x" * size)]
|
||
if member.endswith(".rtf"):
|
||
entries.append(("test.xlsx", workbook_bytes()))
|
||
response = upload_archive(archive_client, zip_bytes(entries))
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == code
|
||
assert not OrganizationSourceRecord.objects.exists()
|
||
|
||
|
||
def test_rtf_count_limit_is_preserved_inside_zip(archive_client):
|
||
archive = zip_bytes(
|
||
[("test.xlsx", workbook_bytes())] + [(f"{i}.rtf", RTF) for i in range(501)]
|
||
)
|
||
response = upload_archive(archive_client, archive)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "too_many_attachments"
|
||
|
||
|
||
def test_encrypted_zip_returns_validation_error(archive_client):
|
||
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
|
||
# Mark the member encrypted in both ZIP headers; no password is accepted.
|
||
struct.pack_into("<H", raw, 6, struct.unpack_from("<H", raw, 6)[0] | 1)
|
||
central = raw.index(b"PK\x01\x02")
|
||
struct.pack_into(
|
||
"<H", raw, central + 8, struct.unpack_from("<H", raw, central + 8)[0] | 1
|
||
)
|
||
response = upload_archive(archive_client, bytes(raw))
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "encrypted_archive_member"
|
||
|
||
|
||
def test_invalid_utf8_zip_filename_returns_validation_error(archive_client):
|
||
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
|
||
central = raw.index(b"PK\x01\x02")
|
||
struct.pack_into("<H", raw, central + 8, 0x800)
|
||
raw[central + 46] = 0xFF
|
||
response = upload_archive(archive_client, bytes(raw))
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "invalid_archive"
|
||
|
||
|
||
def test_zip_upload_size_limit_returns_413(archive_client, monkeypatch):
|
||
monkeypatch.setattr(
|
||
"apps.parsers.integrated_structure_archive.MAX_ARCHIVE_BYTES", 100
|
||
)
|
||
response = upload_archive(
|
||
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
|
||
)
|
||
assert response.status_code == 413
|
||
assert response.data["errors"][0]["code"] == "archive_too_large"
|
||
assert not BackgroundJob.objects.exists()
|
||
|
||
|
||
def test_many_zip_directory_entries_are_bounded(archive_client):
|
||
raw = zip_bytes([(f"folder-{i}/", b"") for i in range(1025)])
|
||
response = upload_archive(archive_client, raw)
|
||
assert response.status_code == 422
|
||
assert response.data["errors"][0]["code"] == "too_many_archive_entries"
|