feat: import integrated structures from a ZIP archive
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-10-08 10:34:42 +02:00
parent 70dfc55f3d
commit 4e548fcc8e
8 changed files with 581 additions and 47 deletions

View File

@@ -1,8 +1,50 @@
# Частичная загрузка интегрированных структур
# Загрузка интегрированных структур из ZIP
`POST /api/v1/parsers/upload/integrated_structure_documents/` принимает multipart:
`workbook` (XLSX с листами `is_stages` и `is_stages_docs`) и необязательные
`attachments` (RTF). Требуются существующие права администратора загрузки.
одно файловое поле `archive` с ZIP. Требуются существующие права администратора
загрузки. Прежние поля `workbook` и `attachments` больше не принимаются;
frontend и backend следует обновлять согласованно.
В nginx frontend для этого endpoint установлен предел 126 МиБ с запасом
для multipart-заголовков; размер самого ZIP проверяет backend (125 МиБ).
Внутри архива должен быть ровно один XLSX с листами `is_stages` и
`is_stages_docs`; RTF необязательны. Допускаются подпапки. Имя каждого RTF
должно быть числовым `doc_id` из Excel (например, `000093.rtf` для `doc_id=93`).
Имена файлов должны быть уникальны без учёта регистра, даже в разных подпапках.
Служебные `.DS_Store` и файлы внутри `__MACOSX/` игнорируются.
Ограничения:
- ZIP: до 125 МиБ; до 1024 записей, включая директории и служебные файлы.
- XLSX: до 20 МиБ.
- RTF: до 500 файлов, до 5 МиБ каждый и до 100 МиБ суммарно.
- Всё распакованное содержимое: до 120 МиБ, включая служебные файлы.
- Допустимы ZIP без сжатия или с Deflate; шифрование, символические ссылки,
абсолютные пути, `..`, обратные слеши и посторонние файлы запрещены.
Backend читает архив в ограниченные временные потоки без извлечения по путям
из ZIP. После проверки архива запускается прежний синхронный XLSX/RTF-импортёр;
временные потоки закрываются и при ошибках. Hash/idempotency, атомарная
публикация и статистика импорта сохраняются. Артефактом остаётся исходный XLSX.
Передача одного ZIP устраняет ограничение Django на количество файлов multipart;
лимит самого импортёра на 500 RTF сохраняется.
Пример запроса (с уже настроенной авторизацией клиента):
```text
POST /api/v1/parsers/upload/integrated_structure_documents/
Content-Type: multipart/form-data; boundary=...
archive: structures.zip
```
Отсутствующий ZIP, лишние файлы multipart или старый формат запроса возвращают
422 `archive_required`. ZIP больше 125 МиБ возвращает 413 `archive_too_large`.
Некорректный ZIP, опасные пути, нарушения лимитов содержимого и ошибки XLSX/RTF
возвращают 422 с кодом причины; данные при этом не публикуются. В успешном
ответе остаются HTTP 200 и прежние поля `data`, `artifact_id` и `task_id`.
## Частичная загрузка
Backend сначала проверяет весь файл и вложения. ОКПО сопоставляются только с
организациями канонического справочника (`directory_imported_at` заполнен).

View File

@@ -7,8 +7,7 @@ from apps.parsers.serializers import (
)
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
from apps.parsers.views import (
INTEGRATED_ATTACHMENT_PARAM,
INTEGRATED_WORKBOOK_PARAM,
INTEGRATED_ARCHIVE_PARAM,
MEDIA_NEWS_UPLOAD_FILE_PARAM,
RESULT_DETAIL_PARAMS,
RESULT_LIST_PARAMS,
@@ -125,11 +124,14 @@ def _upload_view(descriptor: ParserSourceDescriptor):
@swagger_auto_schema(
operation_summary=f"{descriptor.title}: загрузить файл",
operation_description=(
f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
"Загрузка одного ZIP с XLSX и RTF. Архив проверяется и "
"обрабатывается синхронно существующим импортёром."
if descriptor.key == "integrated_structure_documents"
else f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
"Файл обрабатывается через Celery."
),
manual_parameters=(
[INTEGRATED_WORKBOOK_PARAM, INTEGRATED_ATTACHMENT_PARAM]
[INTEGRATED_ARCHIVE_PARAM]
if descriptor.key == "integrated_structure_documents"
else [
MEDIA_NEWS_UPLOAD_FILE_PARAM
@@ -151,7 +153,7 @@ def _upload_view(descriptor: ParserSourceDescriptor):
409: "Загрузка уже выполняется",
413: "Файл превышает лимит",
415: "Неподдерживаемый тип файла",
422: "Неверные листы, идентификаторы или вложения",
422: "Некорректный архив, листы, идентификаторы или вложения",
},
)
def post(self, request):

View File

@@ -0,0 +1,147 @@
"""Bounded ZIP expansion into temporary streams for the existing XLSX/RTF importer."""
import stat
import zipfile
import zlib
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from tempfile import SpooledTemporaryFile
from typing import BinaryIO
from apps.parsers.integrated_structure_workbook import (
MAX_WORKBOOK_BYTES,
IntegratedWorkbookError,
)
from apps.parsers.integrated_structures import (
MAX_ATTACHMENT_BYTES,
MAX_ATTACHMENTS,
MAX_TOTAL_ATTACHMENT_BYTES,
)
from django.core.files import File
MAX_ARCHIVE_BYTES = 125 * 1024 * 1024
MAX_ARCHIVE_ENTRIES = 1024
MAX_EXPANDED_BYTES = MAX_WORKBOOK_BYTES + MAX_TOTAL_ATTACHMENT_BYTES
COPY_CHUNK_BYTES = 1024 * 1024
@dataclass(frozen=True)
class IntegratedStructureArchive:
workbook: File
attachments: list[File]
def _member_name(member: zipfile.ZipInfo) -> str:
path = member.orig_filename
parts = path.rstrip("/").split("/")
file_type = stat.S_IFMT(member.external_attr >> 16)
if (
not path
or any(part in ("", ".", "..") for part in parts)
or any(char in path for char in ("\\", ":", "\x00"))
or file_type not in (0, stat.S_IFREG, stat.S_IFDIR)
):
raise IntegratedWorkbookError("unsafe_archive_member")
return parts[-1]
def _archive_members(
archive: zipfile.ZipFile,
) -> tuple[zipfile.ZipInfo, list[zipfile.ZipInfo]]:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_ENTRIES:
raise IntegratedWorkbookError("too_many_archive_entries")
if sum(member.file_size for member in members) > MAX_EXPANDED_BYTES:
raise IntegratedWorkbookError("archive_expanded_size_exceeded")
workbooks = []
attachments = []
filenames = set()
for member in members:
name = _validated_member_name(member)
if name is None:
continue
if name.casefold() in filenames:
raise IntegratedWorkbookError("duplicate_archive_filename")
filenames.add(name.casefold())
if name.lower().endswith(".xlsx"):
workbooks.append(member)
else:
attachments.append(member)
if len(workbooks) != 1:
raise IntegratedWorkbookError("archive_workbook_count")
if len(attachments) > MAX_ATTACHMENTS:
raise IntegratedWorkbookError("too_many_attachments")
if sum(member.file_size for member in attachments) > MAX_TOTAL_ATTACHMENT_BYTES:
raise IntegratedWorkbookError("attachments_too_large")
return workbooks[0], attachments
def _validated_member_name(member: zipfile.ZipInfo) -> str | None:
name = _member_name(member)
if member.flag_bits & 1:
raise IntegratedWorkbookError("encrypted_archive_member")
if member.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
raise IntegratedWorkbookError("unsupported_archive_compression")
if member.is_dir():
if member.file_size:
raise IntegratedWorkbookError("unsafe_archive_member")
return None
if name == ".DS_Store" or member.filename.startswith("__MACOSX/"):
return None
if name.lower().endswith(".xlsx"):
limit, error_code = MAX_WORKBOOK_BYTES, "workbook_too_large"
elif name.lower().endswith(".rtf"):
limit, error_code = MAX_ATTACHMENT_BYTES, "attachment_too_large"
else:
raise IntegratedWorkbookError("unsupported_archive_member")
if member.file_size > limit:
raise IntegratedWorkbookError(error_code)
return name
def _read_member(
archive: zipfile.ZipFile, member: zipfile.ZipInfo, stack: ExitStack
) -> File:
# Archive paths never become filesystem paths. Large files spill to private
# temporary files; all streams are closed on validation/import failure too.
target = stack.enter_context(SpooledTemporaryFile(max_size=COPY_CHUNK_BYTES))
remaining = member.file_size
with archive.open(member) as source:
while chunk := source.read(min(COPY_CHUNK_BYTES, remaining + 1)):
if len(chunk) > remaining:
raise IntegratedWorkbookError("invalid_archive")
target.write(chunk)
remaining -= len(chunk)
if remaining:
raise IntegratedWorkbookError("invalid_archive")
target.seek(0)
return File(target, name=member.filename.rsplit("/", 1)[-1])
@contextmanager
def unpack_integrated_structure_archive(
handle: BinaryIO,
) -> Iterator[IntegratedStructureArchive]:
"""Validate the entire ZIP before import, keeping temporary streams alive."""
with ExitStack() as stack:
try:
handle.seek(0, 2)
if handle.tell() > MAX_ARCHIVE_BYTES:
raise IntegratedWorkbookError("archive_too_large")
handle.seek(0)
archive = stack.enter_context(zipfile.ZipFile(handle))
workbook_member, attachment_members = _archive_members(archive)
workbook = _read_member(archive, workbook_member, stack)
attachments = [
_read_member(archive, member, stack) for member in attachment_members
]
except (
zipfile.BadZipFile,
zipfile.LargeZipFile,
EOFError,
UnicodeDecodeError,
zlib.error,
) as exc:
raise IntegratedWorkbookError("invalid_archive") from exc
yield IntegratedStructureArchive(workbook, attachments)

View File

@@ -411,20 +411,17 @@ ROPK_SANCTIONS_UPLOAD_FILE_PARAM = openapi.Parameter(
type=openapi.TYPE_FILE,
required=True,
)
INTEGRATED_WORKBOOK_PARAM = openapi.Parameter(
"workbook",
INTEGRATED_ARCHIVE_PARAM = openapi.Parameter(
"archive",
openapi.IN_FORM,
description="Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ",
description=(
"ZIP до 125 МиБ: один XLSX с листами is_stages и is_stages_docs "
"(до 20 МиБ) и до 500 RTF (до 5 МиБ каждый, суммарно до 100 МиБ). "
"Допускаются подпапки; имя RTF должно соответствовать doc_id."
),
type=openapi.TYPE_FILE,
required=True,
)
INTEGRATED_ATTACHMENT_PARAM = openapi.Parameter(
"attachments",
openapi.IN_FORM,
description="Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256",
type=openapi.TYPE_FILE,
required=False,
)
RESULT_LIST_PARAMS = [
PAGE_PARAM,
PAGE_SIZE_PARAM,
@@ -3136,26 +3133,41 @@ class ParserUploadView(APIView):
if descriptor.admin_only and not request.user.is_staff:
return Response(status=status.HTTP_403_FORBIDDEN)
if source_key == ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS:
from apps.parsers.integrated_structure_archive import (
MAX_ARCHIVE_BYTES,
unpack_integrated_structure_archive,
)
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
)
from apps.parsers.integrated_structures import import_integrated_structures
from django.core.exceptions import TooManyFilesSent
workbook = request.FILES.get("workbook")
if workbook is None or not workbook.name.lower().endswith(".xlsx"):
try:
uploaded_files = request.FILES
archive = uploaded_files.get("archive")
file_count = sum(len(files) for _, files in uploaded_files.lists())
except TooManyFilesSent:
archive = None
file_count = 0
if (
archive is None
or not archive.name.lower().endswith(".zip")
or file_count != 1
):
return api_error_response(
[
{
"code": "workbook_required",
"field": "workbook",
"message": "Требуется XLSX workbook",
"code": "archive_required",
"field": "archive",
"message": "Требуется один ZIP-архив с XLSX и RTF",
}
],
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
)
if workbook.size > 20 * 1024 * 1024:
if archive.size > MAX_ARCHIVE_BYTES:
return api_error_response(
[{"code": "workbook_too_large", "message": "XLSX слишком велик"}],
[{"code": "archive_too_large", "message": "ZIP превышает 125 МиБ"}],
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
)
job = BackgroundJobService.create_job(
@@ -3166,12 +3178,13 @@ class ParserUploadView(APIView):
)
job.mark_started()
try:
artifact, result = import_integrated_structures(
workbook=workbook,
original_name=workbook.name,
attachments=request.FILES.getlist("attachments"),
uploaded_by_id=request.user.id,
)
with unpack_integrated_structure_archive(archive) as files:
artifact, result = import_integrated_structures(
workbook=files.workbook,
original_name=files.workbook.name,
attachments=files.attachments,
uploaded_by_id=request.user.id,
)
except IntegratedWorkbookError as exc:
job.fail(error=str(exc))
return api_error_response(

View File

@@ -5289,26 +5289,21 @@
"post": {
"operationId": "api_v1_parsers_upload_integrated_structure_documents_create",
"summary": "Документы интегрированных структур: загрузить файл",
"description": "Ручная загрузка файла для источника: События и документы интегрированных структур. Файл обрабатывается через Celery.",
"description": "Загрузка одного ZIP с XLSX и RTF. Архив проверяется и обрабатывается синхронно существующим импортёром.",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"properties": {
"workbook": {
"description": "Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ",
"type": "string",
"format": "binary"
},
"attachments": {
"description": "Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256",
"archive": {
"description": "ZIP до 125 МиБ: один XLSX с листами is_stages и is_stages_docs (до 20 МиБ) и до 500 RTF (до 5 МиБ каждый, суммарно до 100 МиБ). Допускаются подпапки; имя RTF должно соответствовать doc_id.",
"type": "string",
"format": "binary"
}
},
"required": [
"workbook"
"archive"
]
}
}
@@ -5339,7 +5334,7 @@
"description": "Неподдерживаемый тип файла"
},
"422": {
"description": "Неверные листы, идентификаторы или вложения"
"description": "Некорректный архив, листы, идентификаторы или вложения"
}
},
"tags": [

View File

@@ -0,0 +1,322 @@
"""ZIP upload contract, atomic import and rejection of unsafe archive members."""
import stat
import struct
from io import BytesIO
from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo
import pytest
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
ParserSourceArtifact,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.utils import timezone
from openpyxl import load_workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
from tests.apps.user.factories import UserFactory
pytestmark = pytest.mark.django_db
UPLOAD_URL = "/api/v1/parsers/upload/integrated_structure_documents/"
RTF = b"{\\rtf1\\ansi document}"
def zip_bytes(entries, *, compression=ZIP_DEFLATED):
stream = BytesIO()
with ZipFile(stream, "w", compression=compression) as archive:
for name, data in entries:
archive.writestr(name, data)
return stream.getvalue()
@pytest.fixture
def archive_client(settings, tmp_path):
settings.PARSER_PRIVATE_ARTIFACT_ROOT = tmp_path / "private"
settings.MEDIA_ROOT = tmp_path / "public"
Organization.objects.create(
name="АО Тест",
inn="1234567890",
ogrn="1027700132195",
okpo="00084008",
directory_imported_at=timezone.now(),
)
client = APIClient()
client.force_authenticate(UserFactory.create_superuser())
return client
def upload_archive(client, data, *, name="structures.zip", extra=None):
return client.post(
UPLOAD_URL,
{"archive": SimpleUploadedFile(name, data), **(extra or {})},
format="multipart",
)
def test_zip_with_100_rtf_bypasses_multipart_limit_and_publishes_documents(
archive_client,
):
workbook = load_workbook(BytesIO(workbook_bytes()))
documents = workbook["is_stages_docs"]
row = list(next(documents.iter_rows(min_row=2, values_only=True)))
for document_id in range(94, 193):
row[3] = document_id
documents.append(row)
stream = BytesIO()
workbook.save(stream)
workbook.close()
archive = zip_bytes(
[("structures/Документы ИС.xlsx", stream.getvalue())]
+ [(f"structures/documents/{doc_id:06d}.rtf", RTF) for doc_id in range(93, 193)]
)
response = upload_archive(archive_client, archive)
assert response.status_code == 200
assert response.data["data"]["attachments_count"] == 100
assert response.data["data"]["missing_attachments_count"] == 0
assert IntegratedStructureDocument.objects.count() == 100
document = IntegratedStructureDocument.objects.get(external_id="93")
assert document.attachment_filename == "000093.rtf"
with document.attachment.file.open("rb") as stored:
assert stored.read() == RTF
assert BackgroundJob.objects.get().status == JobStatus.SUCCESS
repeated = upload_archive(archive_client, archive)
assert repeated.status_code == 200
assert repeated.data["data"]["reason"] == "identical_batch"
assert IntegratedStructureDocument.objects.count() == 100
def test_zip_without_rtf_preserves_missing_attachment_reporting(archive_client):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 200
assert response.data["data"]["missing_attachments_count"] == 1
assert IntegratedStructureDocument.objects.get().attachment_id is None
def test_macos_metadata_and_directories_do_not_become_attachments(archive_client):
response = upload_archive(
archive_client,
zip_bytes(
[
("structures/", b""),
("structures/test.xlsx", workbook_bytes()),
("structures/000093.rtf", RTF),
("structures/.DS_Store", b"metadata"),
("__MACOSX/structures/._000093.rtf", b"metadata"),
]
),
)
assert response.status_code == 200
assert response.data["data"]["attachments_count"] == 1
@pytest.mark.parametrize(
"member",
[
"../000093.rtf",
"/000093.rtf",
"files/../000093.rtf",
"C:/000093.rtf",
"files\\000093.rtf",
],
)
def test_unsafe_paths_reject_whole_archive_before_publication(archive_client, member):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), (member, RTF)])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
assert not OrganizationSourceRecord.objects.exists()
assert not ParserSourceArtifact.objects.exists()
assert not IntegratedStructureAttachment.objects.exists()
@pytest.mark.parametrize(
("extra_entries", "code"),
[
([("second.xlsx", b"xlsx")], "archive_workbook_count"),
([("unexpected.pdf", b"pdf")], "unsupported_archive_member"),
([("000093.rtf", b"not RTF")], "invalid_rtf_signature"),
([("999999.rtf", RTF)], "unknown_or_ambiguous_attachment_document"),
([("a/000093.rtf", RTF), ("b/000093.rtf", RTF)], "duplicate_archive_filename"),
([("93.rtf", RTF), ("000093.rtf", RTF)], "duplicate_attachment"),
],
)
def test_invalid_zip_or_document_set_has_no_partial_writes(
archive_client, extra_entries, code
):
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes()), *extra_entries])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == code
assert not OrganizationSourceRecord.objects.exists()
assert not ParserSourceArtifact.objects.exists()
assert BackgroundJob.objects.get().status == JobStatus.FAILURE
def test_symbolic_link_is_rejected(archive_client):
link = ZipInfo("000093.rtf")
link.create_system = 3
link.external_attr = (stat.S_IFLNK | 0o777) << 16
response = upload_archive(
archive_client,
zip_bytes([("test.xlsx", workbook_bytes()), (link, b"/etc/passwd")]),
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "unsafe_archive_member"
@pytest.mark.parametrize("data", [b"not a zip", zip_bytes([])])
def test_invalid_or_empty_zip_returns_validation_error(archive_client, data):
response = upload_archive(archive_client, data)
assert response.status_code == 422
assert not OrganizationSourceRecord.objects.exists()
def test_crc_failure_returns_validation_error_without_publication(archive_client):
raw = bytearray(
zip_bytes(
[("test.xlsx", workbook_bytes()), ("000093.rtf", RTF)],
compression=ZIP_STORED,
)
)
with ZipFile(BytesIO(raw)) as archive:
member = archive.getinfo("000093.rtf")
name_size, extra_size = struct.unpack_from("<HH", raw, member.header_offset + 26)
data_offset = member.header_offset + 30 + name_size + extra_size
raw[data_offset] ^= 1
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "invalid_archive"
assert not ParserSourceArtifact.objects.exists()
def test_old_loose_file_contract_is_rejected(archive_client):
response = archive_client.post(
UPLOAD_URL,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
format="multipart",
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
def test_extra_uploaded_files_are_rejected(archive_client):
response = upload_archive(
archive_client,
zip_bytes([("test.xlsx", workbook_bytes())]),
extra={"attachments": SimpleUploadedFile("000093.rtf", RTF)},
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
def test_zip_upload_preserves_admin_permission(archive_client):
archive_client.force_authenticate(UserFactory.create_user())
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 403
assert not BackgroundJob.objects.exists()
def test_many_loose_files_return_validation_error_instead_of_500(archive_client):
archive_client.raise_request_exception = False
response = archive_client.post(
UPLOAD_URL,
{
"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes()),
"attachments": [SimpleUploadedFile(f"{i}.rtf", RTF) for i in range(100)],
},
format="multipart",
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_required"
assert not BackgroundJob.objects.exists()
def test_compressed_zip_is_bounded_by_expanded_size(archive_client, monkeypatch):
monkeypatch.setattr(
"apps.parsers.integrated_structure_archive.MAX_EXPANDED_BYTES", 100
)
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "archive_expanded_size_exceeded"
assert not ParserSourceArtifact.objects.exists()
@pytest.mark.parametrize(
("member", "size", "code"),
[
("test.xlsx", 20 * 1024 * 1024 + 1, "workbook_too_large"),
("000093.rtf", 5 * 1024 * 1024 + 1, "attachment_too_large"),
],
)
def test_member_size_is_checked_before_expansion(archive_client, member, size, code):
entries = [(member, b"x" * size)]
if member.endswith(".rtf"):
entries.append(("test.xlsx", workbook_bytes()))
response = upload_archive(archive_client, zip_bytes(entries))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == code
assert not OrganizationSourceRecord.objects.exists()
def test_rtf_count_limit_is_preserved_inside_zip(archive_client):
archive = zip_bytes(
[("test.xlsx", workbook_bytes())] + [(f"{i}.rtf", RTF) for i in range(501)]
)
response = upload_archive(archive_client, archive)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "too_many_attachments"
def test_encrypted_zip_returns_validation_error(archive_client):
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
# Mark the member encrypted in both ZIP headers; no password is accepted.
struct.pack_into("<H", raw, 6, struct.unpack_from("<H", raw, 6)[0] | 1)
central = raw.index(b"PK\x01\x02")
struct.pack_into(
"<H", raw, central + 8, struct.unpack_from("<H", raw, central + 8)[0] | 1
)
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "encrypted_archive_member"
def test_invalid_utf8_zip_filename_returns_validation_error(archive_client):
raw = bytearray(zip_bytes([("test.xlsx", workbook_bytes())]))
central = raw.index(b"PK\x01\x02")
struct.pack_into("<H", raw, central + 8, 0x800)
raw[central + 46] = 0xFF
response = upload_archive(archive_client, bytes(raw))
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "invalid_archive"
def test_zip_upload_size_limit_returns_413(archive_client, monkeypatch):
monkeypatch.setattr(
"apps.parsers.integrated_structure_archive.MAX_ARCHIVE_BYTES", 100
)
response = upload_archive(
archive_client, zip_bytes([("test.xlsx", workbook_bytes())])
)
assert response.status_code == 413
assert response.data["errors"][0]["code"] == "archive_too_large"
assert not BackgroundJob.objects.exists()
def test_many_zip_directory_entries_are_bounded(archive_client):
raw = zip_bytes([(f"folder-{i}/", b"") for i in range(1025)])
response = upload_archive(archive_client, raw)
assert response.status_code == 422
assert response.data["errors"][0]["code"] == "too_many_archive_entries"

View File

@@ -9,7 +9,10 @@ from openpyxl import load_workbook
from organizations.models import Organization, OrganizationSourceRecord
from rest_framework.test import APIClient
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
from tests.apps.parsers.test_integrated_structure_workbook import (
archive_bytes,
workbook_bytes,
)
from tests.apps.user.factories import UserFactory
@@ -53,8 +56,7 @@ class IntegratedStructurePartialImportTest(TestCase):
return client.post(
"/api/v1/parsers/upload/integrated_structure_documents/",
{
"workbook": SimpleUploadedFile("test.xlsx", raw),
"attachments": list(attachments),
"archive": SimpleUploadedFile("test.zip", archive_bytes(raw, attachments)),
},
format="multipart",
)

View File

@@ -1,5 +1,6 @@
from io import BytesIO
from unittest.mock import patch
from zipfile import ZIP_DEFLATED, ZipFile
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
@@ -42,6 +43,16 @@ def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137):
return stream.getvalue()
def archive_bytes(raw, attachments=()):
stream = BytesIO()
with ZipFile(stream, "w", compression=ZIP_DEFLATED) as archive:
archive.writestr("test.xlsx", raw)
for attachment in attachments:
attachment.seek(0)
archive.writestr(attachment.name, attachment.read())
return stream.getvalue()
class IntegratedWorkbookTest(SimpleTestCase):
def test_parses_relations_and_preserves_zero_padded_okpo(self):
result = parse_integrated_workbook(BytesIO(workbook_bytes()))
@@ -244,14 +255,14 @@ class IntegratedStructureImportTest(TestCase):
client.force_authenticate(user)
response = client.post(
url,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 403)
client.force_authenticate(admin)
response = client.post(
url,
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 200)
@@ -275,7 +286,7 @@ class IntegratedStructureImportTest(TestCase):
):
response = client.post(
"/api/v1/parsers/upload/integrated_structure_documents/",
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
{"archive": SimpleUploadedFile("test.zip", archive_bytes(workbook_bytes()))},
format="multipart",
)
self.assertEqual(response.status_code, 500)