feat: import integrated structures from a ZIP archive
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-10-08 10:34:42 +02:00
parent 70dfc55f3d
commit 4e548fcc8e
8 changed files with 581 additions and 47 deletions

View File

@@ -7,8 +7,7 @@ from apps.parsers.serializers import (
)
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
from apps.parsers.views import (
INTEGRATED_ATTACHMENT_PARAM,
INTEGRATED_WORKBOOK_PARAM,
INTEGRATED_ARCHIVE_PARAM,
MEDIA_NEWS_UPLOAD_FILE_PARAM,
RESULT_DETAIL_PARAMS,
RESULT_LIST_PARAMS,
@@ -125,11 +124,14 @@ def _upload_view(descriptor: ParserSourceDescriptor):
@swagger_auto_schema(
operation_summary=f"{descriptor.title}: загрузить файл",
operation_description=(
f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
"Загрузка одного ZIP с XLSX и RTF. Архив проверяется и "
"обрабатывается синхронно существующим импортёром."
if descriptor.key == "integrated_structure_documents"
else f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
"Файл обрабатывается через Celery."
),
manual_parameters=(
[INTEGRATED_WORKBOOK_PARAM, INTEGRATED_ATTACHMENT_PARAM]
[INTEGRATED_ARCHIVE_PARAM]
if descriptor.key == "integrated_structure_documents"
else [
MEDIA_NEWS_UPLOAD_FILE_PARAM
@@ -151,7 +153,7 @@ def _upload_view(descriptor: ParserSourceDescriptor):
409: "Загрузка уже выполняется",
413: "Файл превышает лимит",
415: "Неподдерживаемый тип файла",
422: "Неверные листы, идентификаторы или вложения",
422: "Некорректный архив, листы, идентификаторы или вложения",
},
)
def post(self, request):

View File

@@ -0,0 +1,147 @@
"""Bounded ZIP expansion into temporary streams for the existing XLSX/RTF importer."""
import stat
import zipfile
import zlib
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass
from tempfile import SpooledTemporaryFile
from typing import BinaryIO
from apps.parsers.integrated_structure_workbook import (
MAX_WORKBOOK_BYTES,
IntegratedWorkbookError,
)
from apps.parsers.integrated_structures import (
MAX_ATTACHMENT_BYTES,
MAX_ATTACHMENTS,
MAX_TOTAL_ATTACHMENT_BYTES,
)
from django.core.files import File
MAX_ARCHIVE_BYTES = 125 * 1024 * 1024
MAX_ARCHIVE_ENTRIES = 1024
MAX_EXPANDED_BYTES = MAX_WORKBOOK_BYTES + MAX_TOTAL_ATTACHMENT_BYTES
COPY_CHUNK_BYTES = 1024 * 1024
@dataclass(frozen=True)
class IntegratedStructureArchive:
workbook: File
attachments: list[File]
def _member_name(member: zipfile.ZipInfo) -> str:
path = member.orig_filename
parts = path.rstrip("/").split("/")
file_type = stat.S_IFMT(member.external_attr >> 16)
if (
not path
or any(part in ("", ".", "..") for part in parts)
or any(char in path for char in ("\\", ":", "\x00"))
or file_type not in (0, stat.S_IFREG, stat.S_IFDIR)
):
raise IntegratedWorkbookError("unsafe_archive_member")
return parts[-1]
def _archive_members(
archive: zipfile.ZipFile,
) -> tuple[zipfile.ZipInfo, list[zipfile.ZipInfo]]:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_ENTRIES:
raise IntegratedWorkbookError("too_many_archive_entries")
if sum(member.file_size for member in members) > MAX_EXPANDED_BYTES:
raise IntegratedWorkbookError("archive_expanded_size_exceeded")
workbooks = []
attachments = []
filenames = set()
for member in members:
name = _validated_member_name(member)
if name is None:
continue
if name.casefold() in filenames:
raise IntegratedWorkbookError("duplicate_archive_filename")
filenames.add(name.casefold())
if name.lower().endswith(".xlsx"):
workbooks.append(member)
else:
attachments.append(member)
if len(workbooks) != 1:
raise IntegratedWorkbookError("archive_workbook_count")
if len(attachments) > MAX_ATTACHMENTS:
raise IntegratedWorkbookError("too_many_attachments")
if sum(member.file_size for member in attachments) > MAX_TOTAL_ATTACHMENT_BYTES:
raise IntegratedWorkbookError("attachments_too_large")
return workbooks[0], attachments
def _validated_member_name(member: zipfile.ZipInfo) -> str | None:
name = _member_name(member)
if member.flag_bits & 1:
raise IntegratedWorkbookError("encrypted_archive_member")
if member.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
raise IntegratedWorkbookError("unsupported_archive_compression")
if member.is_dir():
if member.file_size:
raise IntegratedWorkbookError("unsafe_archive_member")
return None
if name == ".DS_Store" or member.filename.startswith("__MACOSX/"):
return None
if name.lower().endswith(".xlsx"):
limit, error_code = MAX_WORKBOOK_BYTES, "workbook_too_large"
elif name.lower().endswith(".rtf"):
limit, error_code = MAX_ATTACHMENT_BYTES, "attachment_too_large"
else:
raise IntegratedWorkbookError("unsupported_archive_member")
if member.file_size > limit:
raise IntegratedWorkbookError(error_code)
return name
def _read_member(
archive: zipfile.ZipFile, member: zipfile.ZipInfo, stack: ExitStack
) -> File:
# Archive paths never become filesystem paths. Large files spill to private
# temporary files; all streams are closed on validation/import failure too.
target = stack.enter_context(SpooledTemporaryFile(max_size=COPY_CHUNK_BYTES))
remaining = member.file_size
with archive.open(member) as source:
while chunk := source.read(min(COPY_CHUNK_BYTES, remaining + 1)):
if len(chunk) > remaining:
raise IntegratedWorkbookError("invalid_archive")
target.write(chunk)
remaining -= len(chunk)
if remaining:
raise IntegratedWorkbookError("invalid_archive")
target.seek(0)
return File(target, name=member.filename.rsplit("/", 1)[-1])
@contextmanager
def unpack_integrated_structure_archive(
handle: BinaryIO,
) -> Iterator[IntegratedStructureArchive]:
"""Validate the entire ZIP before import, keeping temporary streams alive."""
with ExitStack() as stack:
try:
handle.seek(0, 2)
if handle.tell() > MAX_ARCHIVE_BYTES:
raise IntegratedWorkbookError("archive_too_large")
handle.seek(0)
archive = stack.enter_context(zipfile.ZipFile(handle))
workbook_member, attachment_members = _archive_members(archive)
workbook = _read_member(archive, workbook_member, stack)
attachments = [
_read_member(archive, member, stack) for member in attachment_members
]
except (
zipfile.BadZipFile,
zipfile.LargeZipFile,
EOFError,
UnicodeDecodeError,
zlib.error,
) as exc:
raise IntegratedWorkbookError("invalid_archive") from exc
yield IntegratedStructureArchive(workbook, attachments)

View File

@@ -411,20 +411,17 @@ ROPK_SANCTIONS_UPLOAD_FILE_PARAM = openapi.Parameter(
type=openapi.TYPE_FILE,
required=True,
)
INTEGRATED_WORKBOOK_PARAM = openapi.Parameter(
"workbook",
INTEGRATED_ARCHIVE_PARAM = openapi.Parameter(
"archive",
openapi.IN_FORM,
description="Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ",
description=(
"ZIP до 125 МиБ: один XLSX с листами is_stages и is_stages_docs "
"(до 20 МиБ) и до 500 RTF (до 5 МиБ каждый, суммарно до 100 МиБ). "
"Допускаются подпапки; имя RTF должно соответствовать doc_id."
),
type=openapi.TYPE_FILE,
required=True,
)
INTEGRATED_ATTACHMENT_PARAM = openapi.Parameter(
"attachments",
openapi.IN_FORM,
description="Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256",
type=openapi.TYPE_FILE,
required=False,
)
RESULT_LIST_PARAMS = [
PAGE_PARAM,
PAGE_SIZE_PARAM,
@@ -3136,26 +3133,41 @@ class ParserUploadView(APIView):
if descriptor.admin_only and not request.user.is_staff:
return Response(status=status.HTTP_403_FORBIDDEN)
if source_key == ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS:
from apps.parsers.integrated_structure_archive import (
MAX_ARCHIVE_BYTES,
unpack_integrated_structure_archive,
)
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
)
from apps.parsers.integrated_structures import import_integrated_structures
from django.core.exceptions import TooManyFilesSent
workbook = request.FILES.get("workbook")
if workbook is None or not workbook.name.lower().endswith(".xlsx"):
try:
uploaded_files = request.FILES
archive = uploaded_files.get("archive")
file_count = sum(len(files) for _, files in uploaded_files.lists())
except TooManyFilesSent:
archive = None
file_count = 0
if (
archive is None
or not archive.name.lower().endswith(".zip")
or file_count != 1
):
return api_error_response(
[
{
"code": "workbook_required",
"field": "workbook",
"message": "Требуется XLSX workbook",
"code": "archive_required",
"field": "archive",
"message": "Требуется один ZIP-архив с XLSX и RTF",
}
],
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
)
if workbook.size > 20 * 1024 * 1024:
if archive.size > MAX_ARCHIVE_BYTES:
return api_error_response(
[{"code": "workbook_too_large", "message": "XLSX слишком велик"}],
[{"code": "archive_too_large", "message": "ZIP превышает 125 МиБ"}],
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
)
job = BackgroundJobService.create_job(
@@ -3166,12 +3178,13 @@ class ParserUploadView(APIView):
)
job.mark_started()
try:
artifact, result = import_integrated_structures(
workbook=workbook,
original_name=workbook.name,
attachments=request.FILES.getlist("attachments"),
uploaded_by_id=request.user.id,
)
with unpack_integrated_structure_archive(archive) as files:
artifact, result = import_integrated_structures(
workbook=files.workbook,
original_name=files.workbook.name,
attachments=files.attachments,
uploaded_by_id=request.user.id,
)
except IntegratedWorkbookError as exc:
job.fail(error=str(exc))
return api_error_response(

View File

@@ -5289,26 +5289,21 @@
"post": {
"operationId": "api_v1_parsers_upload_integrated_structure_documents_create",
"summary": "Документы интегрированных структур: загрузить файл",
"description": "Ручная загрузка файла для источника: События и документы интегрированных структур. Файл обрабатывается через Celery.",
"description": "Загрузка одного ZIP с XLSX и RTF. Архив проверяется и обрабатывается синхронно существующим импортёром.",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"type": "object",
"properties": {
"workbook": {
"description": "Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ",
"type": "string",
"format": "binary"
},
"attachments": {
"description": "Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256",
"archive": {
"description": "ZIP до 125 МиБ: один XLSX с листами is_stages и is_stages_docs (до 20 МиБ) и до 500 RTF (до 5 МиБ каждый, суммарно до 100 МиБ). Допускаются подпапки; имя RTF должно соответствовать doc_id.",
"type": "string",
"format": "binary"
}
},
"required": [
"workbook"
"archive"
]
}
}
@@ -5339,7 +5334,7 @@
"description": "Неподдерживаемый тип файла"
},
"422": {
"description": "Неверные листы, идентификаторы или вложения"
"description": "Некорректный архив, листы, идентификаторы или вложения"
}
},
"tags": [