feat: import integrated structures from a ZIP archive
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3m15s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
@@ -7,8 +7,7 @@ from apps.parsers.serializers import (
|
||||
)
|
||||
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
|
||||
from apps.parsers.views import (
|
||||
INTEGRATED_ATTACHMENT_PARAM,
|
||||
INTEGRATED_WORKBOOK_PARAM,
|
||||
INTEGRATED_ARCHIVE_PARAM,
|
||||
MEDIA_NEWS_UPLOAD_FILE_PARAM,
|
||||
RESULT_DETAIL_PARAMS,
|
||||
RESULT_LIST_PARAMS,
|
||||
@@ -125,11 +124,14 @@ def _upload_view(descriptor: ParserSourceDescriptor):
|
||||
@swagger_auto_schema(
|
||||
operation_summary=f"{descriptor.title}: загрузить файл",
|
||||
operation_description=(
|
||||
f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
|
||||
"Загрузка одного ZIP с XLSX и RTF. Архив проверяется и "
|
||||
"обрабатывается синхронно существующим импортёром."
|
||||
if descriptor.key == "integrated_structure_documents"
|
||||
else f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
|
||||
"Файл обрабатывается через Celery."
|
||||
),
|
||||
manual_parameters=(
|
||||
[INTEGRATED_WORKBOOK_PARAM, INTEGRATED_ATTACHMENT_PARAM]
|
||||
[INTEGRATED_ARCHIVE_PARAM]
|
||||
if descriptor.key == "integrated_structure_documents"
|
||||
else [
|
||||
MEDIA_NEWS_UPLOAD_FILE_PARAM
|
||||
@@ -151,7 +153,7 @@ def _upload_view(descriptor: ParserSourceDescriptor):
|
||||
409: "Загрузка уже выполняется",
|
||||
413: "Файл превышает лимит",
|
||||
415: "Неподдерживаемый тип файла",
|
||||
422: "Неверные листы, идентификаторы или вложения",
|
||||
422: "Некорректный архив, листы, идентификаторы или вложения",
|
||||
},
|
||||
)
|
||||
def post(self, request):
|
||||
|
||||
147
src/apps/parsers/integrated_structure_archive.py
Normal file
147
src/apps/parsers/integrated_structure_archive.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""Bounded ZIP expansion into temporary streams for the existing XLSX/RTF importer."""
|
||||
|
||||
import stat
|
||||
import zipfile
|
||||
import zlib
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from typing import BinaryIO
|
||||
|
||||
from apps.parsers.integrated_structure_workbook import (
|
||||
MAX_WORKBOOK_BYTES,
|
||||
IntegratedWorkbookError,
|
||||
)
|
||||
from apps.parsers.integrated_structures import (
|
||||
MAX_ATTACHMENT_BYTES,
|
||||
MAX_ATTACHMENTS,
|
||||
MAX_TOTAL_ATTACHMENT_BYTES,
|
||||
)
|
||||
from django.core.files import File
|
||||
|
||||
MAX_ARCHIVE_BYTES = 125 * 1024 * 1024
|
||||
MAX_ARCHIVE_ENTRIES = 1024
|
||||
MAX_EXPANDED_BYTES = MAX_WORKBOOK_BYTES + MAX_TOTAL_ATTACHMENT_BYTES
|
||||
COPY_CHUNK_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IntegratedStructureArchive:
|
||||
workbook: File
|
||||
attachments: list[File]
|
||||
|
||||
|
||||
def _member_name(member: zipfile.ZipInfo) -> str:
|
||||
path = member.orig_filename
|
||||
parts = path.rstrip("/").split("/")
|
||||
file_type = stat.S_IFMT(member.external_attr >> 16)
|
||||
if (
|
||||
not path
|
||||
or any(part in ("", ".", "..") for part in parts)
|
||||
or any(char in path for char in ("\\", ":", "\x00"))
|
||||
or file_type not in (0, stat.S_IFREG, stat.S_IFDIR)
|
||||
):
|
||||
raise IntegratedWorkbookError("unsafe_archive_member")
|
||||
return parts[-1]
|
||||
|
||||
|
||||
def _archive_members(
|
||||
archive: zipfile.ZipFile,
|
||||
) -> tuple[zipfile.ZipInfo, list[zipfile.ZipInfo]]:
|
||||
members = archive.infolist()
|
||||
if len(members) > MAX_ARCHIVE_ENTRIES:
|
||||
raise IntegratedWorkbookError("too_many_archive_entries")
|
||||
if sum(member.file_size for member in members) > MAX_EXPANDED_BYTES:
|
||||
raise IntegratedWorkbookError("archive_expanded_size_exceeded")
|
||||
workbooks = []
|
||||
attachments = []
|
||||
filenames = set()
|
||||
for member in members:
|
||||
name = _validated_member_name(member)
|
||||
if name is None:
|
||||
continue
|
||||
if name.casefold() in filenames:
|
||||
raise IntegratedWorkbookError("duplicate_archive_filename")
|
||||
filenames.add(name.casefold())
|
||||
if name.lower().endswith(".xlsx"):
|
||||
workbooks.append(member)
|
||||
else:
|
||||
attachments.append(member)
|
||||
if len(workbooks) != 1:
|
||||
raise IntegratedWorkbookError("archive_workbook_count")
|
||||
if len(attachments) > MAX_ATTACHMENTS:
|
||||
raise IntegratedWorkbookError("too_many_attachments")
|
||||
if sum(member.file_size for member in attachments) > MAX_TOTAL_ATTACHMENT_BYTES:
|
||||
raise IntegratedWorkbookError("attachments_too_large")
|
||||
return workbooks[0], attachments
|
||||
|
||||
|
||||
def _validated_member_name(member: zipfile.ZipInfo) -> str | None:
|
||||
name = _member_name(member)
|
||||
if member.flag_bits & 1:
|
||||
raise IntegratedWorkbookError("encrypted_archive_member")
|
||||
if member.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
||||
raise IntegratedWorkbookError("unsupported_archive_compression")
|
||||
if member.is_dir():
|
||||
if member.file_size:
|
||||
raise IntegratedWorkbookError("unsafe_archive_member")
|
||||
return None
|
||||
if name == ".DS_Store" or member.filename.startswith("__MACOSX/"):
|
||||
return None
|
||||
if name.lower().endswith(".xlsx"):
|
||||
limit, error_code = MAX_WORKBOOK_BYTES, "workbook_too_large"
|
||||
elif name.lower().endswith(".rtf"):
|
||||
limit, error_code = MAX_ATTACHMENT_BYTES, "attachment_too_large"
|
||||
else:
|
||||
raise IntegratedWorkbookError("unsupported_archive_member")
|
||||
if member.file_size > limit:
|
||||
raise IntegratedWorkbookError(error_code)
|
||||
return name
|
||||
|
||||
|
||||
def _read_member(
|
||||
archive: zipfile.ZipFile, member: zipfile.ZipInfo, stack: ExitStack
|
||||
) -> File:
|
||||
# Archive paths never become filesystem paths. Large files spill to private
|
||||
# temporary files; all streams are closed on validation/import failure too.
|
||||
target = stack.enter_context(SpooledTemporaryFile(max_size=COPY_CHUNK_BYTES))
|
||||
remaining = member.file_size
|
||||
with archive.open(member) as source:
|
||||
while chunk := source.read(min(COPY_CHUNK_BYTES, remaining + 1)):
|
||||
if len(chunk) > remaining:
|
||||
raise IntegratedWorkbookError("invalid_archive")
|
||||
target.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if remaining:
|
||||
raise IntegratedWorkbookError("invalid_archive")
|
||||
target.seek(0)
|
||||
return File(target, name=member.filename.rsplit("/", 1)[-1])
|
||||
|
||||
|
||||
@contextmanager
|
||||
def unpack_integrated_structure_archive(
|
||||
handle: BinaryIO,
|
||||
) -> Iterator[IntegratedStructureArchive]:
|
||||
"""Validate the entire ZIP before import, keeping temporary streams alive."""
|
||||
with ExitStack() as stack:
|
||||
try:
|
||||
handle.seek(0, 2)
|
||||
if handle.tell() > MAX_ARCHIVE_BYTES:
|
||||
raise IntegratedWorkbookError("archive_too_large")
|
||||
handle.seek(0)
|
||||
archive = stack.enter_context(zipfile.ZipFile(handle))
|
||||
workbook_member, attachment_members = _archive_members(archive)
|
||||
workbook = _read_member(archive, workbook_member, stack)
|
||||
attachments = [
|
||||
_read_member(archive, member, stack) for member in attachment_members
|
||||
]
|
||||
except (
|
||||
zipfile.BadZipFile,
|
||||
zipfile.LargeZipFile,
|
||||
EOFError,
|
||||
UnicodeDecodeError,
|
||||
zlib.error,
|
||||
) as exc:
|
||||
raise IntegratedWorkbookError("invalid_archive") from exc
|
||||
yield IntegratedStructureArchive(workbook, attachments)
|
||||
@@ -411,20 +411,17 @@ ROPK_SANCTIONS_UPLOAD_FILE_PARAM = openapi.Parameter(
|
||||
type=openapi.TYPE_FILE,
|
||||
required=True,
|
||||
)
|
||||
INTEGRATED_WORKBOOK_PARAM = openapi.Parameter(
|
||||
"workbook",
|
||||
INTEGRATED_ARCHIVE_PARAM = openapi.Parameter(
|
||||
"archive",
|
||||
openapi.IN_FORM,
|
||||
description="Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ",
|
||||
description=(
|
||||
"ZIP до 125 МиБ: один XLSX с листами is_stages и is_stages_docs "
|
||||
"(до 20 МиБ) и до 500 RTF (до 5 МиБ каждый, суммарно до 100 МиБ). "
|
||||
"Допускаются подпапки; имя RTF должно соответствовать doc_id."
|
||||
),
|
||||
type=openapi.TYPE_FILE,
|
||||
required=True,
|
||||
)
|
||||
INTEGRATED_ATTACHMENT_PARAM = openapi.Parameter(
|
||||
"attachments",
|
||||
openapi.IN_FORM,
|
||||
description="Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256",
|
||||
type=openapi.TYPE_FILE,
|
||||
required=False,
|
||||
)
|
||||
RESULT_LIST_PARAMS = [
|
||||
PAGE_PARAM,
|
||||
PAGE_SIZE_PARAM,
|
||||
@@ -3136,26 +3133,41 @@ class ParserUploadView(APIView):
|
||||
if descriptor.admin_only and not request.user.is_staff:
|
||||
return Response(status=status.HTTP_403_FORBIDDEN)
|
||||
if source_key == ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS:
|
||||
from apps.parsers.integrated_structure_archive import (
|
||||
MAX_ARCHIVE_BYTES,
|
||||
unpack_integrated_structure_archive,
|
||||
)
|
||||
from apps.parsers.integrated_structure_workbook import (
|
||||
IntegratedWorkbookError,
|
||||
)
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
from django.core.exceptions import TooManyFilesSent
|
||||
|
||||
workbook = request.FILES.get("workbook")
|
||||
if workbook is None or not workbook.name.lower().endswith(".xlsx"):
|
||||
try:
|
||||
uploaded_files = request.FILES
|
||||
archive = uploaded_files.get("archive")
|
||||
file_count = sum(len(files) for _, files in uploaded_files.lists())
|
||||
except TooManyFilesSent:
|
||||
archive = None
|
||||
file_count = 0
|
||||
if (
|
||||
archive is None
|
||||
or not archive.name.lower().endswith(".zip")
|
||||
or file_count != 1
|
||||
):
|
||||
return api_error_response(
|
||||
[
|
||||
{
|
||||
"code": "workbook_required",
|
||||
"field": "workbook",
|
||||
"message": "Требуется XLSX workbook",
|
||||
"code": "archive_required",
|
||||
"field": "archive",
|
||||
"message": "Требуется один ZIP-архив с XLSX и RTF",
|
||||
}
|
||||
],
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
)
|
||||
if workbook.size > 20 * 1024 * 1024:
|
||||
if archive.size > MAX_ARCHIVE_BYTES:
|
||||
return api_error_response(
|
||||
[{"code": "workbook_too_large", "message": "XLSX слишком велик"}],
|
||||
[{"code": "archive_too_large", "message": "ZIP превышает 125 МиБ"}],
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
)
|
||||
job = BackgroundJobService.create_job(
|
||||
@@ -3166,12 +3178,13 @@ class ParserUploadView(APIView):
|
||||
)
|
||||
job.mark_started()
|
||||
try:
|
||||
artifact, result = import_integrated_structures(
|
||||
workbook=workbook,
|
||||
original_name=workbook.name,
|
||||
attachments=request.FILES.getlist("attachments"),
|
||||
uploaded_by_id=request.user.id,
|
||||
)
|
||||
with unpack_integrated_structure_archive(archive) as files:
|
||||
artifact, result = import_integrated_structures(
|
||||
workbook=files.workbook,
|
||||
original_name=files.workbook.name,
|
||||
attachments=files.attachments,
|
||||
uploaded_by_id=request.user.id,
|
||||
)
|
||||
except IntegratedWorkbookError as exc:
|
||||
job.fail(error=str(exc))
|
||||
return api_error_response(
|
||||
|
||||
@@ -5289,26 +5289,21 @@
|
||||
"post": {
|
||||
"operationId": "api_v1_parsers_upload_integrated_structure_documents_create",
|
||||
"summary": "Документы интегрированных структур: загрузить файл",
|
||||
"description": "Ручная загрузка файла для источника: События и документы интегрированных структур. Файл обрабатывается через Celery.",
|
||||
"description": "Загрузка одного ZIP с XLSX и RTF. Архив проверяется и обрабатывается синхронно существующим импортёром.",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"multipart/form-data": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"workbook": {
|
||||
"description": "Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ",
|
||||
"type": "string",
|
||||
"format": "binary"
|
||||
},
|
||||
"attachments": {
|
||||
"description": "Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256",
|
||||
"archive": {
|
||||
"description": "ZIP до 125 МиБ: один XLSX с листами is_stages и is_stages_docs (до 20 МиБ) и до 500 RTF (до 5 МиБ каждый, суммарно до 100 МиБ). Допускаются подпапки; имя RTF должно соответствовать doc_id.",
|
||||
"type": "string",
|
||||
"format": "binary"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"workbook"
|
||||
"archive"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -5339,7 +5334,7 @@
|
||||
"description": "Неподдерживаемый тип файла"
|
||||
},
|
||||
"422": {
|
||||
"description": "Неверные листы, идентификаторы или вложения"
|
||||
"description": "Некорректный архив, листы, идентификаторы или вложения"
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
|
||||
Reference in New Issue
Block a user