"""Reusable upload helpers for FNS financial report files.""" from __future__ import annotations import hashlib import re import time import uuid import zipfile from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from apps.core.models import BackgroundJob from apps.core.services import BackgroundJobService from apps.parsers.models import ParserLoadLog from apps.parsers.services import FNSReportService from apps.parsers.tasks import process_fns_file from django.conf import settings from django.utils.text import get_valid_filename FNS_XLSX_FILENAME_RE = re.compile(r"^fin_\d+_\d{13,15}\.xlsx$") @dataclass class FNSUploadResult: """Result of queuing FNS files for processing.""" queued: int = 0 skipped: int = 0 invalid: int = 0 task_ids: list[str] = field(default_factory=list) @dataclass class FNSSyncUploadResult: """Result of synchronous FNS file processing.""" processed: int = 0 skipped: int = 0 invalid: int = 0 failed: int = 0 class FNSUploadService: """Queue uploaded FNS Excel files and ZIP archives for processing.""" @classmethod def queue_uploaded_files( cls, *, files, requested_by_id: int | None ) -> FNSUploadResult: result = FNSUploadResult() seen_hashes: set[str] = set() for uploaded_file in files: status, task_id = cls._queue_file_bytes( file_name=uploaded_file.name, file_content=uploaded_file.read(), requested_by_id=requested_by_id, seen_hashes=seen_hashes, ) cls._accumulate(result=result, status=status, task_id=task_id) return result @classmethod def queue_zip_archive( cls, *, archive_file, requested_by_id: int | None, ) -> FNSUploadResult: result = FNSUploadResult() seen_hashes: set[str] = set() archive_file.seek(0) try: with zipfile.ZipFile(archive_file) as archive: for member in archive.infolist(): if member.is_dir(): continue file_name = cls._extract_member_name(member.filename) if not file_name or not FNS_XLSX_FILENAME_RE.match(file_name): result.invalid += 1 continue status, task_id = cls._queue_file_bytes( file_name=file_name, file_content=archive.read(member), requested_by_id=requested_by_id, seen_hashes=seen_hashes, ) cls._accumulate(result=result, status=status, task_id=task_id) except zipfile.BadZipFile as exc: raise ValueError( "Загруженный файл не является корректным ZIP архивом" ) from exc return result @classmethod def queue_uploaded_zip_archive( cls, *, archive_file, requested_by_id: int | None, ) -> FNSUploadResult: """Persist a ZIP upload and queue archive expansion in Celery.""" from apps.parsers.tasks import process_fns_zip_archive archive_path = cls._store_uploaded_archive(archive_file) if not zipfile.is_zipfile(archive_path): archive_path.unlink(missing_ok=True) raise ValueError("Загруженный файл не является корректным ZIP архивом") task_id = str(uuid.uuid4()) try: BackgroundJobService.create_job( task_id=task_id, task_name="apps.parsers.tasks.process_fns_zip_archive", user_id=requested_by_id, meta={ "source": ParserLoadLog.Source.FNS_REPORTS, "file": archive_path.name, "upload_type": "zip", }, ) task = process_fns_zip_archive.apply_async( args=[str(archive_path)], kwargs={"requested_by_id": requested_by_id}, task_id=task_id, ) except Exception: archive_path.unlink(missing_ok=True) BackgroundJob.objects.filter(task_id=task_id).delete() raise return FNSUploadResult(queued=1, skipped=0, invalid=0, task_ids=[task.id]) @classmethod def queue_server_path( cls, *, server_path: str, requested_by_id: int | None, ) -> FNSUploadResult: """Queue an FNS file that already exists on the worker-visible disk.""" from apps.parsers.tasks import process_fns_zip_archive path = cls._validate_server_path(server_path) if path.suffix.lower() == ".zip": task_name = "apps.parsers.tasks.process_fns_zip_archive" task = process_fns_zip_archive upload_type = "zip_server_path" else: task_name = "apps.parsers.tasks.process_fns_file" task = process_fns_file upload_type = "file_server_path" task_id = str(uuid.uuid4()) try: BackgroundJobService.create_job( task_id=task_id, task_name=task_name, user_id=requested_by_id, meta={ "source": ParserLoadLog.Source.FNS_REPORTS, "file": path.name, "server_path": str(path), "upload_type": upload_type, }, ) async_result = task.apply_async( args=[str(path)], kwargs={"requested_by_id": requested_by_id}, task_id=task_id, ) except Exception: BackgroundJob.objects.filter(task_id=task_id).delete() raise return FNSUploadResult( queued=1, skipped=0, invalid=0, task_ids=[async_result.id], ) @classmethod def queue_zip_archive_path( cls, *, archive_path: str | Path, requested_by_id: int | None, ) -> FNSUploadResult: """Queue valid files from a ZIP archive already stored on shared disk.""" path = Path(archive_path) try: with path.open("rb") as handle: return cls.queue_zip_archive( archive_file=handle, requested_by_id=requested_by_id, ) finally: path.unlink(missing_ok=True) @classmethod def process_uploaded_files_sync( cls, *, files, requested_by_id: int | None ) -> FNSSyncUploadResult: result = FNSSyncUploadResult() seen_hashes: set[str] = set() for uploaded_file in files: status = cls._process_file_bytes_sync( file_name=uploaded_file.name, file_content=uploaded_file.read(), requested_by_id=requested_by_id, seen_hashes=seen_hashes, ) cls._accumulate_sync(result=result, status=status) return result @classmethod def process_zip_archive_sync( cls, *, archive_file, requested_by_id: int | None, ) -> FNSSyncUploadResult: result = FNSSyncUploadResult() seen_hashes: set[str] = set() archive_file.seek(0) try: with zipfile.ZipFile(archive_file) as archive: for member in archive.infolist(): if member.is_dir(): continue file_name = cls._extract_member_name(member.filename) if not file_name or not FNS_XLSX_FILENAME_RE.match(file_name): result.invalid += 1 continue status = cls._process_file_bytes_sync( file_name=file_name, file_content=archive.read(member), requested_by_id=requested_by_id, seen_hashes=seen_hashes, ) cls._accumulate_sync(result=result, status=status) except zipfile.BadZipFile as exc: raise ValueError( "Загруженный файл не является корректным ZIP архивом" ) from exc return result @staticmethod def _extract_member_name(member_name: str) -> str | None: path = PurePosixPath(member_name) if path.is_absolute() or ".." in path.parts: return None if len(path.parts) != 1: return None file_name = path.name return file_name or None @staticmethod def _store_uploaded_archive(archive_file) -> Path: archive_dir = Path( getattr( settings, "FNS_ARCHIVE_UPLOAD_DIRECTORY", Path(settings.FNS_WATCH_DIRECTORY) / "archives", ) ) archive_dir.mkdir(parents=True, exist_ok=True) safe_name = get_valid_filename(archive_file.name or "fns-reports.zip") archive_path = archive_dir / f"{uuid.uuid4()}-{safe_name}" archive_file.seek(0) with archive_path.open("wb") as target: for chunk in archive_file.chunks(): target.write(chunk) archive_file.seek(0) return archive_path @classmethod def _validate_server_path(cls, server_path: str) -> Path: raw_path = Path(server_path) if not raw_path.is_absolute(): raise ValueError("Путь к файлу должен быть абсолютным") path = raw_path.resolve(strict=False) allowed_roots = cls._allowed_server_path_roots() if not any(cls._is_relative_to(path, root) for root in allowed_roots): allowed = ", ".join(str(root) for root in allowed_roots) raise ValueError(f"Путь должен находиться внутри: {allowed}") suffix = path.suffix.lower() if suffix == ".zip": return path if suffix in {".xlsx", ".xlsm"} and FNS_XLSX_FILENAME_RE.match(path.name): return path raise ValueError( "Поддерживаются ZIP архивы или Excel файлы формата " "fin_{id}_{ogrn}.xlsx" ) @staticmethod def _allowed_server_path_roots() -> list[Path]: roots = [ Path(settings.FNS_WATCH_DIRECTORY), Path(settings.FNS_WATCH_DIRECTORY) / "archives", ] configured_archive_dir = getattr( settings, "FNS_ARCHIVE_UPLOAD_DIRECTORY", None, ) if configured_archive_dir: roots.append(Path(configured_archive_dir)) return list(dict.fromkeys(root.resolve(strict=False) for root in roots)) @staticmethod def _is_relative_to(path: Path, root: Path) -> bool: try: path.relative_to(root) except ValueError: return False return True @classmethod def _queue_file_bytes( cls, *, file_name: str, file_content: bytes, requested_by_id: int | None, seen_hashes: set[str], ) -> tuple[str, str | None]: status, file_path, file_hash = cls._prepare_file_bytes( file_name=file_name, file_content=file_content, seen_hashes=seen_hashes, ) if status == "skipped": return "skipped", None if file_path is None or file_hash is None: # pragma: no cover raise RuntimeError("Prepared FNS file is missing processing metadata") task_id = str(uuid.uuid4()) try: BackgroundJobService.create_job( task_id=task_id, task_name="apps.parsers.tasks.process_fns_file", user_id=requested_by_id, meta={ "source": ParserLoadLog.Source.FNS_REPORTS, "file": file_name, }, ) task = process_fns_file.apply_async( args=[str(file_path)], kwargs={"requested_by_id": requested_by_id}, task_id=task_id, ) except Exception: Path(f"{file_path}.lock").unlink(missing_ok=True) BackgroundJob.objects.filter(task_id=task_id).delete() raise seen_hashes.add(file_hash) return "queued", task.id @classmethod def _process_file_bytes_sync( cls, *, file_name: str, file_content: bytes, requested_by_id: int | None, seen_hashes: set[str], ) -> str: from apps.parsers.tasks import _process_fns_file_sync status, file_path, file_hash = cls._prepare_file_bytes( file_name=file_name, file_content=file_content, seen_hashes=seen_hashes, ) if status == "skipped": return "skipped" if file_path is None or file_hash is None: # pragma: no cover raise RuntimeError("Prepared FNS file is missing processing metadata") result = _process_fns_file_sync( str(file_path), task_id=str(uuid.uuid4()), requested_by_id=requested_by_id, raise_on_error=False, ) result_status = result.get("status") if result_status == "success": if file_hash is not None: seen_hashes.add(file_hash) return "processed" if result_status == "skipped": if file_hash is not None: seen_hashes.add(file_hash) return "skipped" return "failed" @classmethod def _prepare_file_bytes( cls, *, file_name: str, file_content: bytes, seen_hashes: set[str], ) -> tuple[str, Path | None, str | None]: file_hash = hashlib.sha256(file_content).hexdigest() if file_hash in seen_hashes or FNSReportService.exists_by_hash(file_hash): return "skipped", None, None upload_dir = Path(settings.FNS_WATCH_DIRECTORY) upload_dir.mkdir(parents=True, exist_ok=True) file_path = upload_dir / file_name if not cls._try_create_fns_lock(file_path): return "skipped", None, None lock_path = Path(f"{file_path}.lock") if file_path.exists(): lock_path.unlink(missing_ok=True) return "skipped", None, None try: file_path.write_bytes(file_content) except Exception: lock_path.unlink(missing_ok=True) raise return "prepared", file_path, file_hash @staticmethod def _try_create_fns_lock(file_path: Path) -> bool: lock_path = Path(f"{file_path}.lock") if lock_path.exists(): try: age_seconds = time.time() - lock_path.stat().st_mtime ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600) if age_seconds > ttl_seconds: lock_path.unlink() else: return False except FileNotFoundError: pass try: lock_path.touch(exist_ok=False) except FileExistsError: return False return True @staticmethod def _accumulate( *, result: FNSUploadResult, status: str, task_id: str | None ) -> None: if status == "queued": result.queued += 1 if task_id: result.task_ids.append(task_id) return if status == "skipped": result.skipped += 1 @staticmethod def _accumulate_sync(*, result: FNSSyncUploadResult, status: str) -> None: if status == "processed": result.processed += 1 return if status == "skipped": result.skipped += 1 return if status == "failed": result.failed += 1