490 lines
16 KiB
Python
490 lines
16 KiB
Python
"""Reusable upload helpers for FNS financial report files."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
import time
|
|
import uuid
|
|
import zipfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
from apps.core.models import BackgroundJob
|
|
from apps.core.services import BackgroundJobService
|
|
from apps.parsers.models import ParserLoadLog
|
|
from apps.parsers.services import FNSReportService
|
|
from apps.parsers.tasks import process_fns_file
|
|
from django.conf import settings
|
|
from django.utils.text import get_valid_filename
|
|
|
|
FNS_XLSX_FILENAME_RE = re.compile(r"^fin_\d+_\d{13,15}\.xlsx$")
|
|
|
|
|
|
@dataclass
|
|
class FNSUploadResult:
|
|
"""Result of queuing FNS files for processing."""
|
|
|
|
queued: int = 0
|
|
skipped: int = 0
|
|
invalid: int = 0
|
|
task_ids: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class FNSSyncUploadResult:
|
|
"""Result of synchronous FNS file processing."""
|
|
|
|
processed: int = 0
|
|
skipped: int = 0
|
|
invalid: int = 0
|
|
failed: int = 0
|
|
|
|
|
|
class FNSUploadService:
|
|
"""Queue uploaded FNS Excel files and ZIP archives for processing."""
|
|
|
|
@classmethod
|
|
def queue_uploaded_files(
|
|
cls, *, files, requested_by_id: int | None
|
|
) -> FNSUploadResult:
|
|
result = FNSUploadResult()
|
|
seen_hashes: set[str] = set()
|
|
|
|
for uploaded_file in files:
|
|
status, task_id = cls._queue_file_bytes(
|
|
file_name=uploaded_file.name,
|
|
file_content=uploaded_file.read(),
|
|
requested_by_id=requested_by_id,
|
|
seen_hashes=seen_hashes,
|
|
)
|
|
cls._accumulate(result=result, status=status, task_id=task_id)
|
|
|
|
return result
|
|
|
|
@classmethod
|
|
def queue_zip_archive(
|
|
cls,
|
|
*,
|
|
archive_file,
|
|
requested_by_id: int | None,
|
|
) -> FNSUploadResult:
|
|
result = FNSUploadResult()
|
|
seen_hashes: set[str] = set()
|
|
|
|
archive_file.seek(0)
|
|
try:
|
|
with zipfile.ZipFile(archive_file) as archive:
|
|
for member in archive.infolist():
|
|
if member.is_dir():
|
|
continue
|
|
|
|
file_name = cls._extract_member_name(member.filename)
|
|
if not file_name or not FNS_XLSX_FILENAME_RE.match(file_name):
|
|
result.invalid += 1
|
|
continue
|
|
|
|
status, task_id = cls._queue_file_bytes(
|
|
file_name=file_name,
|
|
file_content=archive.read(member),
|
|
requested_by_id=requested_by_id,
|
|
seen_hashes=seen_hashes,
|
|
)
|
|
cls._accumulate(result=result, status=status, task_id=task_id)
|
|
except zipfile.BadZipFile as exc:
|
|
raise ValueError(
|
|
"Загруженный файл не является корректным ZIP архивом"
|
|
) from exc
|
|
|
|
return result
|
|
|
|
@classmethod
|
|
def queue_uploaded_zip_archive(
|
|
cls,
|
|
*,
|
|
archive_file,
|
|
requested_by_id: int | None,
|
|
) -> FNSUploadResult:
|
|
"""Persist a ZIP upload and queue archive expansion in Celery."""
|
|
from apps.parsers.tasks import process_fns_zip_archive
|
|
|
|
archive_path = cls._store_uploaded_archive(archive_file)
|
|
if not zipfile.is_zipfile(archive_path):
|
|
archive_path.unlink(missing_ok=True)
|
|
raise ValueError("Загруженный файл не является корректным ZIP архивом")
|
|
|
|
task_id = str(uuid.uuid4())
|
|
try:
|
|
BackgroundJobService.create_job(
|
|
task_id=task_id,
|
|
task_name="apps.parsers.tasks.process_fns_zip_archive",
|
|
user_id=requested_by_id,
|
|
meta={
|
|
"source": ParserLoadLog.Source.FNS_REPORTS,
|
|
"file": archive_path.name,
|
|
"upload_type": "zip",
|
|
},
|
|
)
|
|
task = process_fns_zip_archive.apply_async(
|
|
args=[str(archive_path)],
|
|
kwargs={"requested_by_id": requested_by_id},
|
|
task_id=task_id,
|
|
)
|
|
except Exception:
|
|
archive_path.unlink(missing_ok=True)
|
|
BackgroundJob.objects.filter(task_id=task_id).delete()
|
|
raise
|
|
|
|
return FNSUploadResult(queued=1, skipped=0, invalid=0, task_ids=[task.id])
|
|
|
|
@classmethod
|
|
def queue_server_path(
|
|
cls,
|
|
*,
|
|
server_path: str,
|
|
requested_by_id: int | None,
|
|
) -> FNSUploadResult:
|
|
"""Queue an FNS file that already exists on the worker-visible disk."""
|
|
from apps.parsers.tasks import process_fns_zip_archive
|
|
|
|
path = cls._validate_server_path(server_path)
|
|
if path.suffix.lower() == ".zip":
|
|
task_name = "apps.parsers.tasks.process_fns_zip_archive"
|
|
task = process_fns_zip_archive
|
|
upload_type = "zip_server_path"
|
|
else:
|
|
task_name = "apps.parsers.tasks.process_fns_file"
|
|
task = process_fns_file
|
|
upload_type = "file_server_path"
|
|
|
|
task_id = str(uuid.uuid4())
|
|
try:
|
|
BackgroundJobService.create_job(
|
|
task_id=task_id,
|
|
task_name=task_name,
|
|
user_id=requested_by_id,
|
|
meta={
|
|
"source": ParserLoadLog.Source.FNS_REPORTS,
|
|
"file": path.name,
|
|
"server_path": str(path),
|
|
"upload_type": upload_type,
|
|
},
|
|
)
|
|
async_result = task.apply_async(
|
|
args=[str(path)],
|
|
kwargs={"requested_by_id": requested_by_id},
|
|
task_id=task_id,
|
|
)
|
|
except Exception:
|
|
BackgroundJob.objects.filter(task_id=task_id).delete()
|
|
raise
|
|
|
|
return FNSUploadResult(
|
|
queued=1,
|
|
skipped=0,
|
|
invalid=0,
|
|
task_ids=[async_result.id],
|
|
)
|
|
|
|
@classmethod
|
|
def queue_zip_archive_path(
|
|
cls,
|
|
*,
|
|
archive_path: str | Path,
|
|
requested_by_id: int | None,
|
|
) -> FNSUploadResult:
|
|
"""Queue valid files from a ZIP archive already stored on shared disk."""
|
|
path = Path(archive_path)
|
|
try:
|
|
with path.open("rb") as handle:
|
|
return cls.queue_zip_archive(
|
|
archive_file=handle,
|
|
requested_by_id=requested_by_id,
|
|
)
|
|
finally:
|
|
path.unlink(missing_ok=True)
|
|
|
|
@classmethod
|
|
def process_uploaded_files_sync(
|
|
cls, *, files, requested_by_id: int | None
|
|
) -> FNSSyncUploadResult:
|
|
result = FNSSyncUploadResult()
|
|
seen_hashes: set[str] = set()
|
|
|
|
for uploaded_file in files:
|
|
status = cls._process_file_bytes_sync(
|
|
file_name=uploaded_file.name,
|
|
file_content=uploaded_file.read(),
|
|
requested_by_id=requested_by_id,
|
|
seen_hashes=seen_hashes,
|
|
)
|
|
cls._accumulate_sync(result=result, status=status)
|
|
|
|
return result
|
|
|
|
@classmethod
|
|
def process_zip_archive_sync(
|
|
cls,
|
|
*,
|
|
archive_file,
|
|
requested_by_id: int | None,
|
|
) -> FNSSyncUploadResult:
|
|
result = FNSSyncUploadResult()
|
|
seen_hashes: set[str] = set()
|
|
|
|
archive_file.seek(0)
|
|
try:
|
|
with zipfile.ZipFile(archive_file) as archive:
|
|
for member in archive.infolist():
|
|
if member.is_dir():
|
|
continue
|
|
|
|
file_name = cls._extract_member_name(member.filename)
|
|
if not file_name or not FNS_XLSX_FILENAME_RE.match(file_name):
|
|
result.invalid += 1
|
|
continue
|
|
|
|
status = cls._process_file_bytes_sync(
|
|
file_name=file_name,
|
|
file_content=archive.read(member),
|
|
requested_by_id=requested_by_id,
|
|
seen_hashes=seen_hashes,
|
|
)
|
|
cls._accumulate_sync(result=result, status=status)
|
|
except zipfile.BadZipFile as exc:
|
|
raise ValueError(
|
|
"Загруженный файл не является корректным ZIP архивом"
|
|
) from exc
|
|
|
|
return result
|
|
|
|
@staticmethod
|
|
def _extract_member_name(member_name: str) -> str | None:
|
|
path = PurePosixPath(member_name)
|
|
if path.is_absolute() or ".." in path.parts:
|
|
return None
|
|
if len(path.parts) != 1:
|
|
return None
|
|
file_name = path.name
|
|
return file_name or None
|
|
|
|
@staticmethod
|
|
def _store_uploaded_archive(archive_file) -> Path:
|
|
archive_dir = Path(
|
|
getattr(
|
|
settings,
|
|
"FNS_ARCHIVE_UPLOAD_DIRECTORY",
|
|
Path(settings.FNS_WATCH_DIRECTORY) / "archives",
|
|
)
|
|
)
|
|
archive_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
safe_name = get_valid_filename(archive_file.name or "fns-reports.zip")
|
|
archive_path = archive_dir / f"{uuid.uuid4()}-{safe_name}"
|
|
|
|
archive_file.seek(0)
|
|
with archive_path.open("wb") as target:
|
|
for chunk in archive_file.chunks():
|
|
target.write(chunk)
|
|
archive_file.seek(0)
|
|
return archive_path
|
|
|
|
@classmethod
|
|
def _validate_server_path(cls, server_path: str) -> Path:
|
|
raw_path = Path(server_path)
|
|
if not raw_path.is_absolute():
|
|
raise ValueError("Путь к файлу должен быть абсолютным")
|
|
|
|
path = raw_path.resolve(strict=False)
|
|
allowed_roots = cls._allowed_server_path_roots()
|
|
if not any(cls._is_relative_to(path, root) for root in allowed_roots):
|
|
allowed = ", ".join(str(root) for root in allowed_roots)
|
|
raise ValueError(f"Путь должен находиться внутри: {allowed}")
|
|
|
|
suffix = path.suffix.lower()
|
|
if suffix == ".zip":
|
|
return path
|
|
if suffix in {".xlsx", ".xlsm"} and FNS_XLSX_FILENAME_RE.match(path.name):
|
|
return path
|
|
raise ValueError(
|
|
"Поддерживаются ZIP архивы или Excel файлы формата " "fin_{id}_{ogrn}.xlsx"
|
|
)
|
|
|
|
@staticmethod
|
|
def _allowed_server_path_roots() -> list[Path]:
|
|
roots = [
|
|
Path(settings.FNS_WATCH_DIRECTORY),
|
|
Path(settings.FNS_WATCH_DIRECTORY) / "archives",
|
|
]
|
|
configured_archive_dir = getattr(
|
|
settings,
|
|
"FNS_ARCHIVE_UPLOAD_DIRECTORY",
|
|
None,
|
|
)
|
|
if configured_archive_dir:
|
|
roots.append(Path(configured_archive_dir))
|
|
return list(dict.fromkeys(root.resolve(strict=False) for root in roots))
|
|
|
|
@staticmethod
|
|
def _is_relative_to(path: Path, root: Path) -> bool:
|
|
try:
|
|
path.relative_to(root)
|
|
except ValueError:
|
|
return False
|
|
return True
|
|
|
|
@classmethod
|
|
def _queue_file_bytes(
|
|
cls,
|
|
*,
|
|
file_name: str,
|
|
file_content: bytes,
|
|
requested_by_id: int | None,
|
|
seen_hashes: set[str],
|
|
) -> tuple[str, str | None]:
|
|
status, file_path, file_hash = cls._prepare_file_bytes(
|
|
file_name=file_name,
|
|
file_content=file_content,
|
|
seen_hashes=seen_hashes,
|
|
)
|
|
if status == "skipped":
|
|
return "skipped", None
|
|
if file_path is None or file_hash is None: # pragma: no cover
|
|
raise RuntimeError("Prepared FNS file is missing processing metadata")
|
|
|
|
task_id = str(uuid.uuid4())
|
|
try:
|
|
BackgroundJobService.create_job(
|
|
task_id=task_id,
|
|
task_name="apps.parsers.tasks.process_fns_file",
|
|
user_id=requested_by_id,
|
|
meta={
|
|
"source": ParserLoadLog.Source.FNS_REPORTS,
|
|
"file": file_name,
|
|
},
|
|
)
|
|
task = process_fns_file.apply_async(
|
|
args=[str(file_path)],
|
|
kwargs={"requested_by_id": requested_by_id},
|
|
task_id=task_id,
|
|
)
|
|
except Exception:
|
|
Path(f"{file_path}.lock").unlink(missing_ok=True)
|
|
BackgroundJob.objects.filter(task_id=task_id).delete()
|
|
raise
|
|
|
|
seen_hashes.add(file_hash)
|
|
return "queued", task.id
|
|
|
|
@classmethod
|
|
def _process_file_bytes_sync(
|
|
cls,
|
|
*,
|
|
file_name: str,
|
|
file_content: bytes,
|
|
requested_by_id: int | None,
|
|
seen_hashes: set[str],
|
|
) -> str:
|
|
from apps.parsers.tasks import _process_fns_file_sync
|
|
|
|
status, file_path, file_hash = cls._prepare_file_bytes(
|
|
file_name=file_name,
|
|
file_content=file_content,
|
|
seen_hashes=seen_hashes,
|
|
)
|
|
if status == "skipped":
|
|
return "skipped"
|
|
if file_path is None or file_hash is None: # pragma: no cover
|
|
raise RuntimeError("Prepared FNS file is missing processing metadata")
|
|
|
|
result = _process_fns_file_sync(
|
|
str(file_path),
|
|
task_id=str(uuid.uuid4()),
|
|
requested_by_id=requested_by_id,
|
|
raise_on_error=False,
|
|
)
|
|
result_status = result.get("status")
|
|
if result_status == "success":
|
|
if file_hash is not None:
|
|
seen_hashes.add(file_hash)
|
|
return "processed"
|
|
if result_status == "skipped":
|
|
if file_hash is not None:
|
|
seen_hashes.add(file_hash)
|
|
return "skipped"
|
|
return "failed"
|
|
|
|
@classmethod
|
|
def _prepare_file_bytes(
|
|
cls,
|
|
*,
|
|
file_name: str,
|
|
file_content: bytes,
|
|
seen_hashes: set[str],
|
|
) -> tuple[str, Path | None, str | None]:
|
|
file_hash = hashlib.sha256(file_content).hexdigest()
|
|
if file_hash in seen_hashes or FNSReportService.exists_by_hash(file_hash):
|
|
return "skipped", None, None
|
|
|
|
upload_dir = Path(settings.FNS_WATCH_DIRECTORY)
|
|
upload_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
file_path = upload_dir / file_name
|
|
if not cls._try_create_fns_lock(file_path):
|
|
return "skipped", None, None
|
|
|
|
lock_path = Path(f"{file_path}.lock")
|
|
if file_path.exists():
|
|
lock_path.unlink(missing_ok=True)
|
|
return "skipped", None, None
|
|
|
|
try:
|
|
file_path.write_bytes(file_content)
|
|
except Exception:
|
|
lock_path.unlink(missing_ok=True)
|
|
raise
|
|
|
|
return "prepared", file_path, file_hash
|
|
|
|
@staticmethod
|
|
def _try_create_fns_lock(file_path: Path) -> bool:
|
|
lock_path = Path(f"{file_path}.lock")
|
|
if lock_path.exists():
|
|
try:
|
|
age_seconds = time.time() - lock_path.stat().st_mtime
|
|
ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600)
|
|
if age_seconds > ttl_seconds:
|
|
lock_path.unlink()
|
|
else:
|
|
return False
|
|
except FileNotFoundError:
|
|
pass
|
|
try:
|
|
lock_path.touch(exist_ok=False)
|
|
except FileExistsError:
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def _accumulate(
|
|
*, result: FNSUploadResult, status: str, task_id: str | None
|
|
) -> None:
|
|
if status == "queued":
|
|
result.queued += 1
|
|
if task_id:
|
|
result.task_ids.append(task_id)
|
|
return
|
|
if status == "skipped":
|
|
result.skipped += 1
|
|
|
|
@staticmethod
|
|
def _accumulate_sync(*, result: FNSSyncUploadResult, status: str) -> None:
|
|
if status == "processed":
|
|
result.processed += 1
|
|
return
|
|
if status == "skipped":
|
|
result.skipped += 1
|
|
return
|
|
if status == "failed":
|
|
result.failed += 1
|