fix(fns): queue uploads from worker-visible path
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 21s
CI/CD Pipeline / Build and Push Images (push) Successful in 6s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s

This commit is contained in:
2026-04-29 01:32:31 +02:00
parent 3f2056bac3
commit d6de9a27b3
5 changed files with 214 additions and 4 deletions

View File

@@ -137,6 +137,55 @@ class FNSUploadService:
return FNSUploadResult(queued=1, skipped=0, invalid=0, task_ids=[task.id]) return FNSUploadResult(queued=1, skipped=0, invalid=0, task_ids=[task.id])
@classmethod
def queue_server_path(
cls,
*,
server_path: str,
requested_by_id: int | None,
) -> FNSUploadResult:
"""Queue an FNS file that already exists on the worker-visible disk."""
from apps.parsers.tasks import process_fns_zip_archive
path = cls._validate_server_path(server_path)
if path.suffix.lower() == ".zip":
task_name = "apps.parsers.tasks.process_fns_zip_archive"
task = process_fns_zip_archive
upload_type = "zip_server_path"
else:
task_name = "apps.parsers.tasks.process_fns_file"
task = process_fns_file
upload_type = "file_server_path"
task_id = str(uuid.uuid4())
try:
BackgroundJobService.create_job(
task_id=task_id,
task_name=task_name,
user_id=requested_by_id,
meta={
"source": ParserLoadLog.Source.FNS_REPORTS,
"file": path.name,
"server_path": str(path),
"upload_type": upload_type,
},
)
async_result = task.apply_async(
args=[str(path)],
kwargs={"requested_by_id": requested_by_id},
task_id=task_id,
)
except Exception:
BackgroundJob.objects.filter(task_id=task_id).delete()
raise
return FNSUploadResult(
queued=1,
skipped=0,
invalid=0,
task_ids=[async_result.id],
)
@classmethod @classmethod
def queue_zip_archive_path( def queue_zip_archive_path(
cls, cls,
@@ -240,6 +289,50 @@ class FNSUploadService:
archive_file.seek(0) archive_file.seek(0)
return archive_path return archive_path
@classmethod
def _validate_server_path(cls, server_path: str) -> Path:
raw_path = Path(server_path)
if not raw_path.is_absolute():
raise ValueError("Путь к файлу должен быть абсолютным")
path = raw_path.resolve(strict=False)
allowed_roots = cls._allowed_server_path_roots()
if not any(cls._is_relative_to(path, root) for root in allowed_roots):
allowed = ", ".join(str(root) for root in allowed_roots)
raise ValueError(f"Путь должен находиться внутри: {allowed}")
suffix = path.suffix.lower()
if suffix == ".zip":
return path
if suffix in {".xlsx", ".xlsm"} and FNS_XLSX_FILENAME_RE.match(path.name):
return path
raise ValueError(
"Поддерживаются ZIP архивы или Excel файлы формата " "fin_{id}_{ogrn}.xlsx"
)
@staticmethod
def _allowed_server_path_roots() -> list[Path]:
roots = [
Path(settings.FNS_WATCH_DIRECTORY),
Path(settings.FNS_WATCH_DIRECTORY) / "archives",
]
configured_archive_dir = getattr(
settings,
"FNS_ARCHIVE_UPLOAD_DIRECTORY",
None,
)
if configured_archive_dir:
roots.append(Path(configured_archive_dir))
return list(dict.fromkeys(root.resolve(strict=False) for root in roots))
@staticmethod
def _is_relative_to(path: Path, root: Path) -> bool:
try:
path.relative_to(root)
except ValueError:
return False
return True
@classmethod @classmethod
def _queue_file_bytes( def _queue_file_bytes(
cls, cls,

View File

@@ -367,6 +367,14 @@ class FNSZipUploadSerializer(serializers.Serializer):
return value return value
class FNSServerPathUploadSerializer(serializers.Serializer):
"""Сериализатор для запуска обработки файла, уже лежащего на сервере."""
server_path = serializers.CharField(
help_text="Абсолютный путь к ZIP/XLSX внутри FNS watch directory",
)
class FNSFileUploadSuccessSerializer(serializers.Serializer): class FNSFileUploadSuccessSerializer(serializers.Serializer):
"""Ответ одиночной загрузки FNS в формате frontend.""" """Ответ одиночной загрузки FNS в формате frontend."""

View File

@@ -31,6 +31,7 @@ from apps.parsers.serializers import (
FinancialReportSerializer, FinancialReportSerializer,
FNSFileUploadSerializer, FNSFileUploadSerializer,
FNSFileUploadSuccessSerializer, FNSFileUploadSuccessSerializer,
FNSServerPathUploadSerializer,
FNSZipUploadSerializer, FNSZipUploadSerializer,
GenericParserRecordSerializer, GenericParserRecordSerializer,
IndustrialCertificateSerializer, IndustrialCertificateSerializer,
@@ -74,7 +75,7 @@ from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema from drf_yasg.utils import swagger_auto_schema
from rest_framework import status from rest_framework import status
from rest_framework.exceptions import ValidationError from rest_framework.exceptions import ValidationError
from rest_framework.parsers import FormParser, MultiPartParser from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
from rest_framework.permissions import IsAdminUser, IsAuthenticated from rest_framework.permissions import IsAdminUser, IsAuthenticated
from rest_framework.request import Request from rest_framework.request import Request
from rest_framework.response import Response from rest_framework.response import Response
@@ -904,7 +905,7 @@ class FNSReportUploadView(APIView):
на обработку через Celery. на обработку через Celery.
""" """
parser_classes = [MultiPartParser] parser_classes = [MultiPartParser, JSONParser]
permission_classes = [IsAdminUser] permission_classes = [IsAdminUser]
@swagger_auto_schema( @swagger_auto_schema(
@@ -923,8 +924,18 @@ class FNSReportUploadView(APIView):
name="file", name="file",
in_=openapi.IN_FORM, in_=openapi.IN_FORM,
type=openapi.TYPE_FILE, type=openapi.TYPE_FILE,
required=True, required=False,
description="Файл для загрузки (fin_*.xlsx)", description="Файл для загрузки (fin_*.xlsx или ZIP архив)",
),
openapi.Parameter(
name="server_path",
in_=openapi.IN_FORM,
type=openapi.TYPE_STRING,
required=False,
description=(
"Абсолютный путь к fin_*.xlsx или ZIP на диске worker-а "
"внутри FNS watch directory"
),
), ),
], ],
consumes=["multipart/form-data"], consumes=["multipart/form-data"],
@@ -956,6 +967,27 @@ class FNSReportUploadView(APIView):
}, },
) )
def post(self, request): # noqa def post(self, request): # noqa
if "server_path" in request.data:
serializer = FNSServerPathUploadSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
try:
result = FNSUploadService.queue_server_path(
server_path=serializer.validated_data["server_path"],
requested_by_id=request.user.id,
)
except ValueError as exc:
raise ValidationError({"server_path": str(exc)}) from exc
return Response(
{
"queued": result.queued,
"skipped": result.skipped,
"invalid": result.invalid,
"task_ids": result.task_ids,
},
status=status.HTTP_202_ACCEPTED,
)
uploaded_file = request.FILES.get("file") uploaded_file = request.FILES.get("file")
if uploaded_file and uploaded_file.name.lower().endswith(".zip"): if uploaded_file and uploaded_file.name.lower().endswith(".zip"):
serializer = FNSZipUploadSerializer(data=request.data) serializer = FNSZipUploadSerializer(data=request.data)

View File

@@ -1538,6 +1538,12 @@
<input data-upload-file="${escapeHtml(source.key)}" type="file" accept=".json,.csv,.xml,.html,.htm,.xlsx,.xlsm,.zip"> <input data-upload-file="${escapeHtml(source.key)}" type="file" accept=".json,.csv,.xml,.html,.htm,.xlsx,.xlsm,.zip">
<button class="secondary" data-upload="${escapeHtml(source.key)}" type="button">Загрузить реестр</button> <button class="secondary" data-upload="${escapeHtml(source.key)}" type="button">Загрузить реестр</button>
</div> </div>
${source.key === "fns_financial" ? `
<div class="upload-row">
<input data-upload-path="${escapeHtml(source.key)}" type="text" placeholder="/app/input/fns/archives/fin_ropk.zip">
<button class="secondary" data-upload-server-path="${escapeHtml(source.key)}" type="button">Запустить с диска</button>
</div>
` : ""}
</div> </div>
`).join(""); `).join("");
} }
@@ -2146,6 +2152,28 @@
closeModals(); closeModals();
navigateDashboard(`/dashboard/${encodeURIComponent(target.dataset.sourceLink)}`); navigateDashboard(`/dashboard/${encodeURIComponent(target.dataset.sourceLink)}`);
} }
if (target.dataset.uploadServerPath) {
const pathInput = document.querySelector(`[data-upload-path="${target.dataset.uploadServerPath}"]`);
const serverPath = pathInput?.value?.trim();
if (!serverPath) return;
const source = sourceByKey(target.dataset.uploadServerPath);
const uploadUrl = source?.upload_url || `/api/v1/parsers/upload/${target.dataset.uploadServerPath}/`;
$("manualUploadStatus").classList.remove("hidden");
$("manualUploadStatus").textContent = "Постановка файла с диска в очередь...";
try {
const result = await apiFetch(uploadUrl, {
method: "POST",
body: JSON.stringify({ server_path: serverPath }),
});
const queued = result.queued ?? result.data?.queued;
const taskIds = result.task_ids ?? result.data?.task_ids ?? [];
$("manualUploadStatus").textContent = `Файл с диска принят: queued=${queued ?? 0}${taskIds.length ? `, task=${taskIds[0]}` : ""}`;
await refreshDashboard();
showMainTab("uploads");
} catch (error) {
$("manualUploadStatus").textContent = errorMessage(error);
}
}
if (target.dataset.upload) { if (target.dataset.upload) {
const fileInput = document.querySelector(`[data-upload-file="${target.dataset.upload}"]`); const fileInput = document.querySelector(`[data-upload-file="${target.dataset.upload}"]`);
if (!fileInput || !fileInput.files.length) return; if (!fileInput || !fileInput.files.length) return;

View File

@@ -5,6 +5,7 @@ import os
import tempfile import tempfile
import time import time
import zipfile import zipfile
from types import SimpleNamespace
from unittest.mock import patch from unittest.mock import patch
from apps.core.models import BackgroundJob from apps.core.models import BackgroundJob
@@ -274,6 +275,54 @@ class FNSUploadIntegrationTest(APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
def test_upload_accepts_worker_visible_server_path(self):
with tempfile.TemporaryDirectory() as tmpdir:
watch_dir, processed_dir, failed_dir = self._dirs(tmpdir)
server_path = os.path.join(watch_dir, "archives", "fin_ropk.zip")
with override_settings(
FNS_WATCH_DIRECTORY=watch_dir,
FNS_PROCESSED_DIRECTORY=processed_dir,
FNS_FAILED_DIRECTORY=failed_dir,
), patch(
"apps.parsers.fns_upload.uuid.uuid4",
return_value="server-path-task-id",
), patch(
"apps.parsers.tasks.process_fns_zip_archive.apply_async",
return_value=SimpleNamespace(id="server-path-task-id"),
) as task_mock:
response = self.client.post(
self.upload_url,
{"server_path": server_path},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
self.assertEqual(response.data["queued"], 1)
self.assertEqual(response.data["task_ids"], ["server-path-task-id"])
task_mock.assert_called_once()
job = BackgroundJob.objects.get(task_id="server-path-task-id")
self.assertEqual(job.user_id, self.admin.id)
self.assertEqual(job.meta["server_path"], os.path.realpath(server_path))
self.assertEqual(job.meta["upload_type"], "zip_server_path")
def test_upload_rejects_server_path_outside_fns_directory(self):
with tempfile.TemporaryDirectory() as tmpdir:
watch_dir, processed_dir, failed_dir = self._dirs(tmpdir)
outside_path = os.path.join(tmpdir, "outside", "fin_ropk.zip")
with override_settings(
FNS_WATCH_DIRECTORY=watch_dir,
FNS_PROCESSED_DIRECTORY=processed_dir,
FNS_FAILED_DIRECTORY=failed_dir,
):
response = self.client.post(
self.upload_url,
{"server_path": outside_path},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertFalse(BackgroundJob.objects.exists())
def test_upload_skips_when_file_already_exists(self): def test_upload_skips_when_file_already_exists(self):
content = _build_fns_excel_bytes() content = _build_fns_excel_bytes()
external_id = _digits(5) external_id = _digits(5)