feat(admin): improve uploads and dashboard UX

src/apps/parsers/admin.py

@@ -15,6 +15,7 @@ from apps.parsers.models import (
     Proxy,
 )
 from apps.parsers.serializers import FNSFileUploadSerializer, FNSZipUploadSerializer
+from apps.parsers.services import ProxyToolsSyncError, ProxyToolsSyncService
 from django.contrib import admin, messages
 from django.shortcuts import redirect
 from django.template.response import TemplateResponse
@@ -26,6 +27,7 @@ from django.utils.html import format_html
 class ProxyAdmin(admin.ModelAdmin):
     """Admin для прокси-серверов."""
 
+    change_list_template = "admin/parsers/proxy/change_list.html"
     list_display = [
         "address",
         "country_code",
@@ -73,6 +75,78 @@ class ProxyAdmin(admin.ModelAdmin):
     is_active_badge.short_description = "Статус"
     is_active_badge.admin_order_field = "is_active"
 
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path(
+                "sync-proxy-tools/",
+                self.admin_site.admin_view(self.sync_proxy_tools_view),
+                name="parsers_proxy_sync_proxy_tools",
+            ),
+        ]
+        return custom_urls + urls
+
+    def changelist_view(self, request, extra_context=None):
+        extra_context = extra_context or {}
+        extra_context["sync_proxy_tools_url"] = reverse(
+            "admin:parsers_proxy_sync_proxy_tools"
+        )
+        return super().changelist_view(request, extra_context=extra_context)
+
+    def sync_proxy_tools_view(self, request):
+        changelist_url = reverse("admin:parsers_proxy_changelist")
+
+        if request.method != "POST":
+            self.message_user(
+                request,
+                "Обновление списка прокси доступно только через POST.",
+                level=messages.WARNING,
+            )
+            return redirect(changelist_url)
+
+        try:
+            result = ProxyToolsSyncService.sync_ru_proxies()
+        except ProxyToolsSyncError as exc:
+            self.message_user(
+                request,
+                f"Ошибка обновления списка прокси: {exc}",
+                level=messages.ERROR,
+            )
+            return redirect(changelist_url)
+
+        if result.get("status") == "success":
+            self.message_user(
+                request,
+                (
+                    "Список прокси обновлён: "
+                    f"загружено {result['fetched']}, "
+                    f"создано {result['created']}, "
+                    f"обновлено {result['updated']}, "
+                    f"деактивировано {result['deactivated']}."
+                ),
+                level=messages.SUCCESS,
+            )
+        elif result.get("status") == "skipped":
+            reason = result.get("reason")
+            message = (
+                "Обновление списка прокси пропущено: не задан PROXY_TOOLS_API_KEY."
+                if reason == "missing_api_key"
+                else f"Обновление списка прокси пропущено: {reason}."
+            )
+            self.message_user(
+                request,
+                message,
+                level=messages.WARNING,
+            )
+        else:
+            self.message_user(
+                request,
+                "Обновление списка прокси завершилось без результата.",
+                level=messages.WARNING,
+            )
+
+        return redirect(changelist_url)
+
     actions = ["activate_proxies", "deactivate_proxies", "reset_fail_count"]
 
     @admin.action(description="Активировать выбранные прокси")
@@ -770,6 +844,45 @@ class FinancialReportAdmin(admin.ModelAdmin):
         )
         return super().changelist_view(request, extra_context=extra_context)
 
+    def _message_sync_upload_result(self, request, result, *, archive: bool) -> None:
+        noun = "файлов из архива" if archive else "файлов"
+
+        if result.processed:
+            self.message_user(
+                request,
+                f"Успешно обработано {noun}: {result.processed}.",
+                level=messages.SUCCESS,
+            )
+        if result.skipped:
+            self.message_user(
+                request,
+                f"Пропущено {noun}: {result.skipped}.",
+                level=messages.WARNING,
+            )
+        if result.failed:
+            self.message_user(
+                request,
+                f"Не удалось обработать {noun}: {result.failed}.",
+                level=messages.ERROR,
+            )
+        if result.invalid:
+            self.message_user(
+                request,
+                f"Невалидных элементов в архиве: {result.invalid}.",
+                level=messages.WARNING,
+            )
+        if not result.processed and not result.skipped and not result.failed:
+            empty_message = (
+                "Архив не содержит подходящих файлов."
+                if archive
+                else "Файлы не были обработаны."
+            )
+            self.message_user(
+                request,
+                empty_message,
+                level=messages.WARNING,
+            )
+
     def upload_excel_view(self, request):
         changelist_url = reverse("admin:parsers_financialreport_changelist")
 
@@ -786,39 +899,19 @@ class FinancialReportAdmin(admin.ModelAdmin):
                 return redirect(changelist_url)
 
             try:
-                result = FNSUploadService.queue_uploaded_files(
+                result = FNSUploadService.process_uploaded_files_sync(
                     files=serializer.validated_data["files"],
                     requested_by_id=request.user.id,
                 )
             except Exception as exc:  # noqa: BLE001
                 self.message_user(
                     request,
-                    f"Ошибка постановки файлов в очередь: {exc}",
+                    f"Ошибка синхронной обработки файлов: {exc}",
                     level=messages.ERROR,
                 )
                 return redirect(changelist_url)
 
-            if result.queued:
-                self.message_user(
-                    request,
-                    "Файлов поставлено в очередь: "
-                    f"{result.queued}. Task IDs: {', '.join(result.task_ids[:5])}",
-                    level=messages.SUCCESS,
-                )
-            if result.skipped:
-                self.message_user(
-                    request,
-                    "Пропущено файлов: "
-                    f"{result.skipped} (дубликаты или уже обрабатываются).",
-                    level=messages.WARNING,
-                )
-            if not result.queued and not result.skipped:
-                self.message_user(
-                    request,
-                    "Файлы не были обработаны.",
-                    level=messages.WARNING,
-                )
-
+            self._message_sync_upload_result(request, result, archive=False)
             return redirect(changelist_url)
 
         context = {
@@ -850,7 +943,7 @@ class FinancialReportAdmin(admin.ModelAdmin):
                 return redirect(changelist_url)
 
             try:
-                result = FNSUploadService.queue_zip_archive(
+                result = FNSUploadService.process_zip_archive_sync(
                     archive_file=serializer.validated_data["file"],
                     requested_by_id=request.user.id,
                 )
@@ -862,32 +955,7 @@ class FinancialReportAdmin(admin.ModelAdmin):
                 )
                 return redirect(changelist_url)
 
-            if result.queued:
-                self.message_user(
-                    request,
-                    "Файлов из архива поставлено в очередь: "
-                    f"{result.queued}. Task IDs: {', '.join(result.task_ids[:5])}",
-                    level=messages.SUCCESS,
-                )
-            if result.skipped:
-                self.message_user(
-                    request,
-                    f"Пропущено файлов из архива: {result.skipped}.",
-                    level=messages.WARNING,
-                )
-            if result.invalid:
-                self.message_user(
-                    request,
-                    f"Невалидных элементов в архиве: {result.invalid}.",
-                    level=messages.WARNING,
-                )
-            if not result.queued and not result.skipped and not result.invalid:
-                self.message_user(
-                    request,
-                    "Архив не содержит подходящих файлов.",
-                    level=messages.WARNING,
-                )
-
+            self._message_sync_upload_result(request, result, archive=True)
             return redirect(changelist_url)
 
         context = {

src/apps/parsers/fns_upload.py

@@ -30,6 +30,16 @@ class FNSUploadResult:
     task_ids: list[str] = field(default_factory=list)
 
 
+@dataclass
+class FNSSyncUploadResult:
+    """Result of synchronous FNS file processing."""
+
+    processed: int = 0
+    skipped: int = 0
+    invalid: int = 0
+    failed: int = 0
+
+
 class FNSUploadService:
     """Queue uploaded FNS Excel files and ZIP archives for processing."""
 
@@ -87,6 +97,60 @@ class FNSUploadService:
 
         return result
 
+    @classmethod
+    def process_uploaded_files_sync(
+        cls, *, files, requested_by_id: int | None
+    ) -> FNSSyncUploadResult:
+        result = FNSSyncUploadResult()
+        seen_hashes: set[str] = set()
+
+        for uploaded_file in files:
+            status = cls._process_file_bytes_sync(
+                file_name=uploaded_file.name,
+                file_content=uploaded_file.read(),
+                requested_by_id=requested_by_id,
+                seen_hashes=seen_hashes,
+            )
+            cls._accumulate_sync(result=result, status=status)
+
+        return result
+
+    @classmethod
+    def process_zip_archive_sync(
+        cls,
+        *,
+        archive_file,
+        requested_by_id: int | None,
+    ) -> FNSSyncUploadResult:
+        result = FNSSyncUploadResult()
+        seen_hashes: set[str] = set()
+
+        archive_file.seek(0)
+        try:
+            with zipfile.ZipFile(archive_file) as archive:
+                for member in archive.infolist():
+                    if member.is_dir():
+                        continue
+
+                    file_name = cls._extract_member_name(member.filename)
+                    if not file_name or not FNS_XLSX_FILENAME_RE.match(file_name):
+                        result.invalid += 1
+                        continue
+
+                    status = cls._process_file_bytes_sync(
+                        file_name=file_name,
+                        file_content=archive.read(member),
+                        requested_by_id=requested_by_id,
+                        seen_hashes=seen_hashes,
+                    )
+                    cls._accumulate_sync(result=result, status=status)
+        except zipfile.BadZipFile as exc:
+            raise ValueError(
+                "Загруженный файл не является корректным ZIP архивом"
+            ) from exc
+
+        return result
+
     @staticmethod
     def _extract_member_name(member_name: str) -> str | None:
         path = PurePosixPath(member_name)
@@ -106,27 +170,15 @@ class FNSUploadService:
         requested_by_id: int | None,
         seen_hashes: set[str],
     ) -> tuple[str, str | None]:
-        file_hash = hashlib.sha256(file_content).hexdigest()
-        if file_hash in seen_hashes or FNSReportService.exists_by_hash(file_hash):
+        status, file_path, file_hash = cls._prepare_file_bytes(
+            file_name=file_name,
+            file_content=file_content,
+            seen_hashes=seen_hashes,
+        )
+        if status == "skipped":
             return "skipped", None
-
-        upload_dir = Path(settings.FNS_WATCH_DIRECTORY)
-        upload_dir.mkdir(parents=True, exist_ok=True)
-
-        file_path = upload_dir / file_name
-        if not cls._try_create_fns_lock(file_path):
-            return "skipped", None
-
-        lock_path = Path(f"{file_path}.lock")
-        if file_path.exists():
-            lock_path.unlink(missing_ok=True)
-            return "skipped", None
-
-        try:
-            file_path.write_bytes(file_content)
-        except Exception:
-            lock_path.unlink(missing_ok=True)
-            raise
+        if file_path is None or file_hash is None:  # pragma: no cover
+            raise RuntimeError("Prepared FNS file is missing processing metadata")
 
         task_id = str(uuid.uuid4())
         try:
@@ -145,13 +197,83 @@ class FNSUploadService:
                 task_id=task_id,
             )
         except Exception:
-            lock_path.unlink(missing_ok=True)
+            Path(f"{file_path}.lock").unlink(missing_ok=True)
             BackgroundJob.objects.filter(task_id=task_id).delete()
             raise
 
         seen_hashes.add(file_hash)
         return "queued", task.id
 
+    @classmethod
+    def _process_file_bytes_sync(
+        cls,
+        *,
+        file_name: str,
+        file_content: bytes,
+        requested_by_id: int | None,
+        seen_hashes: set[str],
+    ) -> str:
+        from apps.parsers.tasks import _process_fns_file_sync
+
+        status, file_path, file_hash = cls._prepare_file_bytes(
+            file_name=file_name,
+            file_content=file_content,
+            seen_hashes=seen_hashes,
+        )
+        if status == "skipped":
+            return "skipped"
+        if file_path is None or file_hash is None:  # pragma: no cover
+            raise RuntimeError("Prepared FNS file is missing processing metadata")
+
+        result = _process_fns_file_sync(
+            str(file_path),
+            task_id=str(uuid.uuid4()),
+            requested_by_id=requested_by_id,
+            raise_on_error=False,
+        )
+        result_status = result.get("status")
+        if result_status == "success":
+            if file_hash is not None:
+                seen_hashes.add(file_hash)
+            return "processed"
+        if result_status == "skipped":
+            if file_hash is not None:
+                seen_hashes.add(file_hash)
+            return "skipped"
+        return "failed"
+
+    @classmethod
+    def _prepare_file_bytes(
+        cls,
+        *,
+        file_name: str,
+        file_content: bytes,
+        seen_hashes: set[str],
+    ) -> tuple[str, Path | None, str | None]:
+        file_hash = hashlib.sha256(file_content).hexdigest()
+        if file_hash in seen_hashes or FNSReportService.exists_by_hash(file_hash):
+            return "skipped", None, None
+
+        upload_dir = Path(settings.FNS_WATCH_DIRECTORY)
+        upload_dir.mkdir(parents=True, exist_ok=True)
+
+        file_path = upload_dir / file_name
+        if not cls._try_create_fns_lock(file_path):
+            return "skipped", None, None
+
+        lock_path = Path(f"{file_path}.lock")
+        if file_path.exists():
+            lock_path.unlink(missing_ok=True)
+            return "skipped", None, None
+
+        try:
+            file_path.write_bytes(file_content)
+        except Exception:
+            lock_path.unlink(missing_ok=True)
+            raise
+
+        return "prepared", file_path, file_hash
+
     @staticmethod
     def _try_create_fns_lock(file_path: Path) -> bool:
         lock_path = Path(f"{file_path}.lock")
@@ -182,3 +304,14 @@ class FNSUploadService:
             return
         if status == "skipped":
             result.skipped += 1
+
+    @staticmethod
+    def _accumulate_sync(*, result: FNSSyncUploadResult, status: str) -> None:
+        if status == "processed":
+            result.processed += 1
+            return
+        if status == "skipped":
+            result.skipped += 1
+            return
+        if status == "failed":
+            result.failed += 1