diff --git a/src/apps/parsers/admin.py b/src/apps/parsers/admin.py index 7be01b3..0ec7c1d 100644 --- a/src/apps/parsers/admin.py +++ b/src/apps/parsers/admin.py @@ -2,6 +2,13 @@ Admin configuration for parsers app. """ +import hashlib +import time +import uuid +from pathlib import Path + +from apps.core.models import BackgroundJob +from apps.core.services import BackgroundJobService from apps.parsers.models import ( FinancialReport, FinancialReportLine, @@ -13,7 +20,14 @@ from apps.parsers.models import ( ProcurementRecord, Proxy, ) -from django.contrib import admin +from apps.parsers.serializers import FNSFileUploadSerializer +from apps.parsers.services import FNSReportService +from apps.parsers.tasks import process_fns_file +from django.conf import settings +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse from django.utils.html import format_html @@ -629,9 +643,11 @@ class FinancialReportLineInline(admin.TabularInline): class FinancialReportAdmin(admin.ModelAdmin): """Admin для финансовых отчетов ФНС.""" + change_list_template = "admin/parsers/financialreport/change_list.html" list_display = [ "external_id", "ogrn", + "registry_organization", "file_name", "status_badge", "source", @@ -639,11 +655,20 @@ class FinancialReportAdmin(admin.ModelAdmin): "load_batch", "created_at", ] - list_filter = ["status", "source", "load_batch", "created_at"] - search_fields = ["external_id", "ogrn", "file_name"] + list_filter = ["status", "source", "load_batch", "registry_organization", "created_at"] + search_fields = [ + "external_id", + "ogrn", + "file_name", + "registry_organization__pn_name", + "registry_organization__mn_ogrn", + "registry_organization__mn_inn", + ] + list_select_related = ["registry_organization"] readonly_fields = [ "external_id", "ogrn", + "registry_organization", "file_name", "file_hash", "load_batch", @@ -661,7 +686,15 @@ class FinancialReportAdmin(admin.ModelAdmin): fieldsets = ( ( "Основное", - {"fields": ("external_id", "ogrn", "file_name", "file_hash")}, + { + "fields": ( + "external_id", + "ogrn", + "registry_organization", + "file_name", + "file_hash", + ) + }, ), ( "Статус", @@ -701,6 +734,166 @@ class FinancialReportAdmin(admin.ModelAdmin): status_badge.short_description = "Статус" status_badge.admin_order_field = "status" + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "upload-excel/", + self.admin_site.admin_view(self.upload_excel_view), + name="parsers_financialreport_upload_excel", + ), + ] + return custom_urls + urls + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["upload_excel_url"] = reverse( + "admin:parsers_financialreport_upload_excel" + ) + return super().changelist_view(request, extra_context=extra_context) + + def upload_excel_view(self, request): + changelist_url = reverse("admin:parsers_financialreport_changelist") + + if request.method == "POST": + files = request.FILES.getlist("files") + serializer = FNSFileUploadSerializer(data={"files": files}) + + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации файлов: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + queued, skipped, task_ids = self._enqueue_fns_files( + request, + serializer.validated_data["files"], + ) + except Exception as exc: # noqa: BLE001 + self.message_user( + request, + f"Ошибка постановки файлов в очередь: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + if queued: + self.message_user( + request, + f"Файлов поставлено в очередь: {queued}. Task IDs: {', '.join(task_ids[:5])}", + level=messages.SUCCESS, + ) + if skipped: + self.message_user( + request, + f"Пропущено файлов: {skipped} (дубликаты или уже обрабатываются).", + level=messages.WARNING, + ) + if not queued and not skipped: + self.message_user( + request, + "Файлы не были обработаны.", + level=messages.WARNING, + ) + + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка Excel отчетности ФНС", + "changelist_url": changelist_url, + } + return TemplateResponse( + request, + "admin/parsers/financialreport/upload_excel.html", + context, + ) + + @staticmethod + def _try_create_fns_lock(file_path: Path) -> bool: + lock_path = Path(f"{file_path}.lock") + if lock_path.exists(): + try: + age_seconds = time.time() - lock_path.stat().st_mtime + ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600) + if age_seconds > ttl_seconds: + lock_path.unlink() + else: + return False + except FileNotFoundError: + pass + try: + lock_path.touch(exist_ok=False) + except FileExistsError: + return False + return True + + def _enqueue_fns_files(self, request, files): + upload_dir = Path(settings.FNS_WATCH_DIRECTORY) + upload_dir.mkdir(parents=True, exist_ok=True) + + task_ids = [] + queued = 0 + skipped = 0 + + for file in files: + file_content = file.read() + file_hash = hashlib.sha256(file_content).hexdigest() + file.seek(0) + + if FNSReportService.exists_by_hash(file_hash): + skipped += 1 + continue + + file_path = upload_dir / file.name + if not self._try_create_fns_lock(file_path): + skipped += 1 + continue + + lock_path = Path(f"{file_path}.lock") + if file_path.exists(): + lock_path.unlink(missing_ok=True) + skipped += 1 + continue + + try: + with open(file_path, "wb") as f: + for chunk in file.chunks(): + f.write(chunk) + except Exception: + lock_path.unlink(missing_ok=True) + raise + + task_id = str(uuid.uuid4()) + try: + BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.parsers.tasks.process_fns_file", + user_id=request.user.id, + meta={ + "source": "fns_reports", + "file": file.name, + }, + ) + task = process_fns_file.apply_async( + args=[str(file_path)], + kwargs={"requested_by_id": request.user.id}, + task_id=task_id, + ) + except Exception: + lock_path.unlink(missing_ok=True) + BackgroundJob.objects.filter(task_id=task_id).delete() + raise + + task_ids.append(task.id) + queued += 1 + + return queued, skipped, task_ids + def has_add_permission(self, request): """Запретить создание записей вручную.""" return False diff --git a/src/apps/registers/admin.py b/src/apps/registers/admin.py index 7a2c3db..081380e 100644 --- a/src/apps/registers/admin.py +++ b/src/apps/registers/admin.py @@ -6,7 +6,12 @@ from apps.registers.models import ( RegisterUpload, RegistryMembershipPeriod, ) -from django.contrib import admin +from apps.registers.serializers import RegisterFileUploadSerializer +from apps.registers.services import RegisterImportError, RegisterImportService +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse @admin.register(Register) @@ -53,6 +58,7 @@ class OrganizationAdmin(admin.ModelAdmin): class RegisterUploadAdmin(admin.ModelAdmin): """Admin для загрузок реестров.""" + change_list_template = "admin/registers/registerupload/change_list.html" list_display = [ "id", "registry", @@ -67,6 +73,82 @@ class RegisterUploadAdmin(admin.ModelAdmin): readonly_fields = ["created_at", "updated_at", "file_hash"] ordering = ["-actual_date", "-created_at"] + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "upload-excel/", + self.admin_site.admin_view(self.upload_excel_view), + name="registers_registerupload_upload_excel", + ), + ] + return custom_urls + urls + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["upload_excel_url"] = reverse( + "admin:registers_registerupload_upload_excel" + ) + return super().changelist_view(request, extra_context=extra_context) + + def upload_excel_view(self, request): + changelist_url = reverse("admin:registers_registerupload_changelist") + + if request.method == "POST": + data = request.POST.copy() + uploaded_file = request.FILES.get("file") + if uploaded_file is not None: + data["file"] = uploaded_file + + serializer = RegisterFileUploadSerializer(data=data) + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации загрузки: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + result = RegisterImportService.sync_registry_memberships( + registry=serializer.validated_data["registry"], + uploaded_file=serializer.validated_data["file"], + actual_date=serializer.validated_data.get("actual_date"), + uploaded_by=request.user, + ) + except RegisterImportError as exc: + self.message_user( + request, + f"Ошибка импорта Excel: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + self.message_user( + request, + ( + "Загрузка завершена: " + f"{result['registry_name']}, строк: {result['rows_in_file']}, " + f"новых организаций: {result['organizations_created']}, " + f"обновлено: {result['organizations_updated']}." + ), + level=messages.SUCCESS, + ) + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка списка организаций из Excel", + "changelist_url": changelist_url, + "registries": Register.objects.only("id", "name").order_by("name"), + } + return TemplateResponse( + request, + "admin/registers/registerupload/upload_excel.html", + context, + ) + @admin.register(RegistryMembershipPeriod) class RegistryMembershipPeriodAdmin(admin.ModelAdmin): diff --git a/src/templates/admin/parsers/financialreport/change_list.html b/src/templates/admin/parsers/financialreport/change_list.html new file mode 100644 index 0000000..56dae08 --- /dev/null +++ b/src/templates/admin/parsers/financialreport/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +