From e470189f44081d30dc9fbacd001b76f9f2b9243d Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Fri, 20 Mar 2026 12:31:22 +0100 Subject: [PATCH] feat(admin): add excel upload flows for FNS reports and register lists --- src/apps/parsers/admin.py | 201 +++++++++++++++++- src/apps/registers/admin.py | 84 +++++++- .../parsers/financialreport/change_list.html | 8 + .../parsers/financialreport/upload_excel.html | 36 ++++ .../registers/registerupload/change_list.html | 8 + .../registerupload/upload_excel.html | 47 ++++ tests/apps/parsers/test_admin.py | 4 + tests/apps/registers/test_admin.py | 102 +++++++++ 8 files changed, 485 insertions(+), 5 deletions(-) create mode 100644 src/templates/admin/parsers/financialreport/change_list.html create mode 100644 src/templates/admin/parsers/financialreport/upload_excel.html create mode 100644 src/templates/admin/registers/registerupload/change_list.html create mode 100644 src/templates/admin/registers/registerupload/upload_excel.html create mode 100644 tests/apps/registers/test_admin.py diff --git a/src/apps/parsers/admin.py b/src/apps/parsers/admin.py index 7be01b3..0ec7c1d 100644 --- a/src/apps/parsers/admin.py +++ b/src/apps/parsers/admin.py @@ -2,6 +2,13 @@ Admin configuration for parsers app. """ +import hashlib +import time +import uuid +from pathlib import Path + +from apps.core.models import BackgroundJob +from apps.core.services import BackgroundJobService from apps.parsers.models import ( FinancialReport, FinancialReportLine, @@ -13,7 +20,14 @@ from apps.parsers.models import ( ProcurementRecord, Proxy, ) -from django.contrib import admin +from apps.parsers.serializers import FNSFileUploadSerializer +from apps.parsers.services import FNSReportService +from apps.parsers.tasks import process_fns_file +from django.conf import settings +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse from django.utils.html import format_html @@ -629,9 +643,11 @@ class FinancialReportLineInline(admin.TabularInline): class FinancialReportAdmin(admin.ModelAdmin): """Admin для финансовых отчетов ФНС.""" + change_list_template = "admin/parsers/financialreport/change_list.html" list_display = [ "external_id", "ogrn", + "registry_organization", "file_name", "status_badge", "source", @@ -639,11 +655,20 @@ class FinancialReportAdmin(admin.ModelAdmin): "load_batch", "created_at", ] - list_filter = ["status", "source", "load_batch", "created_at"] - search_fields = ["external_id", "ogrn", "file_name"] + list_filter = ["status", "source", "load_batch", "registry_organization", "created_at"] + search_fields = [ + "external_id", + "ogrn", + "file_name", + "registry_organization__pn_name", + "registry_organization__mn_ogrn", + "registry_organization__mn_inn", + ] + list_select_related = ["registry_organization"] readonly_fields = [ "external_id", "ogrn", + "registry_organization", "file_name", "file_hash", "load_batch", @@ -661,7 +686,15 @@ class FinancialReportAdmin(admin.ModelAdmin): fieldsets = ( ( "Основное", - {"fields": ("external_id", "ogrn", "file_name", "file_hash")}, + { + "fields": ( + "external_id", + "ogrn", + "registry_organization", + "file_name", + "file_hash", + ) + }, ), ( "Статус", @@ -701,6 +734,166 @@ class FinancialReportAdmin(admin.ModelAdmin): status_badge.short_description = "Статус" status_badge.admin_order_field = "status" + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "upload-excel/", + self.admin_site.admin_view(self.upload_excel_view), + name="parsers_financialreport_upload_excel", + ), + ] + return custom_urls + urls + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["upload_excel_url"] = reverse( + "admin:parsers_financialreport_upload_excel" + ) + return super().changelist_view(request, extra_context=extra_context) + + def upload_excel_view(self, request): + changelist_url = reverse("admin:parsers_financialreport_changelist") + + if request.method == "POST": + files = request.FILES.getlist("files") + serializer = FNSFileUploadSerializer(data={"files": files}) + + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации файлов: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + queued, skipped, task_ids = self._enqueue_fns_files( + request, + serializer.validated_data["files"], + ) + except Exception as exc: # noqa: BLE001 + self.message_user( + request, + f"Ошибка постановки файлов в очередь: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + if queued: + self.message_user( + request, + f"Файлов поставлено в очередь: {queued}. Task IDs: {', '.join(task_ids[:5])}", + level=messages.SUCCESS, + ) + if skipped: + self.message_user( + request, + f"Пропущено файлов: {skipped} (дубликаты или уже обрабатываются).", + level=messages.WARNING, + ) + if not queued and not skipped: + self.message_user( + request, + "Файлы не были обработаны.", + level=messages.WARNING, + ) + + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка Excel отчетности ФНС", + "changelist_url": changelist_url, + } + return TemplateResponse( + request, + "admin/parsers/financialreport/upload_excel.html", + context, + ) + + @staticmethod + def _try_create_fns_lock(file_path: Path) -> bool: + lock_path = Path(f"{file_path}.lock") + if lock_path.exists(): + try: + age_seconds = time.time() - lock_path.stat().st_mtime + ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600) + if age_seconds > ttl_seconds: + lock_path.unlink() + else: + return False + except FileNotFoundError: + pass + try: + lock_path.touch(exist_ok=False) + except FileExistsError: + return False + return True + + def _enqueue_fns_files(self, request, files): + upload_dir = Path(settings.FNS_WATCH_DIRECTORY) + upload_dir.mkdir(parents=True, exist_ok=True) + + task_ids = [] + queued = 0 + skipped = 0 + + for file in files: + file_content = file.read() + file_hash = hashlib.sha256(file_content).hexdigest() + file.seek(0) + + if FNSReportService.exists_by_hash(file_hash): + skipped += 1 + continue + + file_path = upload_dir / file.name + if not self._try_create_fns_lock(file_path): + skipped += 1 + continue + + lock_path = Path(f"{file_path}.lock") + if file_path.exists(): + lock_path.unlink(missing_ok=True) + skipped += 1 + continue + + try: + with open(file_path, "wb") as f: + for chunk in file.chunks(): + f.write(chunk) + except Exception: + lock_path.unlink(missing_ok=True) + raise + + task_id = str(uuid.uuid4()) + try: + BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.parsers.tasks.process_fns_file", + user_id=request.user.id, + meta={ + "source": "fns_reports", + "file": file.name, + }, + ) + task = process_fns_file.apply_async( + args=[str(file_path)], + kwargs={"requested_by_id": request.user.id}, + task_id=task_id, + ) + except Exception: + lock_path.unlink(missing_ok=True) + BackgroundJob.objects.filter(task_id=task_id).delete() + raise + + task_ids.append(task.id) + queued += 1 + + return queued, skipped, task_ids + def has_add_permission(self, request): """Запретить создание записей вручную.""" return False diff --git a/src/apps/registers/admin.py b/src/apps/registers/admin.py index 7a2c3db..081380e 100644 --- a/src/apps/registers/admin.py +++ b/src/apps/registers/admin.py @@ -6,7 +6,12 @@ from apps.registers.models import ( RegisterUpload, RegistryMembershipPeriod, ) -from django.contrib import admin +from apps.registers.serializers import RegisterFileUploadSerializer +from apps.registers.services import RegisterImportError, RegisterImportService +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse @admin.register(Register) @@ -53,6 +58,7 @@ class OrganizationAdmin(admin.ModelAdmin): class RegisterUploadAdmin(admin.ModelAdmin): """Admin для загрузок реестров.""" + change_list_template = "admin/registers/registerupload/change_list.html" list_display = [ "id", "registry", @@ -67,6 +73,82 @@ class RegisterUploadAdmin(admin.ModelAdmin): readonly_fields = ["created_at", "updated_at", "file_hash"] ordering = ["-actual_date", "-created_at"] + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "upload-excel/", + self.admin_site.admin_view(self.upload_excel_view), + name="registers_registerupload_upload_excel", + ), + ] + return custom_urls + urls + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["upload_excel_url"] = reverse( + "admin:registers_registerupload_upload_excel" + ) + return super().changelist_view(request, extra_context=extra_context) + + def upload_excel_view(self, request): + changelist_url = reverse("admin:registers_registerupload_changelist") + + if request.method == "POST": + data = request.POST.copy() + uploaded_file = request.FILES.get("file") + if uploaded_file is not None: + data["file"] = uploaded_file + + serializer = RegisterFileUploadSerializer(data=data) + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации загрузки: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + result = RegisterImportService.sync_registry_memberships( + registry=serializer.validated_data["registry"], + uploaded_file=serializer.validated_data["file"], + actual_date=serializer.validated_data.get("actual_date"), + uploaded_by=request.user, + ) + except RegisterImportError as exc: + self.message_user( + request, + f"Ошибка импорта Excel: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + self.message_user( + request, + ( + "Загрузка завершена: " + f"{result['registry_name']}, строк: {result['rows_in_file']}, " + f"новых организаций: {result['organizations_created']}, " + f"обновлено: {result['organizations_updated']}." + ), + level=messages.SUCCESS, + ) + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка списка организаций из Excel", + "changelist_url": changelist_url, + "registries": Register.objects.only("id", "name").order_by("name"), + } + return TemplateResponse( + request, + "admin/registers/registerupload/upload_excel.html", + context, + ) + @admin.register(RegistryMembershipPeriod) class RegistryMembershipPeriodAdmin(admin.ModelAdmin): diff --git a/src/templates/admin/parsers/financialreport/change_list.html b/src/templates/admin/parsers/financialreport/change_list.html new file mode 100644 index 0000000..56dae08 --- /dev/null +++ b/src/templates/admin/parsers/financialreport/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +
  • + Загрузить Excel ФНС +
    • + {{ block.super }} +{% endblock %} diff --git a/src/templates/admin/parsers/financialreport/upload_excel.html b/src/templates/admin/parsers/financialreport/upload_excel.html new file mode 100644 index 0000000..d59c938 --- /dev/null +++ b/src/templates/admin/parsers/financialreport/upload_excel.html @@ -0,0 +1,36 @@ +{% extends "admin/base_site.html" %} + +{% block breadcrumbs %} +
    + Главная + › Финансовые отчеты ФНС + › Загрузка Excel +
    +{% endblock %} + +{% block content %} +
    +
    + {% csrf_token %} +
    +
    + + +

    Ожидаемый формат имени: fin_{id}_{ogrn}.xlsx

    +
    +
    + +
    + + Отмена +
    +
    +
    +{% endblock %} diff --git a/src/templates/admin/registers/registerupload/change_list.html b/src/templates/admin/registers/registerupload/change_list.html new file mode 100644 index 0000000..592d0ff --- /dev/null +++ b/src/templates/admin/registers/registerupload/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +
  • + Загрузить Excel организаций +
    • + {{ block.super }} +{% endblock %} diff --git a/src/templates/admin/registers/registerupload/upload_excel.html b/src/templates/admin/registers/registerupload/upload_excel.html new file mode 100644 index 0000000..31ff2f1 --- /dev/null +++ b/src/templates/admin/registers/registerupload/upload_excel.html @@ -0,0 +1,47 @@ +{% extends "admin/base_site.html" %} + +{% block breadcrumbs %} +
    + Главная + › Загрузки реестров + › Загрузка Excel +
    +{% endblock %} + +{% block content %} +
    +
    + {% csrf_token %} +
    +
    + + +
    + +
    + + +

    Если не указана, будет использована текущая дата.

    +
    + +
    + + +

    + Ожидаемые колонки: pn_name, mn_ogrn, mn_inn, mn_okpo. Опционально: in_kpp. +

    +
    +
    + +
    + + Отмена +
    +
    +
    +{% endblock %} diff --git a/tests/apps/parsers/test_admin.py b/tests/apps/parsers/test_admin.py index 3e0892d..a5312d5 100644 --- a/tests/apps/parsers/test_admin.py +++ b/tests/apps/parsers/test_admin.py @@ -244,3 +244,7 @@ class ParsersAdminTest(TestCase): ) self.assertEqual(admin.lines_count(report), 1) self.assertIn("span", str(admin.status_badge(report))) + self.assertIn("registry_organization", admin.list_display) + self.assertIn("registry_organization__pn_name", admin.search_fields) + route_names = [route.name for route in admin.get_urls()] + self.assertIn("parsers_financialreport_upload_excel", route_names) diff --git a/tests/apps/registers/test_admin.py b/tests/apps/registers/test_admin.py new file mode 100644 index 0000000..2c73c9a --- /dev/null +++ b/tests/apps/registers/test_admin.py @@ -0,0 +1,102 @@ +"""Tests for registers admin configuration.""" + +import io + +from apps.registers.admin import RegisterUploadAdmin +from apps.registers.models import Organization, RegisterUpload, RegistryMembershipPeriod +from django.contrib.admin.sites import AdminSite +from django.contrib.messages.storage.fallback import FallbackStorage +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory, TestCase +from openpyxl import Workbook + +from tests.apps.registers.factories import RegisterFactory +from tests.apps.user.factories import UserFactory + + +def _build_register_excel_upload(filename: str = "registry.xlsx") -> SimpleUploadedFile: + workbook = Workbook() + worksheet = workbook.active + worksheet.append(["pn_name", "mn_ogrn", "mn_inn", "in_kpp", "mn_okpo"]) + worksheet.append( + [ + 'АО "Тестовая организация"', + 1027600980990, + 7601000086, + 760401001, + "07506197", + ] + ) + + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + + return SimpleUploadedFile( + filename, + buffer.getvalue(), + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + + +class RegistersAdminTest(TestCase): + def setUp(self): + self.site = AdminSite() + self.factory = RequestFactory() + self.user = UserFactory.create_superuser() + + def _post_request(self, data): + request = self.factory.post( + "/admin/registers/registerupload/upload-excel/", + data=data, + ) + request.user = self.user + request.session = {} + request._messages = FallbackStorage(request) + return request + + def test_register_upload_admin_has_custom_upload_route(self): + admin = RegisterUploadAdmin(RegisterUpload, self.site) + route_names = [route.name for route in admin.get_urls()] + + self.assertIn("registers_registerupload_upload_excel", route_names) + + def test_register_upload_admin_upload_excel_success(self): + admin = RegisterUploadAdmin(RegisterUpload, self.site) + registry = RegisterFactory() + uploaded_file = _build_register_excel_upload() + request = self._post_request( + { + "registry": str(registry.id), + "actual_date": "2026-03-20", + "file": uploaded_file, + } + ) + + response = admin.upload_excel_view(request) + + self.assertEqual(response.status_code, 302) + self.assertEqual(RegisterUpload.objects.count(), 1) + self.assertEqual(Organization.objects.count(), 1) + self.assertEqual(RegistryMembershipPeriod.objects.count(), 1) + + upload = RegisterUpload.objects.first() + self.assertEqual(upload.registry, registry) + self.assertEqual(upload.actual_date.isoformat(), "2026-03-20") + + def test_register_upload_admin_upload_excel_invalid_extension(self): + admin = RegisterUploadAdmin(RegisterUpload, self.site) + registry = RegisterFactory() + invalid_file = SimpleUploadedFile("registry.txt", b"text/plain") + request = self._post_request( + { + "registry": str(registry.id), + "actual_date": "2026-03-20", + "file": invalid_file, + } + ) + + response = admin.upload_excel_view(request) + + self.assertEqual(response.status_code, 302) + self.assertEqual(RegisterUpload.objects.count(), 0)