diff --git a/.env.prod.example b/.env.prod.example index be3eecf..fd06fa3 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -25,6 +25,9 @@ CELERY_WORKER_CONCURRENCY=4 # Parsers API keys CHECKO_API_KEY=CHANGE_ME_CHECKO_API_KEY ZAKUPKI_TOKEN=CHANGE_ME_ZAKUPKI_TOKEN +# Optional: comma-separated HTTP(S) proxies for parser tasks +# Example: PARSER_PROXIES=http://user:pass@proxy1:8080,http://user:pass@proxy2:8080 +PARSER_PROXIES= # 1 to collect static files during migrate service, 0 to skip COLLECTSTATIC_ON_MIGRATE=1 diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 97de422..39bc1fc 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -63,6 +63,7 @@ jobs: - name: Telegram notify (lint failed) if: failure() + continue-on-error: true run: | set -euo pipefail if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then @@ -78,9 +79,16 @@ jobs: actor=${GITHUB_ACTOR} commit=${COMMIT_MESSAGE}" - curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ + curl -fsS \ + --connect-timeout 5 \ + --max-time 15 \ + --retry 2 \ + --retry-delay 2 \ + --retry-all-errors \ + -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" + --data-urlencode "text=${MSG}" \ + || echo "Telegram notification failed; continue pipeline" test: name: Run Tests @@ -128,6 +136,7 @@ jobs: - name: Telegram notify (test failed) if: failure() + continue-on-error: true run: | set -euo pipefail if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then @@ -143,13 +152,21 @@ jobs: actor=${GITHUB_ACTOR} commit=${COMMIT_MESSAGE}" - curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ + curl -fsS \ + --connect-timeout 5 \ + --max-time 15 \ + --retry 2 \ + --retry-delay 2 \ + --retry-all-errors \ + -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" + --data-urlencode "text=${MSG}" \ + || echo "Telegram notification failed; continue pipeline" notify_success: name: Telegram Notify Success runs-on: ubuntu-latest + timeout-minutes: 1 needs: [lint, test] if: | always() && @@ -160,6 +177,7 @@ jobs: TG_CHANNEL: ${{ secrets.TG_CHANNEL }} steps: - name: Telegram notify (lint+test success) + continue-on-error: true env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | @@ -175,6 +193,13 @@ jobs: actor=${GITHUB_ACTOR} commit=${COMMIT_MESSAGE:-n/a}" - curl -fsS -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ + curl -fsS \ + --connect-timeout 5 \ + --max-time 15 \ + --retry 2 \ + --retry-delay 2 \ + --retry-all-errors \ + -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" + --data-urlencode "text=${MSG}" \ + || echo "Telegram notification failed; continue pipeline" diff --git a/src/apps/parsers/admin.py b/src/apps/parsers/admin.py index 7be01b3..5e3523b 100644 --- a/src/apps/parsers/admin.py +++ b/src/apps/parsers/admin.py @@ -2,6 +2,7 @@ Admin configuration for parsers app. """ +from apps.parsers.fns_upload import FNSUploadService from apps.parsers.models import ( FinancialReport, FinancialReportLine, @@ -13,7 +14,11 @@ from apps.parsers.models import ( ProcurementRecord, Proxy, ) -from django.contrib import admin +from apps.parsers.serializers import FNSFileUploadSerializer, FNSZipUploadSerializer +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse from django.utils.html import format_html @@ -629,9 +634,11 @@ class FinancialReportLineInline(admin.TabularInline): class FinancialReportAdmin(admin.ModelAdmin): """Admin для финансовых отчетов ФНС.""" + change_list_template = "admin/parsers/financialreport/change_list.html" list_display = [ "external_id", "ogrn", + "registry_organization", "file_name", "status_badge", "source", @@ -639,11 +646,26 @@ class FinancialReportAdmin(admin.ModelAdmin): "load_batch", "created_at", ] - list_filter = ["status", "source", "load_batch", "created_at"] - search_fields = ["external_id", "ogrn", "file_name"] + list_filter = [ + "status", + "source", + "load_batch", + "registry_organization", + "created_at", + ] + search_fields = [ + "external_id", + "ogrn", + "file_name", + "registry_organization__pn_name", + "registry_organization__mn_ogrn", + "registry_organization__mn_inn", + ] + list_select_related = ["registry_organization"] readonly_fields = [ "external_id", "ogrn", + "registry_organization", "file_name", "file_hash", "load_batch", @@ -661,7 +683,15 @@ class FinancialReportAdmin(admin.ModelAdmin): fieldsets = ( ( "Основное", - {"fields": ("external_id", "ogrn", "file_name", "file_hash")}, + { + "fields": ( + "external_id", + "ogrn", + "registry_organization", + "file_name", + "file_hash", + ) + }, ), ( "Статус", @@ -701,6 +731,164 @@ class FinancialReportAdmin(admin.ModelAdmin): status_badge.short_description = "Статус" status_badge.admin_order_field = "status" + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "upload-excel/", + self.admin_site.admin_view(self.upload_excel_view), + name="parsers_financialreport_upload_excel", + ), + path( + "upload-zip/", + self.admin_site.admin_view(self.upload_zip_view), + name="parsers_financialreport_upload_zip", + ), + ] + return custom_urls + urls + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["upload_excel_url"] = reverse( + "admin:parsers_financialreport_upload_excel" + ) + extra_context["upload_zip_url"] = reverse( + "admin:parsers_financialreport_upload_zip" + ) + return super().changelist_view(request, extra_context=extra_context) + + def upload_excel_view(self, request): + changelist_url = reverse("admin:parsers_financialreport_changelist") + + if request.method == "POST": + files = request.FILES.getlist("files") + serializer = FNSFileUploadSerializer(data={"files": files}) + + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации файлов: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + result = FNSUploadService.queue_uploaded_files( + files=serializer.validated_data["files"], + requested_by_id=request.user.id, + ) + except Exception as exc: # noqa: BLE001 + self.message_user( + request, + f"Ошибка постановки файлов в очередь: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + if result.queued: + self.message_user( + request, + "Файлов поставлено в очередь: " + f"{result.queued}. Task IDs: {', '.join(result.task_ids[:5])}", + level=messages.SUCCESS, + ) + if result.skipped: + self.message_user( + request, + "Пропущено файлов: " + f"{result.skipped} (дубликаты или уже обрабатываются).", + level=messages.WARNING, + ) + if not result.queued and not result.skipped: + self.message_user( + request, + "Файлы не были обработаны.", + level=messages.WARNING, + ) + + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка Excel отчетности ФНС", + "changelist_url": changelist_url, + } + return TemplateResponse( + request, + "admin/parsers/financialreport/upload_excel.html", + context, + ) + + def upload_zip_view(self, request): + changelist_url = reverse("admin:parsers_financialreport_changelist") + + if request.method == "POST": + serializer = FNSZipUploadSerializer( + data={"file": request.FILES.get("file")} + ) + + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации архива: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + result = FNSUploadService.queue_zip_archive( + archive_file=serializer.validated_data["file"], + requested_by_id=request.user.id, + ) + except Exception as exc: # noqa: BLE001 + self.message_user( + request, + f"Ошибка обработки ZIP архива: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + if result.queued: + self.message_user( + request, + "Файлов из архива поставлено в очередь: " + f"{result.queued}. Task IDs: {', '.join(result.task_ids[:5])}", + level=messages.SUCCESS, + ) + if result.skipped: + self.message_user( + request, + f"Пропущено файлов из архива: {result.skipped}.", + level=messages.WARNING, + ) + if result.invalid: + self.message_user( + request, + f"Невалидных элементов в архиве: {result.invalid}.", + level=messages.WARNING, + ) + if not result.queued and not result.skipped and not result.invalid: + self.message_user( + request, + "Архив не содержит подходящих файлов.", + level=messages.WARNING, + ) + + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка ZIP отчетности ФНС", + "changelist_url": changelist_url, + } + return TemplateResponse( + request, + "admin/parsers/financialreport/upload_zip.html", + context, + ) + def has_add_permission(self, request): """Запретить создание записей вручную.""" return False diff --git a/src/apps/parsers/fns_upload.py b/src/apps/parsers/fns_upload.py new file mode 100644 index 0000000..d2ba6fc --- /dev/null +++ b/src/apps/parsers/fns_upload.py @@ -0,0 +1,184 @@ +"""Reusable upload helpers for FNS financial report files.""" + +from __future__ import annotations + +import hashlib +import re +import time +import uuid +import zipfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath + +from apps.core.models import BackgroundJob +from apps.core.services import BackgroundJobService +from apps.parsers.models import ParserLoadLog +from apps.parsers.services import FNSReportService +from apps.parsers.tasks import process_fns_file +from django.conf import settings + +FNS_XLSX_FILENAME_RE = re.compile(r"^fin_\d+_\d{13,15}\.xlsx$") + + +@dataclass +class FNSUploadResult: + """Result of queuing FNS files for processing.""" + + queued: int = 0 + skipped: int = 0 + invalid: int = 0 + task_ids: list[str] = field(default_factory=list) + + +class FNSUploadService: + """Queue uploaded FNS Excel files and ZIP archives for processing.""" + + @classmethod + def queue_uploaded_files( + cls, *, files, requested_by_id: int | None + ) -> FNSUploadResult: + result = FNSUploadResult() + seen_hashes: set[str] = set() + + for uploaded_file in files: + status, task_id = cls._queue_file_bytes( + file_name=uploaded_file.name, + file_content=uploaded_file.read(), + requested_by_id=requested_by_id, + seen_hashes=seen_hashes, + ) + cls._accumulate(result=result, status=status, task_id=task_id) + + return result + + @classmethod + def queue_zip_archive( + cls, + *, + archive_file, + requested_by_id: int | None, + ) -> FNSUploadResult: + result = FNSUploadResult() + seen_hashes: set[str] = set() + + archive_file.seek(0) + try: + with zipfile.ZipFile(archive_file) as archive: + for member in archive.infolist(): + if member.is_dir(): + continue + + file_name = cls._extract_member_name(member.filename) + if not file_name or not FNS_XLSX_FILENAME_RE.match(file_name): + result.invalid += 1 + continue + + status, task_id = cls._queue_file_bytes( + file_name=file_name, + file_content=archive.read(member), + requested_by_id=requested_by_id, + seen_hashes=seen_hashes, + ) + cls._accumulate(result=result, status=status, task_id=task_id) + except zipfile.BadZipFile as exc: + raise ValueError( + "Загруженный файл не является корректным ZIP архивом" + ) from exc + + return result + + @staticmethod + def _extract_member_name(member_name: str) -> str | None: + path = PurePosixPath(member_name) + if path.is_absolute() or ".." in path.parts: + return None + if len(path.parts) != 1: + return None + file_name = path.name + return file_name or None + + @classmethod + def _queue_file_bytes( + cls, + *, + file_name: str, + file_content: bytes, + requested_by_id: int | None, + seen_hashes: set[str], + ) -> tuple[str, str | None]: + file_hash = hashlib.sha256(file_content).hexdigest() + if file_hash in seen_hashes or FNSReportService.exists_by_hash(file_hash): + return "skipped", None + + upload_dir = Path(settings.FNS_WATCH_DIRECTORY) + upload_dir.mkdir(parents=True, exist_ok=True) + + file_path = upload_dir / file_name + if not cls._try_create_fns_lock(file_path): + return "skipped", None + + lock_path = Path(f"{file_path}.lock") + if file_path.exists(): + lock_path.unlink(missing_ok=True) + return "skipped", None + + try: + file_path.write_bytes(file_content) + except Exception: + lock_path.unlink(missing_ok=True) + raise + + task_id = str(uuid.uuid4()) + try: + BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.parsers.tasks.process_fns_file", + user_id=requested_by_id, + meta={ + "source": ParserLoadLog.Source.FNS_REPORTS, + "file": file_name, + }, + ) + task = process_fns_file.apply_async( + args=[str(file_path)], + kwargs={"requested_by_id": requested_by_id}, + task_id=task_id, + ) + except Exception: + lock_path.unlink(missing_ok=True) + BackgroundJob.objects.filter(task_id=task_id).delete() + raise + + seen_hashes.add(file_hash) + return "queued", task.id + + @staticmethod + def _try_create_fns_lock(file_path: Path) -> bool: + lock_path = Path(f"{file_path}.lock") + if lock_path.exists(): + try: + age_seconds = time.time() - lock_path.stat().st_mtime + ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600) + if age_seconds > ttl_seconds: + lock_path.unlink() + else: + return False + except FileNotFoundError: + pass + try: + lock_path.touch(exist_ok=False) + except FileExistsError: + return False + return True + + @staticmethod + def _accumulate( + *, result: FNSUploadResult, status: str, task_id: str | None + ) -> None: + if status == "queued": + result.queued += 1 + if task_id: + result.task_ids.append(task_id) + return + if status == "skipped": + result.skipped += 1 diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index 7209d06..458d32f 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -4,6 +4,7 @@ Все сериализаторы read-only, так как данные загружаются только через парсеры. """ +from apps.parsers.fns_upload import FNS_XLSX_FILENAME_RE from apps.parsers.models import ( FinancialReport, FinancialReportLine, @@ -269,12 +270,8 @@ class FNSFileUploadSerializer(serializers.Serializer): def validate_files(self, value): """Валидация файлов.""" - import re - - pattern = re.compile(r"^fin_\d+_\d{13,15}\.xlsx$") - for file in value: - if not pattern.match(file.name): + if not FNS_XLSX_FILENAME_RE.match(file.name): raise serializers.ValidationError( f"Неверный формат имени файла: {file.name}. " "Ожидается: fin_{{id}}_{{ogrn}}.xlsx" @@ -283,6 +280,17 @@ class FNSFileUploadSerializer(serializers.Serializer): return value +class FNSZipUploadSerializer(serializers.Serializer): + """Сериализатор для загрузки ZIP архива с FNS Excel файлами.""" + + file = serializers.FileField(help_text="ZIP архив с файлами fin_*.xlsx") + + def validate_file(self, value): + if not value.name.lower().endswith(".zip"): + raise serializers.ValidationError("Поддерживаются только ZIP архивы") + return value + + # ============================================================================= # Служебные модели # ============================================================================= diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 39f7432..436d49b 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -32,6 +32,7 @@ from apps.parsers.services import ( ProxyService, ) from celery import shared_task +from django.conf import settings from requests.adapters import BaseAdapter logger = logging.getLogger(__name__) @@ -41,6 +42,26 @@ DEFAULT_START_YEAR = 2025 DEFAULT_START_MONTH = 1 +def _resolve_proxies(proxies: list[str] | None) -> list[str] | None: + """ + Разрешить итоговый список прокси. + + Приоритет: + 1. Явно переданные в задачу `proxies` + 2. Активные прокси из БД + 3. `settings.PARSER_PROXIES` (например, из ENV) + """ + if proxies is not None: + return proxies + + db_proxies = ProxyService.get_active_proxies_or_none() + if db_proxies: + return db_proxies + + configured_proxies = getattr(settings, "PARSER_PROXIES", []) or [] + return configured_proxies or None + + def _get_or_create_background_job( *, task_id: str, @@ -301,9 +322,7 @@ def parse_industrial_production( ) task_id = self.request.id or str(uuid.uuid4()) - # Если прокси не переданы, берём из БД - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting industrial production parsing (task_id=%s, batch_id=%d, proxies=%d)", @@ -395,9 +414,7 @@ def parse_manufactures( ) task_id = self.request.id or str(uuid.uuid4()) - # Если прокси не переданы, берём из БД - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting manufactures parsing (task_id=%s, batch_id=%d, proxies=%d)", @@ -488,8 +505,7 @@ def parse_industrial_products( ) task_id = self.request.id or str(uuid.uuid4()) - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting industrial products parsing (task_id=%s, batch_id=%d, proxies=%d)", @@ -634,9 +650,7 @@ def parse_inspections( ) task_id = self.request.id or str(uuid.uuid4()) - # Если прокси не переданы, берём из БД - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting inspections parsing (task_id=%s, batch_id=%d, year=%s, month=%s, proxies=%d)", @@ -828,9 +842,7 @@ def sync_inspections( # noqa: C901 ) task_id = self.request.id or str(uuid.uuid4()) - # Если прокси не переданы, берём из БД - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting inspections sync (task_id=%s, batch_id=%d)", task_id, batch_id @@ -1039,9 +1051,7 @@ def parse_procurements( ) task_id = self.request.id or str(uuid.uuid4()) - # Если прокси не переданы, берём из БД - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting procurements parsing " @@ -1174,9 +1184,7 @@ def sync_procurements( # noqa: C901 ) task_id = self.request.id or str(uuid.uuid4()) - # Если прокси не переданы, берём из БД - if proxies is None: - proxies = ProxyService.get_active_proxies_or_none() + proxies = _resolve_proxies(proxies) logger.info( "Starting procurements sync (task_id=%s, batch_id=%d, region=%s, law=%s-FZ)", diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index a04e325..3b2fd77 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -6,14 +6,10 @@ Views для приложения парсеров. """ import csv -import hashlib -import time -import uuid -from pathlib import Path from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from apps.core.response import api_response -from apps.core.services import BackgroundJobService +from apps.parsers.fns_upload import FNSUploadService from apps.parsers.models import ( FinancialReport, IndustrialCertificateRecord, @@ -42,8 +38,6 @@ from apps.parsers.serializers import ( SourceTaskStatusSerializer, ) from apps.parsers.source_cards import SourceCardService -from apps.parsers.tasks import process_fns_file -from django.conf import settings from django.db.models import CharField, Count, Q from django.db.models.functions import Cast from django.http import HttpResponse @@ -532,97 +526,16 @@ class FNSReportUploadView(APIView): serializer = FNSFileUploadSerializer(data=request.data) serializer.is_valid(raise_exception=True) - files = serializer.validated_data["files"] - task_ids = [] - queued = 0 - skipped = 0 - - # Создаём директорию для загрузки - upload_dir = Path(settings.FNS_WATCH_DIRECTORY) - upload_dir.mkdir(parents=True, exist_ok=True) - - from apps.parsers.services import FNSReportService - - def _try_create_fns_lock(file_path: Path) -> bool: - lock_path = Path(f"{file_path}.lock") - if lock_path.exists(): - try: - age_seconds = time.time() - lock_path.stat().st_mtime - ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600) - if age_seconds > ttl_seconds: - lock_path.unlink() - else: - return False - except FileNotFoundError: - pass - try: - lock_path.touch(exist_ok=False) - except FileExistsError: - return False - return True - - for file in files: - # Вычисляем хеш файла - file_content = file.read() - file_hash = hashlib.sha256(file_content).hexdigest() - file.seek(0) - - # Проверяем дубликат - if FNSReportService.exists_by_hash(file_hash): - skipped += 1 - continue - - # Сохраняем файл - file_path = upload_dir / file.name - if not _try_create_fns_lock(file_path): - skipped += 1 - continue - lock_path = Path(f"{file_path}.lock") - - if file_path.exists(): - lock_path.unlink(missing_ok=True) - skipped += 1 - continue - - try: - with open(file_path, "wb") as f: - for chunk in file.chunks(): - f.write(chunk) - except Exception: - lock_path.unlink(missing_ok=True) - raise - - # Ставим в очередь - try: - task_id = str(uuid.uuid4()) - BackgroundJobService.create_job( - task_id=task_id, - task_name="apps.parsers.tasks.process_fns_file", - user_id=request.user.id, - meta={ - "source": ParserLoadLog.Source.FNS_REPORTS, - "file": file.name, - }, - ) - task = process_fns_file.apply_async( - args=[str(file_path)], - kwargs={"requested_by_id": request.user.id}, - task_id=task_id, - ) - except Exception: - lock_path.unlink(missing_ok=True) - from apps.core.models import BackgroundJob - - BackgroundJob.objects.filter(task_id=task_id).delete() - raise - task_ids.append(task.id) - queued += 1 + result = FNSUploadService.queue_uploaded_files( + files=serializer.validated_data["files"], + requested_by_id=request.user.id, + ) return Response( { - "queued": queued, - "skipped": skipped, - "task_ids": task_ids, + "queued": result.queued, + "skipped": result.skipped, + "task_ids": result.task_ids, }, status=status.HTTP_202_ACCEPTED, ) diff --git a/src/apps/registers/admin.py b/src/apps/registers/admin.py index 7a2c3db..081380e 100644 --- a/src/apps/registers/admin.py +++ b/src/apps/registers/admin.py @@ -6,7 +6,12 @@ from apps.registers.models import ( RegisterUpload, RegistryMembershipPeriod, ) -from django.contrib import admin +from apps.registers.serializers import RegisterFileUploadSerializer +from apps.registers.services import RegisterImportError, RegisterImportService +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.urls import path, reverse @admin.register(Register) @@ -53,6 +58,7 @@ class OrganizationAdmin(admin.ModelAdmin): class RegisterUploadAdmin(admin.ModelAdmin): """Admin для загрузок реестров.""" + change_list_template = "admin/registers/registerupload/change_list.html" list_display = [ "id", "registry", @@ -67,6 +73,82 @@ class RegisterUploadAdmin(admin.ModelAdmin): readonly_fields = ["created_at", "updated_at", "file_hash"] ordering = ["-actual_date", "-created_at"] + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "upload-excel/", + self.admin_site.admin_view(self.upload_excel_view), + name="registers_registerupload_upload_excel", + ), + ] + return custom_urls + urls + + def changelist_view(self, request, extra_context=None): + extra_context = extra_context or {} + extra_context["upload_excel_url"] = reverse( + "admin:registers_registerupload_upload_excel" + ) + return super().changelist_view(request, extra_context=extra_context) + + def upload_excel_view(self, request): + changelist_url = reverse("admin:registers_registerupload_changelist") + + if request.method == "POST": + data = request.POST.copy() + uploaded_file = request.FILES.get("file") + if uploaded_file is not None: + data["file"] = uploaded_file + + serializer = RegisterFileUploadSerializer(data=data) + if not serializer.is_valid(): + self.message_user( + request, + f"Ошибка валидации загрузки: {serializer.errors}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + try: + result = RegisterImportService.sync_registry_memberships( + registry=serializer.validated_data["registry"], + uploaded_file=serializer.validated_data["file"], + actual_date=serializer.validated_data.get("actual_date"), + uploaded_by=request.user, + ) + except RegisterImportError as exc: + self.message_user( + request, + f"Ошибка импорта Excel: {exc}", + level=messages.ERROR, + ) + return redirect(changelist_url) + + self.message_user( + request, + ( + "Загрузка завершена: " + f"{result['registry_name']}, строк: {result['rows_in_file']}, " + f"новых организаций: {result['organizations_created']}, " + f"обновлено: {result['organizations_updated']}." + ), + level=messages.SUCCESS, + ) + return redirect(changelist_url) + + context = { + **self.admin_site.each_context(request), + "opts": self.model._meta, + "title": "Загрузка списка организаций из Excel", + "changelist_url": changelist_url, + "registries": Register.objects.only("id", "name").order_by("name"), + } + return TemplateResponse( + request, + "admin/registers/registerupload/upload_excel.html", + context, + ) + @admin.register(RegistryMembershipPeriod) class RegistryMembershipPeriodAdmin(admin.ModelAdmin): diff --git a/src/apps/user/migrations/0009_alter_user_groups.py b/src/apps/user/migrations/0009_alter_user_groups.py new file mode 100644 index 0000000..8629805 --- /dev/null +++ b/src/apps/user/migrations/0009_alter_user_groups.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2026-03-20 11:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ('user', '0008_alter_user_groups'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='groups', + field=models.ManyToManyField(blank=True, help_text='', related_name='custom_user_set', related_query_name='custom_user', to='auth.Group', verbose_name='groups'), + ), + ] diff --git a/src/apps/user/models.py b/src/apps/user/models.py index 42aa7ba..14fc8ee 100644 --- a/src/apps/user/models.py +++ b/src/apps/user/models.py @@ -13,17 +13,16 @@ class User(AbstractUser): # Переопределяем группы и разрешения для избежания конфликта groups = models.ManyToManyField( "auth.Group", - verbose_name=_("groups"), + verbose_name="groups", blank=True, - help_text=_(""), related_name="custom_user_set", related_query_name="custom_user", ) user_permissions = models.ManyToManyField( "auth.Permission", - verbose_name=_("user permissions"), + verbose_name="user permissions", blank=True, - help_text=_("Specific permissions for this user."), + help_text="Specific permissions for this user.", related_name="custom_user_set", related_query_name="custom_user", ) diff --git a/src/settings/base.py b/src/settings/base.py index 7da16a3..28e0348 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -71,7 +71,7 @@ JAZZMIN_SETTINGS = { # User menu "topmenu_links": [ {"name": "Главная", "url": "admin:index", "permissions": ["auth.view_user"]}, - {"name": "API Docs", "url": "/api/docs/", "new_window": True}, + {"name": "API Docs", "url": "schema-swagger-ui", "new_window": True}, {"model": "user.User"}, ], # Side menu @@ -112,7 +112,7 @@ JAZZMIN_SETTINGS = { # Related modal "related_modal_active": True, # UI Tweaks - "custom_css": None, + "custom_css": "admin/css/mostovik-admin-theme.css", "custom_js": None, "use_google_fonts_cdn": True, "show_ui_builder": False, @@ -144,7 +144,7 @@ JAZZMIN_UI_TWEAKS = { "sidebar_nav_compact_style": False, "sidebar_nav_legacy_style": False, "sidebar_nav_flat_style": False, - "theme": "default", + "theme": "darkly", "dark_mode_theme": "darkly", "button_classes": { "primary": "btn-primary", @@ -197,7 +197,9 @@ WSGI_APPLICATION = "core.wsgi.application" ZAKUPKI_TOKEN = os.getenv("ZAKUPKI_TOKEN", "") FNS_LOCK_TTL_SECONDS = 3600 -PARSER_PROXIES = [] +PARSER_PROXIES = [ + item.strip() for item in os.getenv("PARSER_PROXIES", "").split(",") if item.strip() +] BACKUP_ENCRYPTION_KEY = os.getenv("BACKUP_ENCRYPTION_KEY", "") BACKUP_KEY_ID = os.getenv("BACKUP_KEY_ID", "default") BACKUP_EXPORT_DIRECTORY = os.getenv( diff --git a/src/static/admin/css/mostovik-admin-theme.css b/src/static/admin/css/mostovik-admin-theme.css new file mode 100644 index 0000000..f5dce67 --- /dev/null +++ b/src/static/admin/css/mostovik-admin-theme.css @@ -0,0 +1,622 @@ +@import url("https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@400;500;600;700&family=JetBrains+Mono:wght@500&display=swap"); + +:root { + --mx-bg: #031015; + --mx-bg-elevated: #071a20; + --mx-bg-soft: #0c2830; + --mx-surface: rgba(7, 22, 29, 0.9); + --mx-surface-strong: rgba(10, 30, 36, 0.97); + --mx-border: rgba(71, 170, 171, 0.2); + --mx-border-strong: rgba(98, 119, 211, 0.28); + --mx-text: #e1f5f3; + --mx-text-muted: #9ec8c6; + --mx-text-dim: #668c8b; + --mx-azure: #49d0c8; + --mx-violet: #5664c9; + --mx-violet-soft: #4352aa; + --mx-success: #2dd4bf; + --mx-warning: #ffc857; + --mx-danger: #ff6b9f; + --mx-shadow: 0 24px 60px rgba(1, 8, 11, 0.44); +} + +html, +body, +.wrapper, +.content-wrapper, +.main-footer, +.login-page, +.register-page { + background: + radial-gradient(circle at 50% 12%, rgba(58, 208, 198, 0.24), transparent 26%), + radial-gradient(circle at 20% 30%, rgba(26, 133, 136, 0.18), transparent 24%), + radial-gradient(circle at 85% 18%, rgba(86, 100, 201, 0.12), transparent 18%), + linear-gradient(180deg, #020a0d 0%, #041218 22%, var(--mx-bg) 58%, #02080b 100%); + color: var(--mx-text); + font-family: "IBM Plex Sans", "Segoe UI", sans-serif; +} + +body, +.content-wrapper, +.main-footer, +.content-header h1, +.content-header h1 small, +.content-wrapper .content, +.card-title, +.small-box, +.table, +.table th, +.table td, +.form-control, +.form-control:focus, +.select2-container--default .select2-selection--single, +.select2-container--default .select2-selection--multiple, +.select2-results__option, +.btn, +.nav-sidebar .nav-link, +.brand-text, +.login-box-msg, +.help, +.text-muted, +.small, +.errornote, +.messagelist li { + color: var(--mx-text); +} + +.main-header.navbar { + background: linear-gradient(90deg, rgba(4, 14, 18, 0.95), rgba(8, 28, 33, 0.96)); + border-bottom: 1px solid rgba(73, 208, 200, 0.14); + box-shadow: 0 10px 35px rgba(1, 7, 10, 0.32); + backdrop-filter: blur(14px); +} + +.main-header .nav-link, +.main-header .navbar-nav .nav-link { + color: var(--mx-text-muted); +} + +.main-header .nav-link:hover, +.main-header .navbar-nav .nav-link:hover { + color: var(--mx-text); +} + +.main-sidebar { + background: + linear-gradient(180deg, rgba(3, 13, 17, 0.985) 0%, rgba(5, 19, 24, 0.985) 100%), + radial-gradient(circle at top, rgba(73, 208, 200, 0.1), transparent 34%); + border-right: 1px solid rgba(73, 208, 200, 0.1); + box-shadow: inset -1px 0 0 rgba(255, 255, 255, 0.03); + overflow-x: hidden; +} + +.brand-link { + border-bottom: 1px solid rgba(73, 208, 200, 0.14); + background: linear-gradient(90deg, rgba(6, 18, 22, 0.97), rgba(10, 32, 37, 0.96)); +} + +.brand-link .brand-text { + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.nav-sidebar > .nav-item > .nav-link { + margin: 0 0 0.4rem; + border-radius: 14px; + color: var(--mx-text-muted); + transition: background-color 0.18s ease, color 0.18s ease, transform 0.18s ease; +} + +.nav-sidebar > .nav-item > .nav-link:hover, +.nav-sidebar > .nav-item.menu-open > .nav-link, +.nav-sidebar .nav-treeview > .nav-item > .nav-link:hover { + background: rgba(73, 208, 200, 0.1); + color: var(--mx-text); + transform: translateX(2px); +} + +.nav-sidebar > .nav-item > .nav-link.active, +.nav-sidebar .nav-treeview > .nav-item > .nav-link.active { + background: linear-gradient(90deg, rgba(73, 208, 200, 0.18), rgba(86, 100, 201, 0.18)); + border: 1px solid rgba(73, 208, 200, 0.16); + color: #ffffff; + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.04), + 0 8px 18px rgba(6, 18, 22, 0.28); +} + +#jazzy-sidebar .sidebar { + padding: 0.85rem 0.75rem 1rem; +} + +#jazzy-sidebar .user-panel { + margin-left: 0; + margin-right: 0; + padding-left: 0.25rem; + padding-right: 0.25rem; +} + +#jazzy-sidebar .nav-sidebar { + padding-right: 0.1rem; +} + +.nav-sidebar .nav-header { + color: var(--mx-text-dim); + font-size: 0.7rem; + font-weight: 700; + letter-spacing: 0.12em; + text-transform: uppercase; +} + +.content-wrapper { + background-color: transparent; +} + +.content-wrapper > .content { + padding: 1rem 1.1rem 1.4rem 1.15rem; +} + +.content-wrapper > .content > .container-fluid { + padding-left: 0.35rem; + padding-right: 1rem; +} + +.content-header h1 { + font-weight: 700; + letter-spacing: 0.01em; +} + +.content-header, +.main-footer, +.card, +.small-box, +.info-box, +.module, +div.breadcrumbs, +#changelist-filter, +.submit-row, +.inline-group, +.selector, +.selector-available h2, +.selector-chosen h2, +.login-card-body, +.card-body, +.card-header, +.login-logo, +.messagelist li, +.object-tools a, +.paginator, +.results, +fieldset.module, +.tabular.inline-related, +.tabular tr.form-row td, +.tabular tr.form-row th { + background: var(--mx-surface); + border: 1px solid var(--mx-border); + box-shadow: var(--mx-shadow); +} + +.content-header, +.main-footer, +.card, +.info-box, +.module, +.inline-group, +.selector, +.login-card-body, +fieldset.module { + border-radius: 14px; +} + +.card, +.module, +.inline-group, +fieldset.module, +.submit-row, +#changelist-filter, +.paginator { + backdrop-filter: blur(12px); +} + +.card-header, +.module h2, +.module caption, +.inline-group h2, +.selector-available h2, +.selector-chosen h2 { + background: linear-gradient(90deg, rgba(10, 34, 40, 0.96), rgba(12, 40, 47, 0.94)); + border-bottom: 1px solid rgba(73, 208, 200, 0.12); + color: #f6f9ff; + font-weight: 700; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.card-header, +.module h2, +.module caption, +.inline-group h2 { + border-radius: 12px 12px 0 0; +} + +a, +.breadcrumb a, +div.breadcrumbs a, +.historylink, +.viewsitelink { + color: var(--mx-azure); +} + +a:hover, +.breadcrumb a:hover, +div.breadcrumbs a:hover { + color: #88d9ff; +} + +div.breadcrumbs, +.breadcrumb { + color: var(--mx-text-muted); +} + +table, +.results table, +#result_list, +.table { + background: transparent; + border-collapse: separate; + border-spacing: 0; +} + +.results thead th, +.table thead th { + background: rgba(10, 34, 40, 0.96); + border-bottom: 1px solid rgba(73, 208, 200, 0.12); + color: #d5f6f2; + font-size: 0.76rem; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.results tbody tr, +.table tbody tr { + background: rgba(5, 18, 23, 0.68); +} + +.results tbody tr:nth-child(even), +.table tbody tr:nth-child(even) { + background: rgba(8, 25, 31, 0.8); +} + +.results tbody tr:hover, +.table tbody tr:hover { + background: rgba(13, 40, 48, 0.9); +} + +.results td, +.results th, +.table td, +.table th { + border-color: rgba(73, 208, 200, 0.08); + padding-top: 0.78rem; + padding-bottom: 0.78rem; + vertical-align: middle; +} + +input[type="text"], +input[type="password"], +input[type="email"], +input[type="number"], +input[type="url"], +input[type="search"], +input[type="date"], +input[type="file"], +select, +textarea, +.form-control, +.vTextField, +.vLargeTextField, +.vURLField, +.vIntegerField, +.select2-container--default .select2-selection--single, +.select2-container--default .select2-selection--multiple { + background: rgba(4, 14, 18, 0.92); + border: 1px solid rgba(73, 208, 200, 0.14); + border-radius: 10px; + color: var(--mx-text); + min-height: 2.7rem; +} + +textarea, +.vLargeTextField { + min-height: 8rem; +} + +input:focus, +select:focus, +textarea:focus, +.form-control:focus, +.select2-container--default.select2-container--focus .select2-selection--multiple, +.select2-container--default.select2-container--open .select2-selection--single { + background: rgba(7, 22, 27, 0.99); + border-color: rgba(73, 208, 200, 0.44); + box-shadow: 0 0 0 0.22rem rgba(73, 208, 200, 0.13); +} + +label, +.aligned label, +fieldset label, +.required label, +.form-row label { + color: #d8e7ff; + font-weight: 600; +} + +.help, +.helptext, +p.help, +.form-text, +small, +.text-muted { + color: var(--mx-text-dim) !important; +} + +.submit-row, +.object-tools, +.actions, +.paginator { + border-radius: 12px; +} + +.button, +input[type="submit"], +input[type="button"], +.submit-row input, +.object-tools a, +.btn, +a.button { + background: linear-gradient(135deg, rgba(23, 97, 100, 0.62), rgba(42, 72, 133, 0.46)); + border: 1px solid rgba(73, 208, 200, 0.18); + border-radius: 10px; + color: #f5fbff; + font-weight: 700; + letter-spacing: 0.02em; + text-shadow: none; + box-shadow: 0 12px 30px rgba(2, 10, 13, 0.25); +} + +.button.default, +input[type="submit"].default, +.btn-primary, +.object-tools a.addlink { + background: linear-gradient(135deg, rgba(51, 179, 168, 0.9), rgba(65, 112, 196, 0.82)); + border-color: rgba(98, 119, 211, 0.48); + color: #ffffff; +} + +.button:hover, +input[type="submit"]:hover, +input[type="button"]:hover, +.btn:hover, +a.button:hover, +.object-tools a:hover { + filter: brightness(1.08); + transform: translateY(-1px); +} + +.deletelink, +.btn-danger { + background: linear-gradient(135deg, rgba(255, 107, 159, 0.82), rgba(167, 53, 118, 0.88)); + border-color: rgba(255, 107, 159, 0.55); +} + +.messagelist li, +.errornote, +.success, +.warning { + border-radius: 12px; + padding: 0.95rem 1.1rem; +} + +.messagelist .success, +li.success { + border-left: 4px solid var(--mx-success); +} + +.messagelist .warning, +li.warning { + border-left: 4px solid var(--mx-warning); +} + +.messagelist .error, +.errornote, +li.error { + border-left: 4px solid var(--mx-danger); +} + +code, +pre, +tt, +.mono, +.readonly, +.file-upload, +.field-file_hash .readonly { + font-family: "JetBrains Mono", monospace; +} + +pre, +code { + background: rgba(4, 15, 19, 0.94); + border: 1px solid rgba(73, 208, 200, 0.12); + border-radius: 12px; + color: #b6f4eb; +} + +#changelist-filter h2, +#changelist-filter h3 { + color: #eff5ff; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +#changelist-filter li.selected a, +#changelist-filter a:hover { + color: #ffffff; +} + +.login-box .card, +.login-card-body { + border-radius: 16px; +} + +.login-logo a { + color: #f5fbff; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.login-page .card { + background: + linear-gradient(180deg, rgba(6, 18, 22, 0.97), rgba(8, 28, 33, 0.97)), + radial-gradient(circle at top, rgba(73, 208, 200, 0.12), transparent 28%); +} + +.pagination .page-link, +.paginator a, +.paginator .this-page { + background: rgba(4, 15, 19, 0.86); + border: 1px solid rgba(73, 208, 200, 0.12); + color: var(--mx-text); +} + +.pagination .active .page-link, +.paginator .this-page { + background: linear-gradient(135deg, rgba(51, 179, 168, 0.82), rgba(65, 112, 196, 0.76)); + border-color: rgba(73, 208, 200, 0.28); + color: #ffffff; +} + +::-webkit-scrollbar { + width: 12px; + height: 12px; +} + +::-webkit-scrollbar-track { + background: rgba(3, 12, 15, 0.92); +} + +::-webkit-scrollbar-thumb { + background: linear-gradient(180deg, rgba(73, 208, 200, 0.7), rgba(65, 112, 196, 0.66)); + border: 2px solid rgba(3, 12, 15, 0.92); + border-radius: 999px; +} + +.dashboard .admin-dashboard-grid > [class*="col-"], +.dashboard .admin-dashboard-grid .row > [class*="col-"], +.dashboard .card, +.dashboard .card-header, +.dashboard .card-body, +#content-related, +#recent-actions-module, +.timeline, +.timeline-item, +.content-wrapper .content, +.content-wrapper .container-fluid { + min-width: 0; +} + +.dashboard .admin-dashboard-grid { + row-gap: 1rem; + margin-left: 0; + margin-right: 0; + padding-right: 1rem; +} + +.dashboard .card, +.dashboard .module, +#recent-actions-module { + overflow: hidden; +} + +.dashboard .admin-dashboard-grid > [class*="col-"] { + padding-left: 0.6rem; + padding-right: 0.6rem; +} + +.dashboard #content-related { + padding-right: 0.45rem; +} + +.dashboard .card-header h5, +.dashboard .timeline-item, +.dashboard .timeline-header, +.dashboard .table td, +.dashboard .table th { + overflow-wrap: anywhere; +} + +.main-header .navbar-nav, +.main-header .form-inline, +.main-header .input-group { + min-width: 0; +} + +.main-header .form-inline { + flex: 0 1 auto; +} + +.main-header .form-control-navbar { + width: clamp(8rem, 12vw, 12rem); +} + +@media (max-width: 1360px) { + .main-header .form-inline.ml-3 { + margin-left: 0.5rem !important; + } + + .main-header .form-control-navbar { + width: clamp(7rem, 10vw, 10rem); + } +} + +@media (max-width: 991.98px) { + .dashboard .admin-dashboard-grid { + padding-right: 0.15rem; + } + + .dashboard .admin-dashboard-grid > [class*="col-"] { + padding-left: 0.15rem; + padding-right: 0.15rem; + } + + .dashboard #content-related { + padding-right: 0; + } + + .content-wrapper > .content { + padding: 0.85rem 0.7rem 1.1rem; + } + + .content-wrapper > .content > .container-fluid { + padding-left: 0.1rem; + padding-right: 0.1rem; + } + + .main-header.navbar { + flex-wrap: wrap; + row-gap: 0.5rem; + } + + .main-header .form-inline { + width: 100%; + margin-left: 0 !important; + } + + .main-header .input-group, + .main-header .form-control-navbar { + width: 100%; + } +} diff --git a/src/templates/admin/index.html b/src/templates/admin/index.html new file mode 100644 index 0000000..6ffc3a7 --- /dev/null +++ b/src/templates/admin/index.html @@ -0,0 +1,133 @@ +{% extends "admin/base_site.html" %} +{% load i18n static jazzmin %} +{% get_jazzmin_ui_tweaks as jazzmin_ui %} + +{% block bodyclass %}{{ block.super }} dashboard{% endblock %} + +{% block content_title %} {% trans 'Dashboard' %} {% endblock %} + +{% block breadcrumbs %} +
    +
  1. {% trans 'Home' %}
    2. +
  2. {% trans 'Dashboard' %}
    3. +
+{% endblock %} + +{% block content %} + {% get_side_menu using="app_list" as dashboard_list %} + {% if dashboard_list %} + {% widthratio dashboard_list|length 2 1 as middle %} + {% endif %} + +
+
+
+
+ {% for app in dashboard_list %} +
+
+
{{ app.name }}
+
+
+ + + {% for model in app.models %} + + + + + {% endfor %} + +
+ {% if model.url %}{{ model.name }}{% else %}{{ model.name }}{% endif %} + +
+ {% if model.add_url %} + {% trans 'Add' %} + {% endif %} + {% if model.url %} + {% if model.view_only %} + {% trans 'View' %} + {% else %} + {% if model.custom %}{% trans 'Go' %}{% else %}{% trans 'Change' %}{% endif %} + {% endif %} + {% endif %} +
+
+
+
+ + {% if forloop.counter == middle|add:"0" %} +
+
+ {% endif %} + + {% endfor %} +
+
+
+ +
+
+
+

{% trans 'Recent actions' %}

+ {% load log %} + {% get_admin_log 6 as admin_log for_user user %} + {% if not admin_log %} +

{% trans 'None available' %}

+ {% else %} +
+ {% for entry in admin_log %} +
+ {% if entry.is_change %} + + {% elif entry.is_deletion %} + + {% elif entry.is_addition %} + + {% endif %} + +
+ {% blocktrans with timesince=entry.action_time|timesince %}{{ timesince }} ago{% endblocktrans %} +

+ {% if entry.is_deletion or not entry.get_admin_url %} + {{ entry.object_repr }} + {% else %} + {{ entry.object_repr }} + {% endif %} + + {% if entry.model %} + + {% filter capfirst %} + {{ entry.model }} + {% endfilter %} + + {% endif %} +

+ {% if not entry.is_deletion %} +
+ {% if entry.is_addition %} + {{ entry }} + {% else %} +
    + {% action_message_to_list entry as action_message_list %} + {% for action_message in action_message_list %} +
  • {{ action_message.msg|style_bold_first_word }}
    • + {% endfor %} +
+ {% endif %} +
+ {% endif %} +
+
+ {% endfor %} +
+ +
+
+ {% endif %} +
+
+
+
+{% endblock %} diff --git a/src/templates/admin/parsers/financialreport/change_list.html b/src/templates/admin/parsers/financialreport/change_list.html new file mode 100644 index 0000000..6884e20 --- /dev/null +++ b/src/templates/admin/parsers/financialreport/change_list.html @@ -0,0 +1,11 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +
  • + Загрузить Excel ФНС +
    • +
  • + Загрузить ZIP ФНС +
    • + {{ block.super }} +{% endblock %} diff --git a/src/templates/admin/parsers/financialreport/upload_excel.html b/src/templates/admin/parsers/financialreport/upload_excel.html new file mode 100644 index 0000000..d59c938 --- /dev/null +++ b/src/templates/admin/parsers/financialreport/upload_excel.html @@ -0,0 +1,36 @@ +{% extends "admin/base_site.html" %} + +{% block breadcrumbs %} +
    + Главная + › Финансовые отчеты ФНС + › Загрузка Excel +
    +{% endblock %} + +{% block content %} +
    +
    + {% csrf_token %} +
    +
    + + +

    Ожидаемый формат имени: fin_{id}_{ogrn}.xlsx

    +
    +
    + +
    + + Отмена +
    +
    +
    +{% endblock %} diff --git a/src/templates/admin/parsers/financialreport/upload_zip.html b/src/templates/admin/parsers/financialreport/upload_zip.html new file mode 100644 index 0000000..b274dfb --- /dev/null +++ b/src/templates/admin/parsers/financialreport/upload_zip.html @@ -0,0 +1,37 @@ +{% extends "admin/base_site.html" %} + +{% block breadcrumbs %} +
    + Главная + › Финансовые отчеты ФНС + › Загрузка ZIP +
    +{% endblock %} + +{% block content %} +
    +
    + {% csrf_token %} +
    +
    + + +

    + Архив должен содержать файлы вида fin_{id}_{ogrn}.xlsx в корне архива. +

    +
    +
    + +
    + + Отмена +
    +
    +
    +{% endblock %} diff --git a/src/templates/admin/registers/registerupload/change_list.html b/src/templates/admin/registers/registerupload/change_list.html new file mode 100644 index 0000000..592d0ff --- /dev/null +++ b/src/templates/admin/registers/registerupload/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +
  • + Загрузить Excel организаций +
    • + {{ block.super }} +{% endblock %} diff --git a/src/templates/admin/registers/registerupload/upload_excel.html b/src/templates/admin/registers/registerupload/upload_excel.html new file mode 100644 index 0000000..31ff2f1 --- /dev/null +++ b/src/templates/admin/registers/registerupload/upload_excel.html @@ -0,0 +1,47 @@ +{% extends "admin/base_site.html" %} + +{% block breadcrumbs %} +
    + Главная + › Загрузки реестров + › Загрузка Excel +
    +{% endblock %} + +{% block content %} +
    +
    + {% csrf_token %} +
    +
    + + +
    + +
    + + +

    Если не указана, будет использована текущая дата.

    +
    + +
    + + +

    + Ожидаемые колонки: pn_name, mn_ogrn, mn_inn, mn_okpo. Опционально: in_kpp. +

    +
    +
    + +
    + + Отмена +
    +
    +
    +{% endblock %} diff --git a/tests/apps/core/test_celery_module.py b/tests/apps/core/test_celery_module.py index cae69e0..8671906 100644 --- a/tests/apps/core/test_celery_module.py +++ b/tests/apps/core/test_celery_module.py @@ -46,7 +46,9 @@ class CeleryModuleTest(SimpleTestCase): ) app_mock.autodiscover_tasks.assert_called_once_with() self.assertEqual(module.app, app_mock) - self.assertIn("parse-industrial-production-daily", module.app.conf.beat_schedule) + self.assertIn( + "parse-industrial-production-daily", module.app.conf.beat_schedule + ) self.assertIn("parse-manufactures-daily", module.app.conf.beat_schedule) self.assertIn("parse-industrial-products-daily", module.app.conf.beat_schedule) self.assertIn("parse-inspections-weekly", module.app.conf.beat_schedule) @@ -63,12 +65,16 @@ class CeleryModuleTest(SimpleTestCase): apply_async=apply_async_mock ) - with patch.object(module, "settings", SimpleNamespace( - CELERY_STARTUP_REFRESH_ENABLED=True, - CELERY_STARTUP_REFRESH_LOCK_KEY="startup-lock", - CELERY_STARTUP_REFRESH_LOCK_TTL_SECONDS=120, - CELERY_STARTUP_REFRESH_DELAY_SECONDS=45, - )), patch.object(module.cache, "add", return_value=True) as add_mock, patch.dict( + with patch.object( + module, + "settings", + SimpleNamespace( + CELERY_STARTUP_REFRESH_ENABLED=True, + CELERY_STARTUP_REFRESH_LOCK_KEY="startup-lock", + CELERY_STARTUP_REFRESH_LOCK_TTL_SECONDS=120, + CELERY_STARTUP_REFRESH_DELAY_SECONDS=45, + ), + ), patch.object(module.cache, "add", return_value=True) as add_mock, patch.dict( sys.modules, {"apps.parsers.tasks": fake_tasks_module} ): module._queue_startup_sources_refresh() @@ -88,12 +94,16 @@ class CeleryModuleTest(SimpleTestCase): apply_async=apply_async_mock ) - with patch.object(module, "settings", SimpleNamespace( - CELERY_STARTUP_REFRESH_ENABLED=True, - CELERY_STARTUP_REFRESH_LOCK_KEY="startup-lock", - CELERY_STARTUP_REFRESH_LOCK_TTL_SECONDS=120, - CELERY_STARTUP_REFRESH_DELAY_SECONDS=45, - )), patch.object(module.cache, "add", return_value=False), patch.dict( + with patch.object( + module, + "settings", + SimpleNamespace( + CELERY_STARTUP_REFRESH_ENABLED=True, + CELERY_STARTUP_REFRESH_LOCK_KEY="startup-lock", + CELERY_STARTUP_REFRESH_LOCK_TTL_SECONDS=120, + CELERY_STARTUP_REFRESH_DELAY_SECONDS=45, + ), + ), patch.object(module.cache, "add", return_value=False), patch.dict( sys.modules, {"apps.parsers.tasks": fake_tasks_module} ): module._queue_startup_sources_refresh() diff --git a/tests/apps/parsers/test_admin.py b/tests/apps/parsers/test_admin.py index 3e0892d..ec9b883 100644 --- a/tests/apps/parsers/test_admin.py +++ b/tests/apps/parsers/test_admin.py @@ -1,5 +1,10 @@ """Tests for parsers admin configurations.""" +import io +import os +import tempfile +import zipfile + from apps.parsers.admin import ( FinancialReportAdmin, HasCertificateNumberFilter, @@ -22,7 +27,9 @@ from apps.parsers.models import ( ) from django.contrib.admin.sites import AdminSite from django.contrib.messages.storage.fallback import FallbackStorage -from django.test import RequestFactory, TestCase +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory, TestCase, override_settings +from openpyxl import Workbook from tests.apps.parsers.factories import ( IndustrialCertificateRecordFactory, @@ -42,6 +49,41 @@ def _digits(length: int) -> str: return "".join(str(fake.random_int(0, 9)) for _ in range(length)) +def _build_fns_excel_bytes() -> bytes: + workbook = Workbook() + worksheet = workbook.active + year = fake.random_int(min=2020, max=2025) + worksheet.append(["Форма №1", None, year, None]) + worksheet.append([None, "Код", "Начало", "Конец"]) + worksheet.append( + [ + fake.word(), + _digits(4), + fake.random_int(min=10, max=999), + fake.random_int(min=10, max=999), + ] + ) + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + return buffer.getvalue() + + +def _build_fns_zip_upload() -> SimpleUploadedFile: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr( + f"fin_{_digits(5)}_{_digits(13)}.xlsx", + _build_fns_excel_bytes(), + ) + archive.writestr("ignored.txt", b"invalid") + return SimpleUploadedFile( + "fin_ropk.zip", + buffer.getvalue(), + content_type="application/zip", + ) + + class ParsersAdminTest(TestCase): def setUp(self): self.site = AdminSite() @@ -55,6 +97,13 @@ class ParsersAdminTest(TestCase): request._messages = FallbackStorage(request) return request + def _post_request(self, path, data): + request = self.factory.post(path, data=data) + request.user = self.user + request.session = {} + request._messages = FallbackStorage(request) + return request + def test_proxy_admin_actions(self): admin = ProxyAdmin(Proxy, self.site) proxy = ProxyFactory(is_active=False, fail_count=5) @@ -244,3 +293,27 @@ class ParsersAdminTest(TestCase): ) self.assertEqual(admin.lines_count(report), 1) self.assertIn("span", str(admin.status_badge(report))) + self.assertIn("registry_organization", admin.list_display) + self.assertIn("registry_organization__pn_name", admin.search_fields) + route_names = [route.name for route in admin.get_urls()] + self.assertIn("parsers_financialreport_upload_excel", route_names) + self.assertIn("parsers_financialreport_upload_zip", route_names) + + def test_financial_report_admin_upload_zip_view(self): + admin = FinancialReportAdmin(FinancialReport, self.site) + archive_upload = _build_fns_zip_upload() + request = self._post_request( + "/admin/parsers/financialreport/upload-zip/", + {"file": archive_upload}, + ) + + with tempfile.TemporaryDirectory() as tmpdir, override_settings( + FNS_WATCH_DIRECTORY=os.path.join(tmpdir, "watch"), + FNS_PROCESSED_DIRECTORY=os.path.join(tmpdir, "processed"), + FNS_FAILED_DIRECTORY=os.path.join(tmpdir, "failed"), + ): + response = admin.upload_zip_view(request) + + self.assertEqual(response.status_code, 302) + self.assertEqual(FinancialReport.objects.count(), 1) + self.assertEqual(FinancialReportLine.objects.count(), 1) diff --git a/tests/apps/parsers/test_fns_upload.py b/tests/apps/parsers/test_fns_upload.py index 5462c65..820480f 100644 --- a/tests/apps/parsers/test_fns_upload.py +++ b/tests/apps/parsers/test_fns_upload.py @@ -4,9 +4,11 @@ import io import os import tempfile import time +import zipfile from unittest.mock import patch from apps.core.models import BackgroundJob +from apps.parsers.fns_upload import FNSUploadService from apps.parsers.models import FinancialReport, FinancialReportLine from django.core.files.uploadedfile import SimpleUploadedFile from django.test import override_settings @@ -38,6 +40,14 @@ def _build_fns_excel_bytes() -> bytes: return buf.getvalue() +def _build_fns_zip_bytes(file_map: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for file_name, content in file_map.items(): + archive.writestr(file_name, content) + return buffer.getvalue() + + class FNSUploadIntegrationTest(APITestCase): """Tests real upload + processing of FNS files.""" @@ -348,7 +358,7 @@ class FNSUploadIntegrationTest(APITestCase): FNS_WATCH_DIRECTORY=watch_dir, FNS_PROCESSED_DIRECTORY=processed_dir, FNS_FAILED_DIRECTORY=failed_dir, - ), patch("apps.parsers.views.Path.touch", side_effect=FileExistsError): + ), patch("apps.parsers.fns_upload.Path.touch", side_effect=FileExistsError): response = self.client.post( self.upload_url, {"files": [upload]}, @@ -375,7 +385,10 @@ class FNSUploadIntegrationTest(APITestCase): FNS_WATCH_DIRECTORY=watch_dir, FNS_PROCESSED_DIRECTORY=processed_dir, FNS_FAILED_DIRECTORY=failed_dir, - ), patch("apps.parsers.views.open", side_effect=OSError("disk full")): + ), patch( + "apps.parsers.fns_upload.Path.write_bytes", + side_effect=OSError("disk full"), + ): response = self.client.post( self.upload_url, {"files": [upload]}, @@ -406,9 +419,9 @@ class FNSUploadIntegrationTest(APITestCase): FNS_PROCESSED_DIRECTORY=processed_dir, FNS_FAILED_DIRECTORY=failed_dir, ), patch( - "apps.parsers.views.uuid.uuid4", return_value="job-task-id" + "apps.parsers.fns_upload.uuid.uuid4", return_value="job-task-id" ), patch( - "apps.parsers.views.process_fns_file.apply_async", + "apps.parsers.fns_upload.process_fns_file.apply_async", side_effect=RuntimeError("queue down"), ): response = self.client.post( @@ -426,3 +439,54 @@ class FNSUploadIntegrationTest(APITestCase): self.assertFalse( os.path.exists(os.path.join(watch_dir, f"{filename}.lock")) ) + + def test_queue_zip_archive_processes_valid_files_and_skips_invalid(self): + first_name = f"fin_{_digits(5)}_{_digits(13)}.xlsx" + second_name = f"fin_{_digits(5)}_{_digits(13)}.xlsx" + zip_content = _build_fns_zip_bytes( + { + first_name: _build_fns_excel_bytes(), + second_name: _build_fns_excel_bytes(), + "nested/fin_0000001_1234567890123.xlsx": _build_fns_excel_bytes(), + "readme.txt": b"invalid", + } + ) + archive_upload = SimpleUploadedFile( + "fin_ropk.zip", + zip_content, + content_type="application/zip", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ): + result = FNSUploadService.queue_zip_archive( + archive_file=archive_upload, + requested_by_id=self.admin.id, + ) + + self.assertEqual(result.queued, 2) + self.assertEqual(result.skipped, 0) + self.assertEqual(result.invalid, 2) + self.assertEqual(FinancialReport.objects.count(), 2) + self.assertEqual(FinancialReportLine.objects.count(), 2) + + def test_queue_zip_archive_rejects_bad_zip(self): + archive_upload = SimpleUploadedFile( + "fin_ropk.zip", + b"not-a-zip", + content_type="application/zip", + ) + + with self.assertRaisesMessage( + ValueError, + "Загруженный файл не является корректным ZIP архивом", + ): + FNSUploadService.queue_zip_archive( + archive_file=archive_upload, + requested_by_id=self.admin.id, + ) diff --git a/tests/apps/registers/test_admin.py b/tests/apps/registers/test_admin.py new file mode 100644 index 0000000..2c73c9a --- /dev/null +++ b/tests/apps/registers/test_admin.py @@ -0,0 +1,102 @@ +"""Tests for registers admin configuration.""" + +import io + +from apps.registers.admin import RegisterUploadAdmin +from apps.registers.models import Organization, RegisterUpload, RegistryMembershipPeriod +from django.contrib.admin.sites import AdminSite +from django.contrib.messages.storage.fallback import FallbackStorage +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import RequestFactory, TestCase +from openpyxl import Workbook + +from tests.apps.registers.factories import RegisterFactory +from tests.apps.user.factories import UserFactory + + +def _build_register_excel_upload(filename: str = "registry.xlsx") -> SimpleUploadedFile: + workbook = Workbook() + worksheet = workbook.active + worksheet.append(["pn_name", "mn_ogrn", "mn_inn", "in_kpp", "mn_okpo"]) + worksheet.append( + [ + 'АО "Тестовая организация"', + 1027600980990, + 7601000086, + 760401001, + "07506197", + ] + ) + + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + + return SimpleUploadedFile( + filename, + buffer.getvalue(), + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + + +class RegistersAdminTest(TestCase): + def setUp(self): + self.site = AdminSite() + self.factory = RequestFactory() + self.user = UserFactory.create_superuser() + + def _post_request(self, data): + request = self.factory.post( + "/admin/registers/registerupload/upload-excel/", + data=data, + ) + request.user = self.user + request.session = {} + request._messages = FallbackStorage(request) + return request + + def test_register_upload_admin_has_custom_upload_route(self): + admin = RegisterUploadAdmin(RegisterUpload, self.site) + route_names = [route.name for route in admin.get_urls()] + + self.assertIn("registers_registerupload_upload_excel", route_names) + + def test_register_upload_admin_upload_excel_success(self): + admin = RegisterUploadAdmin(RegisterUpload, self.site) + registry = RegisterFactory() + uploaded_file = _build_register_excel_upload() + request = self._post_request( + { + "registry": str(registry.id), + "actual_date": "2026-03-20", + "file": uploaded_file, + } + ) + + response = admin.upload_excel_view(request) + + self.assertEqual(response.status_code, 302) + self.assertEqual(RegisterUpload.objects.count(), 1) + self.assertEqual(Organization.objects.count(), 1) + self.assertEqual(RegistryMembershipPeriod.objects.count(), 1) + + upload = RegisterUpload.objects.first() + self.assertEqual(upload.registry, registry) + self.assertEqual(upload.actual_date.isoformat(), "2026-03-20") + + def test_register_upload_admin_upload_excel_invalid_extension(self): + admin = RegisterUploadAdmin(RegisterUpload, self.site) + registry = RegisterFactory() + invalid_file = SimpleUploadedFile("registry.txt", b"text/plain") + request = self._post_request( + { + "registry": str(registry.id), + "actual_date": "2026-03-20", + "file": invalid_file, + } + ) + + response = admin.upload_excel_view(request) + + self.assertEqual(response.status_code, 302) + self.assertEqual(RegisterUpload.objects.count(), 0)