"""
Admin configuration for parsers app.
"""
import hashlib
import time
import uuid
from pathlib import Path
from apps.core.models import BackgroundJob
from apps.core.services import BackgroundJobService
from apps.parsers.models import (
FinancialReport,
FinancialReportLine,
IndustrialCertificateRecord,
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParserLoadLog,
ProcurementRecord,
Proxy,
)
from apps.parsers.serializers import FNSFileUploadSerializer
from apps.parsers.services import FNSReportService
from apps.parsers.tasks import process_fns_file
from django.conf import settings
from django.contrib import admin, messages
from django.shortcuts import redirect
from django.template.response import TemplateResponse
from django.urls import path, reverse
from django.utils.html import format_html
@admin.register(Proxy)
class ProxyAdmin(admin.ModelAdmin):
"""Admin для прокси-серверов."""
list_display = [
"address",
"is_active_badge",
"fail_count",
"last_used_at",
"created_at",
]
list_filter = ["is_active", "created_at"]
search_fields = ["address"]
readonly_fields = ["created_at", "updated_at", "last_used_at"]
ordering = ["-is_active", "-last_used_at"]
list_per_page = 50
fieldsets = (
("Основное", {"fields": ("address", "is_active")}),
("Статистика", {"fields": ("fail_count", "last_used_at")}),
("Даты", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
def is_active_badge(self, obj):
"""Цветной бейдж активности."""
if obj.is_active:
return format_html(
'Активен'
)
return format_html(
'Неактивен'
)
is_active_badge.short_description = "Статус"
is_active_badge.admin_order_field = "is_active"
actions = ["activate_proxies", "deactivate_proxies", "reset_fail_count"]
@admin.action(description="Активировать выбранные прокси")
def activate_proxies(self, request, queryset):
updated = queryset.update(is_active=True)
self.message_user(request, f"Активировано {updated} прокси")
@admin.action(description="Деактивировать выбранные прокси")
def deactivate_proxies(self, request, queryset):
updated = queryset.update(is_active=False)
self.message_user(request, f"Деактивировано {updated} прокси")
@admin.action(description="Сбросить счётчик ошибок")
def reset_fail_count(self, request, queryset):
updated = queryset.update(fail_count=0)
self.message_user(request, f"Сброшен счётчик для {updated} прокси")
@admin.register(ParserLoadLog)
class ParserLoadLogAdmin(admin.ModelAdmin):
"""Admin для логов загрузки."""
list_display = [
"id",
"source",
"batch_id",
"status_badge",
"records_count",
"created_at",
]
list_filter = ["source", "status", "created_at"]
search_fields = ["batch_id", "error_message"]
readonly_fields = ["created_at", "updated_at"]
ordering = ["-created_at"]
list_per_page = 50
date_hierarchy = "created_at"
fieldsets = (
("Основное", {"fields": ("source", "batch_id", "status")}),
("Результат", {"fields": ("records_count", "error_message")}),
("Даты", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}),
)
def status_badge(self, obj):
"""Цветной бейдж статуса."""
colors = {
"success": "#28a745",
"failed": "#dc3545",
"in_progress": "#ffc107",
"pending": "#6c757d",
}
color = colors.get(obj.status, "#6c757d")
return format_html(
'{}',
color,
obj.get_status_display()
if hasattr(obj, "get_status_display")
else obj.status,
)
status_badge.short_description = "Статус"
status_badge.admin_order_field = "status"
def has_add_permission(self, request):
"""Запретить создание логов вручную."""
return False
class HasCertificateNumberFilter(admin.SimpleListFilter):
"""Фильтр по наличию номера сертификата."""
title = "Номер сертификата"
parameter_name = "has_cert_number"
def lookups(self, request, model_admin):
return [
("yes", "С номером"),
("no", "Без номера"),
]
def queryset(self, request, queryset):
if self.value() == "yes":
return queryset.exclude(certificate_number__in=["-", ""])
if self.value() == "no":
return queryset.filter(certificate_number__in=["-", ""])
return queryset
@admin.register(IndustrialCertificateRecord)
class IndustrialCertificateRecordAdmin(admin.ModelAdmin):
"""Admin для сертификатов промышленного производства."""
list_display = [
"certificate_number",
"organisation_name_short",
"inn",
"ogrn",
"issue_date",
"expiry_date",
"load_batch",
]
list_filter = [HasCertificateNumberFilter, "load_batch", "created_at"]
search_fields = [
"certificate_number",
"organisation_name",
"inn",
"ogrn",
]
readonly_fields = ["created_at", "updated_at", "load_batch"]
ordering = ["-created_at"]
list_per_page = 100
date_hierarchy = "created_at"
raw_id_fields = []
fieldsets = (
(
"Сертификат",
{"fields": ("certificate_number", "issue_date", "expiry_date")},
),
(
"Организация",
{"fields": ("organisation_name", "inn", "ogrn")},
),
(
"Документ",
{"fields": ("certificate_file_url",), "classes": ("collapse",)},
),
(
"Системное",
{
"fields": ("load_batch", "created_at", "updated_at"),
"classes": ("collapse",),
},
),
)
def organisation_name_short(self, obj):
"""Сокращённое название организации."""
name = obj.organisation_name or ""
return name[:60] + "..." if len(name) > 60 else name
organisation_name_short.short_description = "Организация"
organisation_name_short.admin_order_field = "organisation_name"
def has_add_permission(self, request):
"""Запретить создание записей вручную."""
return False
def has_change_permission(self, request, obj=None):
"""Запретить редактирование записей."""
return False
@admin.register(ManufacturerRecord)
class ManufacturerRecordAdmin(admin.ModelAdmin):
"""Admin для реестра производителей."""
list_display = [
"full_legal_name_short",
"inn",
"ogrn",
"address_short",
"load_batch",
"created_at",
]
list_filter = ["load_batch", "created_at"]
search_fields = [
"full_legal_name",
"inn",
"ogrn",
"address",
]
readonly_fields = ["created_at", "updated_at", "load_batch"]
ordering = ["-created_at"]
list_per_page = 100
date_hierarchy = "created_at"
fieldsets = (
(
"Организация",
{"fields": ("full_legal_name", "inn", "ogrn")},
),
(
"Адрес",
{"fields": ("address",)},
),
(
"Системное",
{
"fields": ("load_batch", "created_at", "updated_at"),
"classes": ("collapse",),
},
),
)
def full_legal_name_short(self, obj):
"""Сокращённое название."""
name = obj.full_legal_name or ""
return name[:60] + "..." if len(name) > 60 else name
full_legal_name_short.short_description = "Название"
full_legal_name_short.admin_order_field = "full_legal_name"
def address_short(self, obj):
"""Сокращённый адрес."""
addr = obj.address or ""
return addr[:40] + "..." if len(addr) > 40 else addr
address_short.short_description = "Адрес"
address_short.admin_order_field = "address"
def has_add_permission(self, request):
"""Запретить создание записей вручную."""
return False
def has_change_permission(self, request, obj=None):
"""Запретить редактирование записей."""
return False
@admin.register(IndustrialProductRecord)
class IndustrialProductRecordAdmin(admin.ModelAdmin):
"""Admin для реестра промышленной продукции."""
list_display = [
"registry_number",
"product_name_short",
"organisation_name_short",
"inn",
"ogrn",
"load_batch",
"created_at",
]
list_filter = ["load_batch", "created_at"]
search_fields = [
"registry_number",
"product_name",
"full_organisation_name",
"inn",
"ogrn",
"okpd2_code",
"tnved_code",
]
readonly_fields = ["created_at", "updated_at", "load_batch"]
ordering = ["-created_at"]
list_per_page = 100
date_hierarchy = "created_at"
fieldsets = (
(
"Запись реестра",
{
"fields": (
"registry_number",
"product_name",
"product_model",
"regulatory_document",
)
},
),
(
"Организация",
{"fields": ("full_organisation_name", "inn", "ogrn")},
),
(
"Классификаторы",
{"fields": ("okpd2_code", "tnved_code")},
),
(
"Системное",
{
"fields": ("load_batch", "created_at", "updated_at"),
"classes": ("collapse",),
},
),
)
def product_name_short(self, obj):
"""Сокращённое название продукции."""
name = obj.product_name or ""
return name[:60] + "..." if len(name) > 60 else name
def organisation_name_short(self, obj):
"""Сокращённое название организации."""
name = obj.full_organisation_name or ""
return name[:60] + "..." if len(name) > 60 else name
product_name_short.short_description = "Продукция"
product_name_short.admin_order_field = "product_name"
organisation_name_short.short_description = "Организация"
organisation_name_short.admin_order_field = "full_organisation_name"
def has_add_permission(self, request):
"""Запретить создание записей вручную."""
return False
def has_change_permission(self, request, obj=None):
"""Запретить редактирование записей."""
return False
@admin.register(InspectionRecord)
class InspectionRecordAdmin(admin.ModelAdmin):
"""Admin для проверок из Единого реестра проверок."""
list_display = [
"registration_number",
"organisation_name_short",
"inn",
"control_authority_short",
"inspection_type",
"status_badge",
"start_date",
"load_batch",
]
list_filter = [
"inspection_type",
"inspection_form",
"status",
"load_batch",
"created_at",
]
search_fields = [
"registration_number",
"organisation_name",
"inn",
"ogrn",
"control_authority",
]
readonly_fields = ["created_at", "updated_at", "load_batch"]
ordering = ["-created_at"]
list_per_page = 100
date_hierarchy = "created_at"
fieldsets = (
(
"Проверка",
{
"fields": (
"registration_number",
"inspection_type",
"inspection_form",
"status",
)
},
),
(
"Организация",
{"fields": ("organisation_name", "inn", "ogrn")},
),
(
"Контрольный орган",
{"fields": ("control_authority", "legal_basis")},
),
(
"Сроки и результат",
{"fields": ("start_date", "end_date", "result")},
),
(
"Системное",
{
"fields": ("load_batch", "created_at", "updated_at"),
"classes": ("collapse",),
},
),
)
def organisation_name_short(self, obj):
"""Сокращённое название организации."""
name = obj.organisation_name or ""
return name[:50] + "..." if len(name) > 50 else name
organisation_name_short.short_description = "Организация"
organisation_name_short.admin_order_field = "organisation_name"
def control_authority_short(self, obj):
"""Сокращённое название контрольного органа."""
name = obj.control_authority or ""
return name[:30] + "..." if len(name) > 30 else name
control_authority_short.short_description = "Контр. орган"
control_authority_short.admin_order_field = "control_authority"
def status_badge(self, obj):
"""Цветной бейдж статуса."""
status = obj.status or ""
status_lower = status.lower()
if "завершен" in status_lower:
color = "#28a745"
elif "процесс" in status_lower or "проведен" in status_lower:
color = "#ffc107"
elif "отменен" in status_lower or "прекращ" in status_lower:
color = "#dc3545"
else:
color = "#6c757d"
return format_html(
'{}',
color,
status[:20] if len(status) > 20 else status,
)
status_badge.short_description = "Статус"
status_badge.admin_order_field = "status"
def has_add_permission(self, request):
"""Запретить создание записей вручную."""
return False
def has_change_permission(self, request, obj=None):
"""Запретить редактирование записей."""
return False
@admin.register(ProcurementRecord)
class ProcurementRecordAdmin(admin.ModelAdmin):
"""Admin для государственных закупок."""
list_display = [
"purchase_number",
"purchase_name_short",
"customer_inn",
"customer_name_short",
"max_price",
"law_type",
"status_badge",
"publish_date",
"load_batch",
]
list_filter = [
"law_type",
"status",
"region_code",
"load_batch",
"created_at",
]
search_fields = [
"purchase_number",
"purchase_name",
"customer_inn",
"customer_ogrn",
"customer_name",
]
readonly_fields = ["created_at", "updated_at", "load_batch"]
ordering = ["-created_at"]
list_per_page = 100
date_hierarchy = "created_at"
fieldsets = (
(
"Закупка",
{
"fields": (
"purchase_number",
"purchase_name",
"purchase_object_info",
"law_type",
"status",
)
},
),
(
"Заказчик",
{
"fields": (
"customer_name",
"customer_inn",
"customer_kpp",
"customer_ogrn",
)
},
),
(
"Финансы",
{"fields": ("max_price", "currency_code", "placement_method")},
),
(
"Сроки",
{"fields": ("publish_date", "end_date")},
),
(
"Дополнительно",
{"fields": ("region_code", "href"), "classes": ("collapse",)},
),
(
"Системное",
{
"fields": (
"load_batch",
"data_year",
"data_month",
"created_at",
"updated_at",
),
"classes": ("collapse",),
},
),
)
def purchase_name_short(self, obj):
"""Сокращённое наименование закупки."""
name = obj.purchase_name or ""
return name[:50] + "..." if len(name) > 50 else name
purchase_name_short.short_description = "Наименование"
purchase_name_short.admin_order_field = "purchase_name"
def customer_name_short(self, obj):
"""Сокращённое наименование заказчика."""
name = obj.customer_name or ""
return name[:30] + "..." if len(name) > 30 else name
customer_name_short.short_description = "Заказчик"
customer_name_short.admin_order_field = "customer_name"
def status_badge(self, obj):
"""Цветной бейдж статуса."""
status = obj.status or ""
status_lower = status.lower()
if "опублик" in status_lower or "подача" in status_lower:
color = "#28a745"
elif "завершен" in status_lower or "состоял" in status_lower:
color = "#17a2b8"
elif "отменен" in status_lower or "не состоял" in status_lower:
color = "#dc3545"
else:
color = "#6c757d"
return format_html(
'{}',
color,
status[:20] if len(status) > 20 else status,
)
status_badge.short_description = "Статус"
status_badge.admin_order_field = "status"
def has_add_permission(self, request):
"""Запретить создание записей вручную."""
return False
def has_change_permission(self, request, obj=None):
"""Запретить редактирование записей."""
return False
class FinancialReportLineInline(admin.TabularInline):
"""Inline для строк финансового отчета."""
model = FinancialReportLine
extra = 0
readonly_fields = [
"form_code",
"line_code",
"line_name",
"year",
"period_start",
"period_end",
]
can_delete = False
def has_add_permission(self, request, obj=None):
return False
@admin.register(FinancialReport)
class FinancialReportAdmin(admin.ModelAdmin):
"""Admin для финансовых отчетов ФНС."""
change_list_template = "admin/parsers/financialreport/change_list.html"
list_display = [
"external_id",
"ogrn",
"registry_organization",
"file_name",
"status_badge",
"source",
"lines_count",
"load_batch",
"created_at",
]
list_filter = ["status", "source", "load_batch", "registry_organization", "created_at"]
search_fields = [
"external_id",
"ogrn",
"file_name",
"registry_organization__pn_name",
"registry_organization__mn_ogrn",
"registry_organization__mn_inn",
]
list_select_related = ["registry_organization"]
readonly_fields = [
"external_id",
"ogrn",
"registry_organization",
"file_name",
"file_hash",
"load_batch",
"status",
"source",
"error_message",
"created_at",
"updated_at",
]
ordering = ["-created_at"]
list_per_page = 50
date_hierarchy = "created_at"
inlines = [FinancialReportLineInline]
fieldsets = (
(
"Основное",
{
"fields": (
"external_id",
"ogrn",
"registry_organization",
"file_name",
"file_hash",
)
},
),
(
"Статус",
{"fields": ("status", "source", "error_message")},
),
(
"Системное",
{
"fields": ("load_batch", "created_at", "updated_at"),
"classes": ("collapse",),
},
),
)
def lines_count(self, obj):
"""Количество строк в отчете."""
return obj.lines.count()
lines_count.short_description = "Строк"
def status_badge(self, obj):
"""Цветной бейдж статуса."""
colors = {
"pending": "#6c757d",
"processing": "#ffc107",
"success": "#28a745",
"failed": "#dc3545",
}
color = colors.get(obj.status, "#6c757d")
return format_html(
'{}',
color,
obj.get_status_display(),
)
status_badge.short_description = "Статус"
status_badge.admin_order_field = "status"
def get_urls(self):
urls = super().get_urls()
custom_urls = [
path(
"upload-excel/",
self.admin_site.admin_view(self.upload_excel_view),
name="parsers_financialreport_upload_excel",
),
]
return custom_urls + urls
def changelist_view(self, request, extra_context=None):
extra_context = extra_context or {}
extra_context["upload_excel_url"] = reverse(
"admin:parsers_financialreport_upload_excel"
)
return super().changelist_view(request, extra_context=extra_context)
def upload_excel_view(self, request):
changelist_url = reverse("admin:parsers_financialreport_changelist")
if request.method == "POST":
files = request.FILES.getlist("files")
serializer = FNSFileUploadSerializer(data={"files": files})
if not serializer.is_valid():
self.message_user(
request,
f"Ошибка валидации файлов: {serializer.errors}",
level=messages.ERROR,
)
return redirect(changelist_url)
try:
queued, skipped, task_ids = self._enqueue_fns_files(
request,
serializer.validated_data["files"],
)
except Exception as exc: # noqa: BLE001
self.message_user(
request,
f"Ошибка постановки файлов в очередь: {exc}",
level=messages.ERROR,
)
return redirect(changelist_url)
if queued:
self.message_user(
request,
f"Файлов поставлено в очередь: {queued}. Task IDs: {', '.join(task_ids[:5])}",
level=messages.SUCCESS,
)
if skipped:
self.message_user(
request,
f"Пропущено файлов: {skipped} (дубликаты или уже обрабатываются).",
level=messages.WARNING,
)
if not queued and not skipped:
self.message_user(
request,
"Файлы не были обработаны.",
level=messages.WARNING,
)
return redirect(changelist_url)
context = {
**self.admin_site.each_context(request),
"opts": self.model._meta,
"title": "Загрузка Excel отчетности ФНС",
"changelist_url": changelist_url,
}
return TemplateResponse(
request,
"admin/parsers/financialreport/upload_excel.html",
context,
)
@staticmethod
def _try_create_fns_lock(file_path: Path) -> bool:
lock_path = Path(f"{file_path}.lock")
if lock_path.exists():
try:
age_seconds = time.time() - lock_path.stat().st_mtime
ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600)
if age_seconds > ttl_seconds:
lock_path.unlink()
else:
return False
except FileNotFoundError:
pass
try:
lock_path.touch(exist_ok=False)
except FileExistsError:
return False
return True
def _enqueue_fns_files(self, request, files):
upload_dir = Path(settings.FNS_WATCH_DIRECTORY)
upload_dir.mkdir(parents=True, exist_ok=True)
task_ids = []
queued = 0
skipped = 0
for file in files:
file_content = file.read()
file_hash = hashlib.sha256(file_content).hexdigest()
file.seek(0)
if FNSReportService.exists_by_hash(file_hash):
skipped += 1
continue
file_path = upload_dir / file.name
if not self._try_create_fns_lock(file_path):
skipped += 1
continue
lock_path = Path(f"{file_path}.lock")
if file_path.exists():
lock_path.unlink(missing_ok=True)
skipped += 1
continue
try:
with open(file_path, "wb") as f:
for chunk in file.chunks():
f.write(chunk)
except Exception:
lock_path.unlink(missing_ok=True)
raise
task_id = str(uuid.uuid4())
try:
BackgroundJobService.create_job(
task_id=task_id,
task_name="apps.parsers.tasks.process_fns_file",
user_id=request.user.id,
meta={
"source": "fns_reports",
"file": file.name,
},
)
task = process_fns_file.apply_async(
args=[str(file_path)],
kwargs={"requested_by_id": request.user.id},
task_id=task_id,
)
except Exception:
lock_path.unlink(missing_ok=True)
BackgroundJob.objects.filter(task_id=task_id).delete()
raise
task_ids.append(task.id)
queued += 1
return queued, skipped, task_ids
def has_add_permission(self, request):
"""Запретить создание записей вручную."""
return False
def has_change_permission(self, request, obj=None):
"""Запретить редактирование записей."""
return False