965 lines
35 KiB
Python
965 lines
35 KiB
Python
"""
|
||
Views для приложения парсеров.
|
||
|
||
Все ViewSets только для чтения (GET, GET list).
|
||
Добавление и удаление данных - через парсеры и админку.
|
||
"""
|
||
|
||
import csv
|
||
import hashlib
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag
|
||
from apps.core.response import api_response
|
||
from apps.core.services import BackgroundJobService
|
||
from apps.parsers.models import (
|
||
FinancialReport,
|
||
IndustrialCertificateRecord,
|
||
IndustrialProductRecord,
|
||
InspectionRecord,
|
||
ManufacturerRecord,
|
||
ParserLoadLog,
|
||
ProcurementRecord,
|
||
Proxy,
|
||
)
|
||
from apps.parsers.serializers import (
|
||
FinancialReportDetailSerializer,
|
||
FinancialReportSerializer,
|
||
FNSFileUploadSerializer,
|
||
IndustrialCertificateSerializer,
|
||
IndustrialProductSerializer,
|
||
InspectionSerializer,
|
||
ManufacturerSerializer,
|
||
ParserLoadLogSerializer,
|
||
ProcurementSerializer,
|
||
ProxySerializer,
|
||
SourceCardDetailSerializer,
|
||
SourceCardRefreshRequestSerializer,
|
||
SourceCardRefreshResponseSerializer,
|
||
SourceCardSerializer,
|
||
SourceTaskStatusSerializer,
|
||
)
|
||
from apps.parsers.source_cards import SourceCardService
|
||
from apps.parsers.tasks import process_fns_file
|
||
from django.conf import settings
|
||
from django.db.models import CharField, Count, Q
|
||
from django.db.models.functions import Cast
|
||
from django.http import HttpResponse
|
||
from drf_yasg import openapi
|
||
from drf_yasg.utils import swagger_auto_schema
|
||
from rest_framework import status
|
||
from rest_framework.parsers import MultiPartParser
|
||
from rest_framework.permissions import IsAdminUser, IsAuthenticated
|
||
from rest_framework.response import Response
|
||
from rest_framework.views import APIView
|
||
from rest_framework.viewsets import ReadOnlyModelViewSet
|
||
|
||
# =============================================================================
|
||
# Swagger Tags (для группировки в документации)
|
||
# =============================================================================
|
||
|
||
MINPROMTORG_TAG = swagger_tag("Минпромторг", "minpromtorg")
|
||
PROVERKI_TAG = swagger_tag("Единый реестр проверок", "inspections_registry")
|
||
ZAKUPKI_TAG = swagger_tag("Государственные закупки", "public_procurements")
|
||
FNS_TAG = swagger_tag("ФНС - Бухгалтерская отчетность", "fns_financial_reports")
|
||
SOURCES_TAG = swagger_tag("Источники для фронта", "frontend_sources")
|
||
SYSTEM_TAG = swagger_tag("Системные", "system")
|
||
|
||
PARSER_LOG_ORDERING_FIELDS = {
|
||
"id",
|
||
"batch_id",
|
||
"source",
|
||
"status",
|
||
"records_count",
|
||
"created_at",
|
||
"updated_at",
|
||
}
|
||
|
||
|
||
def _get_parser_logs_queryset(*, search_query: str = ""):
|
||
queryset = ParserLoadLog.objects.all().order_by("-created_at")
|
||
search_term = search_query.strip()
|
||
if not search_term:
|
||
return queryset
|
||
|
||
return queryset.annotate(
|
||
batch_id_text=Cast("batch_id", output_field=CharField())
|
||
).filter(
|
||
Q(source__icontains=search_term)
|
||
| Q(status__icontains=search_term)
|
||
| Q(error_message__icontains=search_term)
|
||
| Q(batch_id_text__icontains=search_term)
|
||
)
|
||
|
||
|
||
def _apply_safe_ordering(queryset, ordering: str, allowed_fields: set[str]):
|
||
order_by_fields = []
|
||
for raw_field in (item.strip() for item in ordering.split(",") if item.strip()):
|
||
field_name = raw_field[1:] if raw_field.startswith("-") else raw_field
|
||
if field_name not in allowed_fields:
|
||
continue
|
||
order_by_fields.append(raw_field)
|
||
|
||
if not order_by_fields:
|
||
return queryset
|
||
|
||
return queryset.order_by(*order_by_fields)
|
||
|
||
|
||
# =============================================================================
|
||
# Минпромторг - Сертификаты промышленного производства
|
||
# =============================================================================
|
||
|
||
|
||
class IndustrialCertificateViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра сертификатов промышленного производства.
|
||
|
||
Данные загружаются из Минпромторга.
|
||
Только чтение - добавление через парсер/админку.
|
||
"""
|
||
|
||
queryset = IndustrialCertificateRecord.objects.all().order_by("-created_at")
|
||
serializer_class = IndustrialCertificateSerializer
|
||
permission_classes = [IsAuthenticated]
|
||
filterset_fields = [
|
||
"inn",
|
||
"ogrn",
|
||
"certificate_number",
|
||
"load_batch",
|
||
"registry_organization",
|
||
]
|
||
search_fields = ["organisation_name", "certificate_number", "inn", "ogrn"]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[MINPROMTORG_TAG],
|
||
operation_summary="Список сертификатов",
|
||
operation_description=(
|
||
"Возвращает список сертификатов промышленного производства.\n"
|
||
"Поддерживает фильтрацию по: inn, ogrn, certificate_number, load_batch.\n"
|
||
"Поддерживает поиск по: organisation_name, certificate_number, inn, ogrn."
|
||
),
|
||
responses={
|
||
200: IndustrialCertificateSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[MINPROMTORG_TAG],
|
||
operation_summary="Детали сертификата",
|
||
operation_description="Возвращает информацию о конкретном сертификате.",
|
||
responses={
|
||
200: IndustrialCertificateSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
# =============================================================================
|
||
# Минпромторг - Реестр производителей
|
||
# =============================================================================
|
||
|
||
|
||
class ManufacturerViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра реестра производителей.
|
||
|
||
Данные загружаются из Минпромторга.
|
||
Только чтение - добавление через парсер/админку.
|
||
"""
|
||
|
||
queryset = ManufacturerRecord.objects.all().order_by("-created_at")
|
||
serializer_class = ManufacturerSerializer
|
||
permission_classes = [IsAuthenticated]
|
||
filterset_fields = ["inn", "ogrn", "load_batch", "registry_organization"]
|
||
search_fields = ["full_legal_name", "inn", "ogrn", "address"]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[MINPROMTORG_TAG],
|
||
operation_summary="Список производителей",
|
||
operation_description=(
|
||
"Возвращает список производителей из реестра Минпромторга.\n"
|
||
"Поддерживает фильтрацию по: inn, ogrn, load_batch.\n"
|
||
"Поддерживает поиск по: full_legal_name, inn, ogrn, address."
|
||
),
|
||
responses={
|
||
200: ManufacturerSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[MINPROMTORG_TAG],
|
||
operation_summary="Детали производителя",
|
||
operation_description="Возвращает информацию о конкретном производителе.",
|
||
responses={
|
||
200: ManufacturerSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
class IndustrialProductViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра реестра промышленной продукции.
|
||
|
||
Данные загружаются из Минпромторга.
|
||
Только чтение - добавление через парсер/админку.
|
||
"""
|
||
|
||
queryset = IndustrialProductRecord.objects.all().order_by("-created_at")
|
||
serializer_class = IndustrialProductSerializer
|
||
permission_classes = [IsAuthenticated]
|
||
filterset_fields = [
|
||
"inn",
|
||
"ogrn",
|
||
"registry_number",
|
||
"load_batch",
|
||
"registry_organization",
|
||
]
|
||
search_fields = [
|
||
"full_organisation_name",
|
||
"product_name",
|
||
"product_model",
|
||
"registry_number",
|
||
"inn",
|
||
"ogrn",
|
||
]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[MINPROMTORG_TAG],
|
||
operation_summary="Список промышленной продукции",
|
||
operation_description=(
|
||
"Возвращает список записей реестра промышленной продукции "
|
||
"Минпромторга.\n"
|
||
"Поддерживает фильтрацию по: inn, ogrn, registry_number, load_batch.\n"
|
||
"Поддерживает поиск по: full_organisation_name, product_name, "
|
||
"product_model, registry_number, inn, ogrn."
|
||
),
|
||
responses={
|
||
200: IndustrialProductSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[MINPROMTORG_TAG],
|
||
operation_summary="Детали записи промышленной продукции",
|
||
operation_description=(
|
||
"Возвращает информацию о конкретной записи реестра промышленной продукции."
|
||
),
|
||
responses={
|
||
200: IndustrialProductSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
# =============================================================================
|
||
# Единый реестр проверок (proverki.gov.ru)
|
||
# =============================================================================
|
||
|
||
|
||
class InspectionViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра проверок из Единого реестра проверок.
|
||
|
||
Данные из ФГИС "Единый реестр проверок" (Генпрокуратура РФ).
|
||
Поддерживает ФЗ-294 (традиционные) и ФЗ-248 (новые с 2021).
|
||
Только чтение - добавление через парсер/админку.
|
||
"""
|
||
|
||
queryset = InspectionRecord.objects.all().order_by("-created_at")
|
||
serializer_class = InspectionSerializer
|
||
permission_classes = [IsAuthenticated]
|
||
filterset_fields = [
|
||
"inn",
|
||
"ogrn",
|
||
"registration_number",
|
||
"is_federal_law_248",
|
||
"data_year",
|
||
"data_month",
|
||
"load_batch",
|
||
"registry_organization",
|
||
]
|
||
search_fields = [
|
||
"organisation_name",
|
||
"registration_number",
|
||
"inn",
|
||
"ogrn",
|
||
"control_authority",
|
||
]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[PROVERKI_TAG],
|
||
operation_summary="Список проверок",
|
||
operation_description=(
|
||
"Возвращает список проверок из Единого реестра.\n"
|
||
"Поддерживает фильтрацию по: inn, ogrn, registration_number, "
|
||
"is_federal_law_248, data_year, data_month, load_batch.\n"
|
||
"Поддерживает поиск по: organisation_name, registration_number, "
|
||
"inn, ogrn, control_authority."
|
||
),
|
||
responses={
|
||
200: InspectionSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[PROVERKI_TAG],
|
||
operation_summary="Детали проверки",
|
||
operation_description="Возвращает информацию о конкретной проверке.",
|
||
responses={
|
||
200: InspectionSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
# =============================================================================
|
||
# Государственные закупки (zakupki.gov.ru)
|
||
# =============================================================================
|
||
|
||
|
||
class ProcurementViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра государственных закупок.
|
||
|
||
Данные из ЕИС zakupki.gov.ru.
|
||
Поддерживает 44-ФЗ и 223-ФЗ.
|
||
Только чтение - добавление через парсер/админку.
|
||
"""
|
||
|
||
queryset = ProcurementRecord.objects.all().order_by("-created_at")
|
||
serializer_class = ProcurementSerializer
|
||
permission_classes = [IsAuthenticated]
|
||
filterset_fields = [
|
||
"customer_inn",
|
||
"customer_ogrn",
|
||
"purchase_number",
|
||
"law_type",
|
||
"status",
|
||
"region_code",
|
||
"data_year",
|
||
"data_month",
|
||
"load_batch",
|
||
"registry_organization",
|
||
]
|
||
search_fields = [
|
||
"purchase_name",
|
||
"purchase_number",
|
||
"customer_name",
|
||
"customer_inn",
|
||
"customer_ogrn",
|
||
]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[ZAKUPKI_TAG],
|
||
operation_summary="Список закупок",
|
||
operation_description=(
|
||
"Возвращает список государственных закупок.\n"
|
||
"Поддерживает фильтрацию по: customer_inn, customer_ogrn, "
|
||
"purchase_number, law_type, status, region_code, "
|
||
"data_year, data_month, load_batch.\n"
|
||
"Поддерживает поиск по: purchase_name, purchase_number, "
|
||
"customer_name, customer_inn, customer_ogrn."
|
||
),
|
||
responses={
|
||
200: ProcurementSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[ZAKUPKI_TAG],
|
||
operation_summary="Детали закупки",
|
||
operation_description="Возвращает информацию о конкретной закупке.",
|
||
responses={
|
||
200: ProcurementSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
# =============================================================================
|
||
# ФНС - Бухгалтерская отчетность
|
||
# =============================================================================
|
||
|
||
|
||
class FinancialReportViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра финансовых отчетов ФНС.
|
||
|
||
Данные загружаются из Excel файлов (fin_{id}_{ogrn}.xlsx).
|
||
Только чтение - добавление через загрузку файлов.
|
||
"""
|
||
|
||
queryset = FinancialReport.objects.annotate(lines_count=Count("lines")).order_by(
|
||
"-created_at"
|
||
)
|
||
permission_classes = [IsAuthenticated]
|
||
filterset_fields = [
|
||
"ogrn",
|
||
"external_id",
|
||
"status",
|
||
"source",
|
||
"load_batch",
|
||
"registry_organization",
|
||
]
|
||
search_fields = ["ogrn", "external_id", "file_name"]
|
||
|
||
def get_serializer_class(self):
|
||
if self.action == "retrieve":
|
||
return FinancialReportDetailSerializer
|
||
return FinancialReportSerializer
|
||
|
||
@swagger_auto_schema(
|
||
tags=[FNS_TAG],
|
||
operation_summary="Список отчетов",
|
||
operation_description=(
|
||
"Возвращает список финансовых отчетов ФНС.\n"
|
||
"Поддерживает фильтрацию по: ogrn, external_id, status, "
|
||
"source, load_batch.\n"
|
||
"Поддерживает поиск по: ogrn, external_id, file_name."
|
||
),
|
||
responses={
|
||
200: FinancialReportSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[FNS_TAG],
|
||
operation_summary="Детали отчета",
|
||
operation_description=(
|
||
"Возвращает детальную информацию об отчете, "
|
||
"включая все строки бухгалтерской отчетности."
|
||
),
|
||
responses={
|
||
200: FinancialReportDetailSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
class FNSReportUploadView(APIView):
|
||
"""
|
||
API для загрузки файлов бухгалтерской отчетности ФНС.
|
||
|
||
Файлы сохраняются во временную директорию и ставятся в очередь
|
||
на обработку через Celery.
|
||
"""
|
||
|
||
parser_classes = [MultiPartParser]
|
||
permission_classes = [IsAdminUser]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[FNS_TAG],
|
||
operation_summary="Загрузка файлов",
|
||
operation_description=(
|
||
"Пакетная загрузка файлов бухгалтерской отчетности.\n\n"
|
||
"**Формат файла:** fin_{id}_{ogrn}.xlsx\n\n"
|
||
"**Ответ:**\n"
|
||
"- queued: количество файлов в очереди\n"
|
||
"- skipped: количество пропущенных (дубликаты)\n"
|
||
"- task_ids: ID задач для отслеживания статуса"
|
||
),
|
||
manual_parameters=[
|
||
openapi.Parameter(
|
||
name="files",
|
||
in_=openapi.IN_FORM,
|
||
type=openapi.TYPE_FILE,
|
||
required=True,
|
||
description="Файл(ы) для загрузки (fin_*.xlsx)",
|
||
),
|
||
],
|
||
consumes=["multipart/form-data"],
|
||
responses={
|
||
202: openapi.Response(
|
||
description="Файлы приняты в обработку",
|
||
schema=openapi.Schema(
|
||
type=openapi.TYPE_OBJECT,
|
||
properties={
|
||
"queued": openapi.Schema(
|
||
type=openapi.TYPE_INTEGER,
|
||
description="Количество файлов в очереди",
|
||
),
|
||
"skipped": openapi.Schema(
|
||
type=openapi.TYPE_INTEGER,
|
||
description="Количество пропущенных",
|
||
),
|
||
"task_ids": openapi.Schema(
|
||
type=openapi.TYPE_ARRAY,
|
||
items=openapi.Schema(type=openapi.TYPE_STRING),
|
||
description="ID задач Celery",
|
||
),
|
||
},
|
||
),
|
||
),
|
||
400: CommonResponses.BAD_REQUEST,
|
||
**ErrorResponses.ADMIN,
|
||
},
|
||
)
|
||
def post(self, request): # noqa
|
||
serializer = FNSFileUploadSerializer(data=request.data)
|
||
serializer.is_valid(raise_exception=True)
|
||
|
||
files = serializer.validated_data["files"]
|
||
task_ids = []
|
||
queued = 0
|
||
skipped = 0
|
||
|
||
# Создаём директорию для загрузки
|
||
upload_dir = Path(settings.FNS_WATCH_DIRECTORY)
|
||
upload_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
from apps.parsers.services import FNSReportService
|
||
|
||
def _try_create_fns_lock(file_path: Path) -> bool:
|
||
lock_path = Path(f"{file_path}.lock")
|
||
if lock_path.exists():
|
||
try:
|
||
age_seconds = time.time() - lock_path.stat().st_mtime
|
||
ttl_seconds = getattr(settings, "FNS_LOCK_TTL_SECONDS", 3600)
|
||
if age_seconds > ttl_seconds:
|
||
lock_path.unlink()
|
||
else:
|
||
return False
|
||
except FileNotFoundError:
|
||
pass
|
||
try:
|
||
lock_path.touch(exist_ok=False)
|
||
except FileExistsError:
|
||
return False
|
||
return True
|
||
|
||
for file in files:
|
||
# Вычисляем хеш файла
|
||
file_content = file.read()
|
||
file_hash = hashlib.sha256(file_content).hexdigest()
|
||
file.seek(0)
|
||
|
||
# Проверяем дубликат
|
||
if FNSReportService.exists_by_hash(file_hash):
|
||
skipped += 1
|
||
continue
|
||
|
||
# Сохраняем файл
|
||
file_path = upload_dir / file.name
|
||
if not _try_create_fns_lock(file_path):
|
||
skipped += 1
|
||
continue
|
||
lock_path = Path(f"{file_path}.lock")
|
||
|
||
if file_path.exists():
|
||
lock_path.unlink(missing_ok=True)
|
||
skipped += 1
|
||
continue
|
||
|
||
try:
|
||
with open(file_path, "wb") as f:
|
||
for chunk in file.chunks():
|
||
f.write(chunk)
|
||
except Exception:
|
||
lock_path.unlink(missing_ok=True)
|
||
raise
|
||
|
||
# Ставим в очередь
|
||
try:
|
||
task_id = str(uuid.uuid4())
|
||
BackgroundJobService.create_job(
|
||
task_id=task_id,
|
||
task_name="apps.parsers.tasks.process_fns_file",
|
||
user_id=request.user.id,
|
||
meta={
|
||
"source": ParserLoadLog.Source.FNS_REPORTS,
|
||
"file": file.name,
|
||
},
|
||
)
|
||
task = process_fns_file.apply_async(
|
||
args=[str(file_path)],
|
||
kwargs={"requested_by_id": request.user.id},
|
||
task_id=task_id,
|
||
)
|
||
except Exception:
|
||
lock_path.unlink(missing_ok=True)
|
||
from apps.core.models import BackgroundJob
|
||
|
||
BackgroundJob.objects.filter(task_id=task_id).delete()
|
||
raise
|
||
task_ids.append(task.id)
|
||
queued += 1
|
||
|
||
return Response(
|
||
{
|
||
"queued": queued,
|
||
"skipped": skipped,
|
||
"task_ids": task_ids,
|
||
},
|
||
status=status.HTTP_202_ACCEPTED,
|
||
)
|
||
|
||
|
||
# =============================================================================
|
||
# Frontend-oriented source cards
|
||
# =============================================================================
|
||
|
||
|
||
class SourceCardListView(APIView):
|
||
"""List of aggregated source cards for frontend pages."""
|
||
|
||
permission_classes = [IsAuthenticated]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SOURCES_TAG],
|
||
operation_summary="Список карточек источников",
|
||
operation_description=(
|
||
"Возвращает агрегированный список карточек источников данных "
|
||
"для фронтенда."
|
||
),
|
||
responses={
|
||
200: SourceCardSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def get(self, request):
|
||
cards = SourceCardService.list_cards()
|
||
serializer = SourceCardSerializer(cards, many=True)
|
||
return api_response(serializer.data)
|
||
|
||
|
||
class SourceTaskStatusListView(APIView):
|
||
"""Tabular list of parsing source statuses for frontend screens."""
|
||
|
||
permission_classes = [IsAuthenticated]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SOURCES_TAG],
|
||
operation_summary="Статусы задач парсинга",
|
||
operation_description=(
|
||
"Возвращает табличный список статусов источников данных для экрана "
|
||
"мониторинга парсинга."
|
||
),
|
||
responses={
|
||
200: SourceTaskStatusSerializer(many=True),
|
||
**ErrorResponses.AUTHENTICATED,
|
||
},
|
||
)
|
||
def get(self, request):
|
||
rows = SourceCardService.list_task_statuses()
|
||
serializer = SourceTaskStatusSerializer(rows, many=True)
|
||
return api_response(serializer.data)
|
||
|
||
|
||
class SourceCardDetailView(APIView):
|
||
"""Detailed frontend card for a single source."""
|
||
|
||
permission_classes = [IsAuthenticated]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SOURCES_TAG],
|
||
operation_summary="Детали карточки источника",
|
||
operation_description=(
|
||
"Возвращает детальную информацию по одной карточке источника, "
|
||
"включая подисточники, последние загрузки и активные задачи."
|
||
),
|
||
responses={
|
||
200: SourceCardDetailSerializer,
|
||
**ErrorResponses.AUTHENTICATED_NOT_FOUND,
|
||
},
|
||
)
|
||
def get(self, request, slug: str):
|
||
card = SourceCardService.get_card(slug)
|
||
serializer = SourceCardDetailSerializer(card)
|
||
return api_response(serializer.data)
|
||
|
||
|
||
class SourceCardRefreshView(APIView):
|
||
"""Manual refresh trigger for a frontend source card."""
|
||
|
||
permission_classes = [IsAdminUser]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SOURCES_TAG],
|
||
operation_summary="Запуск обновления карточки источника",
|
||
operation_description=(
|
||
"Ставит обновление карточки в очередь и возвращает связанные task_id."
|
||
),
|
||
request_body=SourceCardRefreshRequestSerializer,
|
||
responses={
|
||
202: SourceCardRefreshResponseSerializer,
|
||
400: CommonResponses.BAD_REQUEST,
|
||
**ErrorResponses.ADMIN_NOT_FOUND,
|
||
},
|
||
)
|
||
def post(self, request, slug: str):
|
||
serializer = SourceCardRefreshRequestSerializer(data=request.data)
|
||
serializer.is_valid(raise_exception=True)
|
||
|
||
payload = SourceCardService.refresh_card(
|
||
slug=slug,
|
||
requested_by_id=request.user.id if request.user.is_authenticated else None,
|
||
params=serializer.validated_data.get("params", {}),
|
||
)
|
||
output = SourceCardRefreshResponseSerializer(payload)
|
||
return api_response(output.data, status_code=status.HTTP_202_ACCEPTED)
|
||
|
||
|
||
# =============================================================================
|
||
# Системные (логи загрузок, прокси)
|
||
# =============================================================================
|
||
|
||
|
||
class ParserLoadLogViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра логов загрузок парсеров.
|
||
|
||
Информация о каждой загрузке данных из внешних источников.
|
||
Только для администраторов.
|
||
"""
|
||
|
||
serializer_class = ParserLoadLogSerializer
|
||
permission_classes = [IsAdminUser]
|
||
filterset_fields = ["source", "status", "batch_id"]
|
||
ordering_fields = list(PARSER_LOG_ORDERING_FIELDS)
|
||
|
||
def get_queryset(self):
|
||
return _get_parser_logs_queryset(
|
||
search_query=self.request.query_params.get("search", "")
|
||
)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SYSTEM_TAG],
|
||
operation_summary="Список логов загрузок",
|
||
operation_description=(
|
||
"Возвращает историю загрузок данных парсерами.\n"
|
||
"Доступно только администраторам.\n"
|
||
"Поддерживает фильтрацию по: source, status, batch_id.\n"
|
||
"Поддерживает search по source, status, batch_id и error_message.\n"
|
||
"Поддерживает ordering по: id, batch_id, source, status, records_count, "
|
||
"created_at, updated_at."
|
||
),
|
||
manual_parameters=[
|
||
openapi.Parameter(
|
||
name="search",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_STRING,
|
||
required=False,
|
||
description="Поиск по source, status, batch_id и error_message",
|
||
),
|
||
openapi.Parameter(
|
||
name="ordering",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_STRING,
|
||
required=False,
|
||
description=(
|
||
"Сортировка по id, batch_id, source, status, records_count, "
|
||
"created_at, updated_at. Для обратной сортировки используйте префикс -"
|
||
),
|
||
),
|
||
],
|
||
responses={
|
||
200: ParserLoadLogSerializer(many=True),
|
||
**ErrorResponses.ADMIN,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SYSTEM_TAG],
|
||
operation_summary="Детали загрузки",
|
||
operation_description="Возвращает информацию о конкретной загрузке.",
|
||
responses={
|
||
200: ParserLoadLogSerializer,
|
||
**ErrorResponses.ADMIN_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|
||
|
||
|
||
class ParserLoadLogExportView(APIView):
|
||
"""Экспорт истории загрузок парсеров в CSV."""
|
||
|
||
permission_classes = [IsAdminUser]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SYSTEM_TAG],
|
||
operation_summary="Экспорт истории загрузок",
|
||
operation_description=(
|
||
"Выгружает историю загрузок парсеров в CSV. "
|
||
"Поддерживает те же фильтры, search и ordering, что и список логов."
|
||
),
|
||
manual_parameters=[
|
||
openapi.Parameter(
|
||
name="source",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_STRING,
|
||
required=False,
|
||
description="Фильтр по коду источника",
|
||
),
|
||
openapi.Parameter(
|
||
name="status",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_STRING,
|
||
required=False,
|
||
description="Фильтр по статусу",
|
||
),
|
||
openapi.Parameter(
|
||
name="batch_id",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_INTEGER,
|
||
required=False,
|
||
description="Фильтр по ID пакета",
|
||
),
|
||
openapi.Parameter(
|
||
name="search",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_STRING,
|
||
required=False,
|
||
description="Поиск по source, status, batch_id и error_message",
|
||
),
|
||
openapi.Parameter(
|
||
name="ordering",
|
||
in_=openapi.IN_QUERY,
|
||
type=openapi.TYPE_STRING,
|
||
required=False,
|
||
description="Сортировка по тем же полям, что и в списке логов",
|
||
),
|
||
],
|
||
responses={
|
||
200: "CSV файл",
|
||
**ErrorResponses.ADMIN,
|
||
},
|
||
)
|
||
def get(self, request):
|
||
queryset = _get_parser_logs_queryset(
|
||
search_query=request.query_params.get("search", "")
|
||
)
|
||
|
||
source = request.query_params.get("source")
|
||
status_value = request.query_params.get("status")
|
||
batch_id = request.query_params.get("batch_id")
|
||
|
||
if source:
|
||
queryset = queryset.filter(source=source)
|
||
if status_value:
|
||
queryset = queryset.filter(status=status_value)
|
||
if batch_id:
|
||
queryset = queryset.filter(batch_id=batch_id)
|
||
|
||
queryset = _apply_safe_ordering(
|
||
queryset,
|
||
request.query_params.get("ordering", ""),
|
||
PARSER_LOG_ORDERING_FIELDS,
|
||
)
|
||
|
||
serializer = ParserLoadLogSerializer(queryset, many=True)
|
||
response = HttpResponse(content_type="text/csv; charset=utf-8")
|
||
response["Content-Disposition"] = 'attachment; filename="parser-load-logs.csv"'
|
||
|
||
writer = csv.writer(response)
|
||
writer.writerow(
|
||
[
|
||
"id",
|
||
"batch_id",
|
||
"source",
|
||
"source_display",
|
||
"records_count",
|
||
"organizations_count",
|
||
"status",
|
||
"error_message",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
)
|
||
|
||
for row in serializer.data:
|
||
writer.writerow(
|
||
[
|
||
row["id"],
|
||
row["batch_id"],
|
||
row["source"],
|
||
row["source_display"],
|
||
row["records_count"],
|
||
row["organizations_count"],
|
||
row["status"],
|
||
row["error_message"],
|
||
row["created_at"],
|
||
row["updated_at"],
|
||
]
|
||
)
|
||
|
||
return response
|
||
|
||
|
||
class ProxyViewSet(ReadOnlyModelViewSet):
|
||
"""
|
||
API для просмотра списка прокси-серверов.
|
||
|
||
Используется для отладки и мониторинга парсеров.
|
||
Только для администраторов.
|
||
"""
|
||
|
||
queryset = Proxy.objects.all().order_by("-last_used_at")
|
||
serializer_class = ProxySerializer
|
||
permission_classes = [IsAdminUser]
|
||
filterset_fields = ["is_active"]
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SYSTEM_TAG],
|
||
operation_summary="Список прокси",
|
||
operation_description=(
|
||
"Возвращает список прокси-серверов для парсеров.\n"
|
||
"Доступно только администраторам.\n"
|
||
"Поддерживает фильтрацию по: is_active."
|
||
),
|
||
responses={
|
||
200: ProxySerializer(many=True),
|
||
**ErrorResponses.ADMIN,
|
||
},
|
||
)
|
||
def list(self, request, *args, **kwargs):
|
||
return super().list(request, *args, **kwargs)
|
||
|
||
@swagger_auto_schema(
|
||
tags=[SYSTEM_TAG],
|
||
operation_summary="Детали прокси",
|
||
operation_description="Возвращает информацию о конкретном прокси.",
|
||
responses={
|
||
200: ProxySerializer,
|
||
**ErrorResponses.ADMIN_NOT_FOUND,
|
||
},
|
||
)
|
||
def retrieve(self, request, *args, **kwargs):
|
||
return super().retrieve(request, *args, **kwargs)
|