feat: add parser source dashboard and scheduling
All checks were successful
CI/CD Pipeline / Code Quality Checks (pull_request) Successful in 1m55s
CI/CD Pipeline / Run Tests (pull_request) Successful in 3m6s
CI/CD Pipeline / Run API Inventory E2E Tests (pull_request) Successful in 2m48s
CI/CD Pipeline / Telegram Notify Success (pull_request) Successful in 19s

This commit is contained in:
2026-04-28 01:08:51 +02:00
parent 1249071a33
commit 26d61f7181
58 changed files with 7713 additions and 141 deletions

View File

@@ -0,0 +1,161 @@
"""Frontend-oriented parser result API v1 routes."""
from apps.parsers.serializers import (
ParserResultRecordSerializer,
ParserRunResponseSerializer,
)
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
from apps.parsers.views import (
RESULT_DETAIL_PARAMS,
RESULT_LIST_PARAMS,
UPLOAD_FILE_PARAM,
ParserUploadView,
SourceResultDetailView,
SourceResultListView,
source_result_swagger_tag,
)
from django.urls import path
from drf_yasg.utils import swagger_auto_schema
app_name = "parser_results"
ROUTE_TITLES = {
"zakupki": "ЕИС Закупки",
"proverki": "Проверки Генпрокуратуры",
}
def _result_source_descriptors():
"""Вернуть уникальные источники с явным API route."""
seen_routes = set()
for descriptor in PARSER_SOURCES.values():
if not descriptor.api_route or descriptor.api_route in seen_routes:
continue
seen_routes.add(descriptor.api_route)
yield descriptor
def _upload_source_descriptors():
"""Вернуть уникальные источники с route ручной загрузки."""
seen_routes = set()
for descriptor in PARSER_SOURCES.values():
if not descriptor.supports_file_upload:
continue
upload_route = descriptor.upload_api_route
if not upload_route or upload_route in seen_routes:
continue
seen_routes.add(upload_route)
yield descriptor
def _route_title(descriptor: ParserSourceDescriptor) -> str:
"""Вернуть человекочитаемое название публичного route."""
return ROUTE_TITLES.get(descriptor.api_route, descriptor.title)
def _route_description(descriptor: ParserSourceDescriptor) -> str:
"""Вернуть описание данных, опубликованных на одном route."""
scopes = [
item.data_scope
for item in PARSER_SOURCES.values()
if item.api_route == descriptor.api_route
]
return "; ".join(dict.fromkeys(scopes))
def _list_view(descriptor: ParserSourceDescriptor):
tag = source_result_swagger_tag(descriptor.key)
title = _route_title(descriptor)
description = _route_description(descriptor)
class SourceListView(SourceResultListView):
source_key = descriptor.key
@swagger_auto_schema(
operation_summary=f"{title}: get_list",
operation_description=(
f"Список записей источника: {description}. "
"Фильтры передаются query-параметрами."
),
manual_parameters=RESULT_LIST_PARAMS,
tags=[tag],
responses={200: ParserResultRecordSerializer(many=True)},
)
def get(self, request):
return super().get(request)
return SourceListView.as_view()
def _detail_view(descriptor: ParserSourceDescriptor):
tag = source_result_swagger_tag(descriptor.key)
title = _route_title(descriptor)
description = _route_description(descriptor)
class SourceDetailView(SourceResultDetailView):
source_key = descriptor.key
@swagger_auto_schema(
operation_summary=f"{title}: get",
operation_description=(
f"Одна запись источника: {description}. "
"Query-параметры дополнительно сужают выборку."
),
manual_parameters=RESULT_DETAIL_PARAMS,
tags=[tag],
responses={200: ParserResultRecordSerializer, 404: "Запись не найдена"},
)
def get(self, request, pk: int):
return super().get(request, pk=pk)
return SourceDetailView.as_view()
def _upload_view(descriptor: ParserSourceDescriptor):
tag = source_result_swagger_tag(descriptor.key)
class SourceUploadView(ParserUploadView):
@swagger_auto_schema(
operation_summary=f"{descriptor.title}: загрузить файл",
operation_description=(
f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
"Файл обрабатывается через Celery."
),
manual_parameters=[UPLOAD_FILE_PARAM],
consumes=["multipart/form-data"],
tags=[tag],
responses={202: ParserRunResponseSerializer, 400: "Ошибка валидации"},
)
def post(self, request):
return super().post(request, source_key=descriptor.key)
return SourceUploadView.as_view()
urlpatterns = []
for source_descriptor in _result_source_descriptors():
route_name = source_descriptor.api_route.replace("/", "-")
urlpatterns.extend(
[
path(
f"{source_descriptor.api_route}/",
_list_view(source_descriptor),
name=f"{route_name}-get-list",
),
path(
f"{source_descriptor.api_route}/<int:pk>/",
_detail_view(source_descriptor),
name=f"{route_name}-get",
),
]
)
for source_descriptor in _upload_source_descriptors():
upload_route = source_descriptor.upload_api_route
route_name = upload_route.replace("/", "-")
urlpatterns.append(
path(
f"{upload_route}/",
_upload_view(source_descriptor),
name=f"{route_name}-upload",
)
)