avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Align frontend API contracts

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-03-22 13:21:02 +01:00
parent 0da5b4abe2
commit e639b3c792
35 changed files with 1362 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/apps/parsers/migrations/0014_parsingsettings.py Normal file
View File

@@ -0,0 +1,34 @@
# Generated by Django 3.2.25 on 2026-03-22 11:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('parsers', '0013_auto_20260320_1010'),
]
operations = [
migrations.CreateModel(
name='ParsingSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')),
('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')),
('singleton_key', models.PositiveSmallIntegerField(default=1, editable=False, unique=True, verbose_name='ключ singleton-записи')),
('manufacturers_and_products', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='производители и продукция')),
('public_procurements', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='госзакупки')),
('defense_unreliable_suppliers', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='weekly', max_length=20, verbose_name='недобросовестные поставщики ОПК')),
('planned_inspections', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='monthly', max_length=20, verbose_name='плановые проверки')),
('arbitration_cases', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='арбитражные дела')),
('bankruptcy_procedures', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='банкротные процедуры')),
('information_security_registries', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='yearly', max_length=20, verbose_name='реестры информационной безопасности')),
],
options={
'verbose_name': 'настройки парсинга',
'verbose_name_plural': 'настройки парсинга',
'db_table': 'parsers_parsing_settings',
},
),
]

67
src/apps/parsers/models.py
View File

@@ -846,3 +846,70 @@ class FinancialReportLine(models.Model):
def __str__(self) -> str:
return f"{self.line_code} ({self.line_name[:30]}) - {self.year}"
class ParsingSettings(TimestampMixin, models.Model):
"""Singleton-настройки периодичности обновления источников парсинга."""
class Frequency(models.TextChoices):
DAILY = "daily", _("Ежедневно")
WEEKLY = "weekly", _("Еженедельно")
MONTHLY = "monthly", _("Ежемесячно")
YEARLY = "yearly", _("Ежегодно")
singleton_key = models.PositiveSmallIntegerField(
_("ключ singleton-записи"),
default=1,
unique=True,
editable=False,
)
manufacturers_and_products = models.CharField(
_("производители и продукция"),
max_length=20,
choices=Frequency.choices,
default=Frequency.DAILY,
)
public_procurements = models.CharField(
_("госзакупки"),
max_length=20,
choices=Frequency.choices,
default=Frequency.DAILY,
)
defense_unreliable_suppliers = models.CharField(
_("недобросовестные поставщики ОПК"),
max_length=20,
choices=Frequency.choices,
default=Frequency.WEEKLY,
)
planned_inspections = models.CharField(
_("плановые проверки"),
max_length=20,
choices=Frequency.choices,
default=Frequency.MONTHLY,
)
arbitration_cases = models.CharField(
_("арбитражные дела"),
max_length=20,
choices=Frequency.choices,
default=Frequency.DAILY,
)
bankruptcy_procedures = models.CharField(
_("банкротные процедуры"),
max_length=20,
choices=Frequency.choices,
default=Frequency.DAILY,
)
information_security_registries = models.CharField(
_("реестры информационной безопасности"),
max_length=20,
choices=Frequency.choices,
default=Frequency.YEARLY,
)
class Meta:
db_table = "parsers_parsing_settings"
verbose_name = _("настройки парсинга")
verbose_name_plural = _("настройки парсинга")
def __str__(self) -> str:
return "Настройки парсинга"

62
src/apps/parsers/serializers.py
View File

@@ -12,6 +12,7 @@ from apps.parsers.models import (
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParsingSettings,
ParserLoadLog,
ProcurementRecord,
Proxy,
@@ -262,22 +263,45 @@ class FNSFileUploadSerializer(serializers.Serializer):
Принимает список Excel файлов в формате fin_{id}_{ogrn}.xlsx
"""
file = serializers.FileField(
required=False,
help_text="Одиночный файл для загрузки (fin_*.xlsx)",
)
files = serializers.ListField(
child=serializers.FileField(),
required=False,
allow_empty=False,
help_text="Список файлов для загрузки (fin_*.xlsx)",
)
def validate_files(self, value):
def validate(self, attrs):
files = attrs.get("files")
single_file = attrs.get("file")
if single_file and files:
raise serializers.ValidationError(
{"file": "Используйте либо file, либо files."}
)
if single_file:
files = [single_file]
if not files:
raise serializers.ValidationError(
{"file": "Нужно передать file или files."}
)
attrs["files"] = self._validate_uploaded_files(files)
return attrs
def _validate_uploaded_files(self, files):
"""Валидация файлов."""
for file in value:
for file in files:
if not FNS_XLSX_FILENAME_RE.match(file.name):
raise serializers.ValidationError(
f"Неверный формат имени файла: {file.name}. "
"Ожидается: fin_{{id}}_{{ogrn}}.xlsx"
)
return value
return files
class FNSZipUploadSerializer(serializers.Serializer):
@@ -291,6 +315,22 @@ class FNSZipUploadSerializer(serializers.Serializer):
return value
class ParsingSettingsSerializer(serializers.ModelSerializer):
"""Настройки периодичности обновления источников парсинга."""
class Meta:
model = ParsingSettings
fields = [
"manufacturers_and_products",
"public_procurements",
"defense_unreliable_suppliers",
"planned_inspections",
"arbitration_cases",
"bankruptcy_procedures",
"information_security_registries",
]
# =============================================================================
# Служебные модели
# =============================================================================
@@ -374,6 +414,22 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
return 0
class ParserLoadLogListSerializer(serializers.Serializer):
"""Строка списка логов в frontend-friendly формате."""
id = serializers.IntegerField(read_only=True)
batch_id = serializers.IntegerField(read_only=True)
source = serializers.CharField(read_only=True)
source_label = serializers.CharField(read_only=True, allow_null=True)
records_count = serializers.IntegerField(read_only=True)
organizations_count = serializers.IntegerField(read_only=True)
status = serializers.CharField(read_only=True)
status_label = serializers.CharField(read_only=True)
error_message = serializers.CharField(read_only=True, allow_blank=True)
created_at = serializers.DateTimeField(read_only=True)
updated_at = serializers.DateTimeField(read_only=True)
class ProxySerializer(serializers.ModelSerializer):
"""
Прокси-сервер для парсеров.

25
src/apps/parsers/source_cards.py
View File

@@ -195,11 +195,36 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
)
SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS}
SOURCE_CARD_BY_PARSER_SOURCE = {
source_item.parser_source: definition
for definition in SOURCE_CARD_DEFINITIONS
for source_item in definition.source_items
if source_item.parser_source
}
class SourceCardService:
"""Builds aggregated source cards for frontend pages."""
@classmethod
def get_card_slug_by_parser_source(cls, parser_source: str | None) -> str | None:
definition = SOURCE_CARD_BY_PARSER_SOURCE.get(parser_source)
return definition.slug if definition else None
@classmethod
def get_card_title_by_parser_source(cls, parser_source: str | None) -> str | None:
definition = SOURCE_CARD_BY_PARSER_SOURCE.get(parser_source)
return definition.title if definition else None
@classmethod
def get_parser_sources_by_card_slug(cls, slug: str) -> list[str]:
definition = SOURCE_CARD_BY_SLUG.get(slug)
if definition is None:
return []
return [
item.parser_source for item in definition.source_items if item.parser_source
]
@classmethod
def list_cards(cls) -> list[dict[str, Any]]:
return [cls.get_card(definition.slug) for definition in SOURCE_CARD_DEFINITIONS]

9
src/apps/parsers/urls.py
View File

@@ -11,6 +11,7 @@ from apps.parsers.views import (
IndustrialProductViewSet,
InspectionViewSet,
ManufacturerViewSet,
ParsingSettingsView,
ParserLoadLogExportView,
ParserLoadLogViewSet,
ProcurementViewSet,
@@ -78,6 +79,14 @@ fns_urlpatterns = [
path("", include(fns_router.urls)),
]
# =============================================================================
# Parsing settings: /api/v1/parsing/
# =============================================================================
parsing_urlpatterns = [
path("settings/", ParsingSettingsView.as_view(), name="parsing-settings"),
]
# =============================================================================
# Frontend sources: /api/v1/sources/
# =============================================================================

310
src/apps/parsers/views.py
View File

@@ -16,6 +16,7 @@ from apps.parsers.models import (
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParsingSettings,
ParserLoadLog,
ProcurementRecord,
Proxy,
@@ -27,6 +28,8 @@ from apps.parsers.serializers import (
IndustrialCertificateSerializer,
IndustrialProductSerializer,
InspectionSerializer,
ParserLoadLogListSerializer,
ParsingSettingsSerializer,
ManufacturerSerializer,
ParserLoadLogSerializer,
ProcurementSerializer,
@@ -38,6 +41,7 @@ from apps.parsers.serializers import (
SourceTaskStatusSerializer,
)
from apps.parsers.source_cards import SourceCardService
from django.core.paginator import Paginator
from django.db.models import CharField, Count, Q
from django.db.models.functions import Cast
from django.http import HttpResponse
@@ -60,6 +64,7 @@ ZAKUPKI_TAG = swagger_tag("Государственные закупки", "publ
FNS_TAG = swagger_tag("ФНС - Бухгалтерская отчетность", "fns_financial_reports")
SOURCES_TAG = swagger_tag("Источники для фронта", "frontend_sources")
SYSTEM_TAG = swagger_tag("Системные", "system")
PARSING_TAG = swagger_tag("Настройки парсинга", "parsing_settings")
PARSER_LOG_ORDERING_FIELDS = {
"id",
@@ -71,6 +76,18 @@ PARSER_LOG_ORDERING_FIELDS = {
"updated_at",
}
PARSER_LOG_STATUS_LABELS = {
"success": "Успешно",
"failed": "Ошибка",
"failure": "Ошибка",
"error": "Ошибка",
"in_progress": "В процессе",
"pending": "В очереди",
"started": "В процессе",
"retry": "Повтор",
"skipped": "Пропущено",
}
def _get_parser_logs_queryset(*, search_query: str = ""):
queryset = ParserLoadLog.objects.all().order_by("-created_at")
@@ -102,6 +119,183 @@ def _apply_safe_ordering(queryset, ordering: str, allowed_fields: set[str]):
return queryset.order_by(*order_by_fields)
def _matches_parser_log_search(row: dict, search_term: str) -> bool:
normalized_search = search_term.casefold()
for value in row.values():
if value is None:
continue
if normalized_search in str(value).casefold():
return True
return False
def _sort_parser_log_rows(rows: list[dict], ordering: str) -> list[dict]:
allowed_fields = {
"id",
"batch_id",
"source",
"source_label",
"status",
"status_label",
"records_count",
"organizations_count",
"created_at",
"updated_at",
}
sorted_rows = list(rows)
order_by_fields = [item.strip() for item in ordering.split(",") if item.strip()]
for raw_field in reversed(order_by_fields):
field_name = raw_field[1:] if raw_field.startswith("-") else raw_field
if field_name not in allowed_fields:
continue
reverse = raw_field.startswith("-")
sorted_rows.sort(
key=lambda row: (row.get(field_name) is None, row.get(field_name)),
reverse=reverse,
)
return sorted_rows
def _build_page_url(request, page_number: int) -> str:
query_params = request.query_params.copy()
query_params["page"] = page_number
return request.build_absolute_uri(f"{request.path}?{query_params.urlencode()}")
def _paginate_results(request, rows: list[dict]):
page_size_raw = request.query_params.get("page_size", "20")
page_raw = request.query_params.get("page", "1")
try:
page_size = max(1, min(int(page_size_raw), 100))
page_number = max(1, int(page_raw))
except (TypeError, ValueError) as exc:
raise ValidationError(
{"detail": "Параметры page и page_size должны быть положительными целыми числами."}
) from exc
paginator = Paginator(rows, page_size)
page_obj = paginator.get_page(page_number)
return {
"count": paginator.count,
"next": (
_build_page_url(request, page_obj.next_page_number())
if page_obj.has_next()
else None
),
"previous": (
_build_page_url(request, page_obj.previous_page_number())
if page_obj.has_previous()
else None
),
"results": list(page_obj.object_list),
}
def _get_parser_log_status_label(status_value: str) -> str:
return PARSER_LOG_STATUS_LABELS.get(status_value, status_value)
def _get_parser_log_source_label(log: ParserLoadLog) -> str:
return (
SourceCardService.get_card_title_by_parser_source(log.source)
or log.get_source_display()
)
def _get_parser_log_source_value(log: ParserLoadLog) -> str:
return SourceCardService.get_card_slug_by_parser_source(log.source) or log.source
def _get_parser_log_organizations_count(log: ParserLoadLog) -> int:
if log.source == ParserLoadLog.Source.FNS_REPORTS:
return (
FinancialReport.objects.filter(load_batch=log.batch_id)
.exclude(ogrn="")
.values("ogrn")
.distinct()
.count()
)
if log.source == ParserLoadLog.Source.INDUSTRIAL:
return (
IndustrialCertificateRecord.objects.filter(load_batch=log.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if log.source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS:
return (
IndustrialProductRecord.objects.filter(load_batch=log.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if log.source == ParserLoadLog.Source.MANUFACTURES:
return (
ManufacturerRecord.objects.filter(load_batch=log.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if log.source == ParserLoadLog.Source.INSPECTIONS:
return (
InspectionRecord.objects.filter(load_batch=log.batch_id)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if log.source == ParserLoadLog.Source.PROCUREMENTS:
return (
ProcurementRecord.objects.filter(load_batch=log.batch_id)
.exclude(customer_inn="")
.values("customer_inn")
.distinct()
.count()
)
return 0
def _serialize_parser_log_row(log: ParserLoadLog) -> dict:
return {
"id": log.id,
"batch_id": log.batch_id,
"source": _get_parser_log_source_value(log),
"source_label": _get_parser_log_source_label(log),
"records_count": log.records_count,
"organizations_count": _get_parser_log_organizations_count(log),
"status": log.status,
"status_label": _get_parser_log_status_label(log.status),
"error_message": log.error_message,
"created_at": log.created_at,
"updated_at": log.updated_at,
}
def _apply_parser_log_filters(request):
queryset = _get_parser_logs_queryset(search_query="")
source_value = request.query_params.get("source", "").strip()
if source_value:
card_sources = SourceCardService.get_parser_sources_by_card_slug(source_value)
if card_sources:
queryset = queryset.filter(source__in=card_sources)
else:
queryset = queryset.filter(source=source_value)
status_value = request.query_params.get("status")
if status_value:
queryset = queryset.filter(status=status_value)
batch_id = request.query_params.get("batch_id")
if batch_id:
queryset = queryset.filter(batch_id=batch_id)
return queryset
# =============================================================================
# Минпромторг - Сертификаты промышленного производства
# =============================================================================
@@ -531,6 +725,17 @@ class FNSReportUploadView(APIView):
requested_by_id=request.user.id,
)
if "file" in request.data and "files" not in request.data:
message = (
"Файл загружен"
if result.queued
else "Файл уже обрабатывается или был загружен ранее"
)
return Response(
{"success": True, "message": message},
status=status.HTTP_200_OK,
)
return Response(
{
"queued": result.queued,
@@ -541,6 +746,50 @@ class FNSReportUploadView(APIView):
)
class ParsingSettingsView(APIView):
"""Получение и изменение singleton-настроек парсинга."""
permission_classes = [IsAdminUser]
@staticmethod
def _get_settings() -> ParsingSettings:
settings_obj, _ = ParsingSettings.objects.get_or_create(singleton_key=1)
return settings_obj
@swagger_auto_schema(
tags=[PARSING_TAG],
operation_summary="Получить настройки парсинга",
responses={
200: ParsingSettingsSerializer,
**ErrorResponses.ADMIN,
},
)
def get(self, request):
serializer = ParsingSettingsSerializer(self._get_settings())
return Response(serializer.data, status=status.HTTP_200_OK)
@swagger_auto_schema(
tags=[PARSING_TAG],
operation_summary="Изменить настройки парсинга",
request_body=ParsingSettingsSerializer,
responses={
200: ParsingSettingsSerializer,
400: CommonResponses.BAD_REQUEST,
**ErrorResponses.ADMIN,
},
)
def patch(self, request):
settings_obj = self._get_settings()
serializer = ParsingSettingsSerializer(
settings_obj,
data=request.data,
partial=True,
)
serializer.is_valid(raise_exception=True)
serializer.save()
return Response(serializer.data, status=status.HTTP_200_OK)
# =============================================================================
# Frontend-oriented source cards
# =============================================================================
@@ -643,7 +892,20 @@ class SourceCardRefreshView(APIView):
params=serializer.validated_data.get("params", {}),
)
output = SourceCardRefreshResponseSerializer(payload)
return api_response(output.data, status_code=status.HTTP_202_ACCEPTED)
serialized_payload = output.data
tasks = serialized_payload.get("tasks", [])
task_id = tasks[0]["task_id"] if tasks else None
response_payload = {
"task_id": task_id,
"status": "accepted",
}
if len(tasks) > 1:
response_payload["tasks"] = tasks
response_payload["source_card"] = serialized_payload.get("source_card")
return Response(
response_payload,
status=status.HTTP_202_ACCEPTED,
)
# =============================================================================
@@ -665,9 +927,7 @@ class ParserLoadLogViewSet(ReadOnlyModelViewSet):
ordering_fields = list(PARSER_LOG_ORDERING_FIELDS)
def get_queryset(self):
return _get_parser_logs_queryset(
search_query=self.request.query_params.get("search", "")
)
return _apply_parser_log_filters(self.request)
@swagger_auto_schema(
tags=[SYSTEM_TAG],
@@ -705,7 +965,13 @@ class ParserLoadLogViewSet(ReadOnlyModelViewSet):
},
)
def list(self, request, *args, **kwargs):
return super().list(request, *args, **kwargs)
rows = [_serialize_parser_log_row(item) for item in self.get_queryset()]
search_term = request.query_params.get("search", "").strip()
if search_term:
rows = [row for row in rows if _matches_parser_log_search(row, search_term)]
rows = _sort_parser_log_rows(rows, request.query_params.get("ordering", ""))
serializer = ParserLoadLogListSerializer(rows, many=True)
return Response(_paginate_results(request, serializer.data))
@swagger_auto_schema(
tags=[SYSTEM_TAG],
@@ -775,28 +1041,12 @@ class ParserLoadLogExportView(APIView):
},
)
def get(self, request):
queryset = _get_parser_logs_queryset(
search_query=request.query_params.get("search", "")
)
rows = [_serialize_parser_log_row(item) for item in _apply_parser_log_filters(request)]
search_term = request.query_params.get("search", "").strip()
if search_term:
rows = [row for row in rows if _matches_parser_log_search(row, search_term)]
rows = _sort_parser_log_rows(rows, request.query_params.get("ordering", ""))
source = request.query_params.get("source")
status_value = request.query_params.get("status")
batch_id = request.query_params.get("batch_id")
if source:
queryset = queryset.filter(source=source)
if status_value:
queryset = queryset.filter(status=status_value)
if batch_id:
queryset = queryset.filter(batch_id=batch_id)
queryset = _apply_safe_ordering(
queryset,
request.query_params.get("ordering", ""),
PARSER_LOG_ORDERING_FIELDS,
)
serializer = ParserLoadLogSerializer(queryset, many=True)
response = HttpResponse(content_type="text/csv; charset=utf-8")
response["Content-Disposition"] = 'attachment; filename="parser-load-logs.csv"'
@@ -806,26 +1056,28 @@ class ParserLoadLogExportView(APIView):
"id",
"batch_id",
"source",
"source_display",
"source_label",
"records_count",
"organizations_count",
"status",
"status_label",
"error_message",
"created_at",
"updated_at",
]
)
for row in serializer.data:
for row in rows:
writer.writerow(
[
row["id"],
row["batch_id"],
row["source"],
row["source_display"],
row["source_label"],
row["records_count"],
row["organizations_count"],
row["status"],
row["status_label"],
row["error_message"],
row["created_at"],
row["updated_at"],