From 89607356b78ba25f0a57b1f18f111b69eeed2580 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 12 May 2026 17:48:54 +0200 Subject: [PATCH] Add organization stats endpoint --- src/apps/parsers/views.py | 84 ++++++++++++++++++++--- src/core/api_v1_urls.py | 4 +- src/registers/urls.py | 5 ++ src/registers/views.py | 105 +++++++++++++++++++++++++++- tests/apps/parsers/test_views.py | 106 +++++++++++++++++++++++++++++ tests/apps/registers/test_views.py | 51 +++++++++++++- 6 files changed, 342 insertions(+), 13 deletions(-) diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 4e5dfef..c4ea58c 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -69,7 +69,7 @@ from apps.registers.models import RegistryMembershipPeriod from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.db.models import CharField, Count, Q -from django.db.models.functions import Cast +from django.db.models.functions import Cast, Lower from django.http import HttpResponse from django.utils.text import get_valid_filename from django.views.generic import TemplateView @@ -1452,7 +1452,10 @@ def _save_uploaded_parser_file(uploaded_file) -> str: def _model_payload(record) -> dict: payload = {} for field in record._meta.fields: - value = getattr(record, field.name) + if getattr(field, "many_to_one", False): + value = getattr(record, field.attname) + else: + value = getattr(record, field.name) if hasattr(value, "isoformat"): value = value.isoformat() payload[field.name] = value @@ -1502,13 +1505,20 @@ def _native_record_to_result( inn = record.customer_inn ogrn = record.customer_ogrn elif source == ParserLoadLog.Source.FNS_REPORTS: + registry_organization = record.registry_organization external_id = record.external_id - organisation_name = "" + organisation_name = ( + registry_organization.pn_name if registry_organization else "" + ) title = record.file_name record_date = "" status_value = record.status url = "" - inn = "" + inn = ( + str(registry_organization.mn_inn) + if registry_organization and registry_organization.mn_inn + else "" + ) ogrn = record.ogrn else: external_id = record.registration_number @@ -2027,14 +2037,34 @@ def source_result_swagger_tag(source_key: str) -> str: return SOURCE_RESULT_TAGS.get(source_key, PARSERS_TAG) -def _safe_ordering(ordering: str, field_map: dict[str, str]) -> list[str]: +CASE_INSENSITIVE_RESULT_ORDERING_FIELDS = { + "external_id", + "organisation_name", + "title", + "record_date", + "status", + "url", +} + + +def _safe_ordering( + ordering: str, + field_map: dict[str, str], + *, + case_insensitive_fields: set[str] | None = None, +) -> list: + case_insensitive_fields = case_insensitive_fields or set() result = [] for raw_field in (item.strip() for item in ordering.split(",") if item.strip()): desc = raw_field.startswith("-") api_field = raw_field[1:] if desc else raw_field model_field = field_map.get(api_field) if model_field: - result.append(f"-{model_field}" if desc else model_field) + if api_field in case_insensitive_fields: + expression = Lower(model_field) + result.append(expression.desc() if desc else expression.asc()) + else: + result.append(f"-{model_field}" if desc else model_field) return result @@ -2076,15 +2106,21 @@ def _native_field_map(source: str) -> dict[str, str]: "organisation_name": "customer_name", "title": "purchase_name", "record_date": "publish_date", + "amount": "max_price_amount", "status": "status", } if source == ParserLoadLog.Source.FNS_REPORTS: return { - **common, + "id": "id", + "load_batch": "load_batch", "external_id": "external_id", + "inn": "registry_organization__mn_inn", "ogrn": "ogrn", + "organisation_name": "registry_organization__pn_name", "title": "file_name", "status": "status", + "created_at": "created_at", + "updated_at": "updated_at", } return { **common, @@ -2131,6 +2167,7 @@ def _native_search_q(source: str, search: str) -> Q: Q(file_name__icontains=search) | Q(external_id__icontains=search) | Q(ogrn__icontains=search) + | Q(registry_organization__pn_name__icontains=search) | Q(status__icontains=search) ) return ( @@ -2155,6 +2192,21 @@ def _generic_search_q(search: str) -> Q: ) +def _apply_native_search(queryset, source: str, search: str): + if source != ParserLoadLog.Source.FNS_REPORTS: + return queryset.filter(_native_search_q(source, search)) + + return queryset.annotate( + registry_organization_inn_text=Cast( + "registry_organization__mn_inn", + output_field=CharField(), + ), + ).filter( + _native_search_q(source, search) + | Q(registry_organization_inn_text__icontains=search) + ) + + def _route_model_sources(descriptor) -> set[str]: return { item.source @@ -2178,6 +2230,8 @@ def _result_sources_for_request(descriptor, params: dict) -> set[str]: def _filter_native_result_queryset(source: str, params: dict, sources: set[str]): queryset = NATIVE_RECORD_MODELS[source].objects.all() + if source == ParserLoadLog.Source.FNS_REPORTS: + queryset = queryset.select_related("registry_organization") if not sources: queryset = queryset.none() field_map = _native_field_map(source) @@ -2189,8 +2243,12 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str]) if params.get("record_date") and field_map.get("record_date"): queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]}) if params.get("search"): - queryset = queryset.filter(_native_search_q(source, params["search"])) - ordering = _safe_ordering(params.get("ordering") or "-created_at", field_map) + queryset = _apply_native_search(queryset, source, params["search"]) + ordering = _safe_ordering( + params.get("ordering") or "-created_at", + field_map, + case_insensitive_fields=CASE_INSENSITIVE_RESULT_ORDERING_FIELDS, + ) return queryset.order_by(*(ordering or ["-created_at"])) @@ -2215,11 +2273,17 @@ def _filter_generic_result_queryset(sources: set[str], params: dict): "organisation_name": "organisation_name", "title": "title", "record_date": "record_date", + "amount": "amount", "status": "status", + "url": "url", "created_at": "created_at", "updated_at": "updated_at", } - ordering = _safe_ordering(params.get("ordering") or "-created_at", field_map) + ordering = _safe_ordering( + params.get("ordering") or "-created_at", + field_map, + case_insensitive_fields=CASE_INSENSITIVE_RESULT_ORDERING_FIELDS, + ) return queryset.order_by(*(ordering or ["-created_at"])) diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index 7fbf28e..5ae9f5d 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -35,7 +35,7 @@ from apps.parsers.urls import ( zakupki_urlpatterns, ) from django.urls import include, path -from registers.urls import registers_urlpatterns +from registers.urls import registers_urlpatterns, stat_urlpatterns app_name = "api_v1" @@ -66,6 +66,8 @@ urlpatterns = [ path("fns/", include((fns_urlpatterns, "fns"))), # Результаты новых источников без перекрытия старых API выше path("", include("apps.parsers.api_result_urls", namespace="parser_results")), + # Сводные frontend-счетчики + path("stat/", include((stat_urlpatterns, "stat"))), # Управление parser Celery задачами и dashboard data path("parsers/", include("apps.parsers.urls")), # Агрегированные карточки источников для фронтенда diff --git a/src/registers/urls.py b/src/registers/urls.py index 496d619..ef24197 100644 --- a/src/registers/urls.py +++ b/src/registers/urls.py @@ -10,6 +10,7 @@ from registers.views import ( RegisterUploadView, RegisterViewSet, RegistryOrganizationListView, + RegistryStatsView, ) app_name = "registers" @@ -45,4 +46,8 @@ registers_v2_urlpatterns = [ path("", include(router.urls)), ] +stat_urlpatterns = [ + path("organizations/", RegistryStatsView.as_view(), name="organization-summary"), +] + urlpatterns = [] diff --git a/src/registers/views.py b/src/registers/views.py index 0bf7062..62cdec8 100644 --- a/src/registers/views.py +++ b/src/registers/views.py @@ -3,6 +3,7 @@ from __future__ import annotations from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag +from apps.core.response import api_response from django.db.models import Count, Q from django.shortcuts import get_object_or_404 from drf_yasg import openapi @@ -17,7 +18,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.viewsets import ReadOnlyModelViewSet -from registers.models import Organization, Register +from registers.models import Organization, Register, RegistryMembershipPeriod from registers.pagination import RegistersPagination from registers.serializers import ( FixedRegisterFileUploadSerializer, @@ -44,6 +45,44 @@ REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG = { "rosatom-opk": "Реестр госкорпорации Росатом ОПК", } +REGISTRY_STAT_CARD_DEFINITIONS = ( + { + "slug": "opk", + "title": "Организации, входящие в реестр предприятий ОПК", + "registry_name": "Реестр предприятий ОПК", + }, + { + "slug": "rosatom", + "title": "Организации, входящие в реестр госкорпорации Росатом", + "registry_name": "Реестр госкорпорации Росатом", + }, + { + "slug": "roscosmos", + "title": "Организации, входящие в реестр госкорпорации Роскосмос", + "registry_name": "Реестр госкорпорации Роскосмос", + }, + { + "slug": "rosatom-opk", + "title": "Организации, входящие в реестр госкорпорации Росатом ОПК", + "registry_name": "Реестр госкорпорации Росатом ОПК", + }, + { + "slug": "rosatom-goz", + "title": "Организации, входящие в реестр госкорпорации Росатом ГОЗ", + "registry_name": "Реестр госкорпорации Росатом ГОЗ", + }, + { + "slug": "roscosmos-goz", + "title": "Организации, входящие в реестр госкорпорации Роскосмос ГОЗ", + "registry_name": "Реестр госкорпорации Роскосмос ГОЗ", + }, + { + "slug": "roscosmos-opk", + "title": "Организации, входящие в реестр госкорпорации Роскосмос ОПК", + "registry_name": "Реестр госкорпорации Роскосмос ОПК", + }, +) + def _start_snapshot_refresh_task() -> None: refresh_all_organization_data_snapshots.delay() @@ -217,6 +256,70 @@ class OrganizationViewSet(ReadOnlyModelViewSet): return super().retrieve(request, *args, **kwargs) +class RegistryStatsView(APIView): + """Сводные счетчики организаций для frontend stats cards.""" + + permission_classes = [IsAuthenticated] + + def get(self, request): + registry_names = [ + item["registry_name"] for item in REGISTRY_STAT_CARD_DEFINITIONS + ] + active_counts_by_name = { + row["registry__name"]: row["organizations_count"] + for row in ( + RegistryMembershipPeriod.objects.filter( + ended_at__isnull=True, + registry__name__in=registry_names, + ) + .values("registry__name") + .annotate( + organizations_count=Count("organization_id", distinct=True), + ) + ) + } + total_organizations = Organization.objects.count() + active_registry_organizations = ( + RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) + .order_by() + .values("organization_id") + .distinct() + .count() + ) + + cards = [ + { + "slug": "total", + "title": "Общее количество организаций", + "registry_name": None, + "organizations_count": total_organizations, + "order": 0, + } + ] + for order, definition in enumerate(REGISTRY_STAT_CARD_DEFINITIONS, start=10): + cards.append( + { + "slug": definition["slug"], + "title": definition["title"], + "registry_name": definition["registry_name"], + "organizations_count": active_counts_by_name.get( + definition["registry_name"], + 0, + ), + "order": order, + } + ) + + return api_response( + { + "total_organizations": total_organizations, + "active_registry_organizations": active_registry_organizations, + "counts": {item["slug"]: item["organizations_count"] for item in cards}, + "cards": cards, + } + ) + + class RegistryOrganizationListView(ListAPIView): """API списка организаций конкретного реестра.""" diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 5ebf730..78e7aec 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -697,6 +697,112 @@ class ParsersViewSetTest(APITestCase): self.assertEqual(unified_response.data["data"][0]["id"], report.id) self.assertEqual(unified_response.data["data"][0]["title"], report.file_name) + def test_fns_financial_results_searches_and_orders_by_registry_organization(self): + alpha_org = RegisterOrganizationFactory( + pn_name='alpha "ФНС"', + mn_ogrn=1027700000001, + mn_inn=7701000001, + ) + beta_org = RegisterOrganizationFactory( + pn_name='Beta "ФНС"', + mn_ogrn=1027700000002, + mn_inn=7701000002, + ) + alpha_report = FinancialReport.objects.create( + external_id="fns-alpha", + ogrn=str(alpha_org.mn_ogrn), + registry_organization=alpha_org, + file_name=f"fin_fns-alpha_{alpha_org.mn_ogrn}.xlsx", + file_hash=fake.sha256(raw_output=False), + load_batch=1, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + beta_report = FinancialReport.objects.create( + external_id="fns-beta", + ogrn=str(beta_org.mn_ogrn), + registry_organization=beta_org, + file_name=f"fin_fns-beta_{beta_org.mn_ogrn}.xlsx", + file_hash=fake.sha256(raw_output=False), + load_batch=1, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + self.client.force_authenticate(self.user) + + name_response = self.client.get( + "/api/v1/parsers/results/fns_financial/", + {"search": "alpha"}, + ) + inn_response = self.client.get( + "/api/v1/parsers/results/fns_financial/", + {"search": str(beta_org.mn_inn)}, + ) + ordered_response = self.client.get( + "/api/v1/parsers/results/fns_financial/", + {"ordering": "organisation_name"}, + ) + + self.assertEqual(name_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in name_response.data["data"]], + [alpha_report.id], + ) + self.assertEqual( + name_response.data["data"][0]["organisation_name"], + alpha_org.pn_name, + ) + self.assertEqual(name_response.data["data"][0]["inn"], str(alpha_org.mn_inn)) + self.assertEqual(inn_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in inn_response.data["data"]], + [beta_report.id], + ) + self.assertEqual(ordered_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in ordered_response.data["data"]], + [alpha_report.id, beta_report.id], + ) + + def test_eis_results_order_text_case_insensitively_and_by_amount(self): + low_amount = GenericParserRecord.objects.create( + load_batch=1, + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + external_id="amount-low", + organisation_name="alpha customer", + title="Low amount", + amount="10.00", + ) + high_amount = GenericParserRecord.objects.create( + load_batch=1, + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + external_id="amount-high", + organisation_name="Beta customer", + title="High amount", + amount="20.00", + ) + self.client.force_authenticate(self.user) + + amount_response = self.client.get( + "/api/v1/eis/procurements-44fz/", + {"ordering": "amount"}, + ) + name_response = self.client.get( + "/api/v1/eis/procurements-44fz/", + {"ordering": "-organisation_name"}, + ) + + self.assertEqual(amount_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in amount_response.data["data"]], + [low_amount.id, high_amount.id], + ) + self.assertEqual(name_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in name_response.data["data"]], + [high_amount.id, low_amount.id], + ) + def test_system_logs_admin_only(self): log = ParserLoadLogFactory() url_logs = reverse("api_v1:system:parser-logs-list") diff --git a/tests/apps/registers/test_views.py b/tests/apps/registers/test_views.py index e108c82..827cf10 100644 --- a/tests/apps/registers/test_views.py +++ b/tests/apps/registers/test_views.py @@ -6,7 +6,7 @@ import io from datetime import date from unittest.mock import patch -from apps.registers.models import Organization, RegistryMembershipPeriod +from apps.registers.models import Organization, Register, RegistryMembershipPeriod from django.core.files.uploadedfile import SimpleUploadedFile from django.db import IntegrityError from django.urls import reverse @@ -198,6 +198,55 @@ class RegistersViewsTest(APITestCase): "/api/v2/registers/rosatom-goz/upload/", ) + def test_stat_organizations_endpoint_returns_registry_cards(self): + opk_registry, _ = Register.objects.get_or_create(name="Реестр предприятий ОПК") + rosatom_registry, _ = Register.objects.get_or_create( + name="Реестр госкорпорации Росатом" + ) + roscosmos_goz_registry, _ = Register.objects.get_or_create( + name="Реестр госкорпорации Роскосмос ГОЗ" + ) + opk_organization = OrganizationFactory() + shared_organization = OrganizationFactory() + inactive_organization = OrganizationFactory() + RegistryMembershipPeriodFactory( + registry=opk_registry, + organization=opk_organization, + ) + RegistryMembershipPeriodFactory( + registry=rosatom_registry, + organization=opk_organization, + ) + RegistryMembershipPeriodFactory( + registry=rosatom_registry, + organization=shared_organization, + ) + RegistryMembershipPeriodFactory( + registry=roscosmos_goz_registry, + organization=inactive_organization, + ended_at=date(2026, 6, 1), + ) + + response = self.client.get("/api/v1/stat/organizations/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + payload = response.data["data"] + self.assertEqual(payload["total_organizations"], 3) + self.assertEqual(payload["active_registry_organizations"], 2) + self.assertEqual(payload["counts"]["total"], 3) + self.assertEqual(payload["counts"]["opk"], 1) + self.assertEqual(payload["counts"]["rosatom"], 2) + self.assertEqual(payload["counts"]["roscosmos-goz"], 0) + cards_by_slug = {item["slug"]: item for item in payload["cards"]} + self.assertEqual( + cards_by_slug["total"]["title"], + "Общее количество организаций", + ) + self.assertEqual( + cards_by_slug["rosatom"]["registry_name"], + "Реестр госкорпорации Росатом", + ) + def test_v2_registry_slug_upload_does_not_refresh_snapshots_after_import_error(self): rows = [ {