Add organization stats endpoint
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 24s
CI/CD Pipeline / Build and Push Images (push) Successful in 10s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s

This commit is contained in:
2026-05-12 17:48:54 +02:00
parent ab3e672a8d
commit 89607356b7
6 changed files with 342 additions and 13 deletions

View File

@@ -69,7 +69,7 @@ from apps.registers.models import RegistryMembershipPeriod
from django.core.files.storage import default_storage
from django.core.paginator import Paginator
from django.db.models import CharField, Count, Q
from django.db.models.functions import Cast
from django.db.models.functions import Cast, Lower
from django.http import HttpResponse
from django.utils.text import get_valid_filename
from django.views.generic import TemplateView
@@ -1452,7 +1452,10 @@ def _save_uploaded_parser_file(uploaded_file) -> str:
def _model_payload(record) -> dict:
payload = {}
for field in record._meta.fields:
value = getattr(record, field.name)
if getattr(field, "many_to_one", False):
value = getattr(record, field.attname)
else:
value = getattr(record, field.name)
if hasattr(value, "isoformat"):
value = value.isoformat()
payload[field.name] = value
@@ -1502,13 +1505,20 @@ def _native_record_to_result(
inn = record.customer_inn
ogrn = record.customer_ogrn
elif source == ParserLoadLog.Source.FNS_REPORTS:
registry_organization = record.registry_organization
external_id = record.external_id
organisation_name = ""
organisation_name = (
registry_organization.pn_name if registry_organization else ""
)
title = record.file_name
record_date = ""
status_value = record.status
url = ""
inn = ""
inn = (
str(registry_organization.mn_inn)
if registry_organization and registry_organization.mn_inn
else ""
)
ogrn = record.ogrn
else:
external_id = record.registration_number
@@ -2027,14 +2037,34 @@ def source_result_swagger_tag(source_key: str) -> str:
return SOURCE_RESULT_TAGS.get(source_key, PARSERS_TAG)
def _safe_ordering(ordering: str, field_map: dict[str, str]) -> list[str]:
CASE_INSENSITIVE_RESULT_ORDERING_FIELDS = {
"external_id",
"organisation_name",
"title",
"record_date",
"status",
"url",
}
def _safe_ordering(
ordering: str,
field_map: dict[str, str],
*,
case_insensitive_fields: set[str] | None = None,
) -> list:
case_insensitive_fields = case_insensitive_fields or set()
result = []
for raw_field in (item.strip() for item in ordering.split(",") if item.strip()):
desc = raw_field.startswith("-")
api_field = raw_field[1:] if desc else raw_field
model_field = field_map.get(api_field)
if model_field:
result.append(f"-{model_field}" if desc else model_field)
if api_field in case_insensitive_fields:
expression = Lower(model_field)
result.append(expression.desc() if desc else expression.asc())
else:
result.append(f"-{model_field}" if desc else model_field)
return result
@@ -2076,15 +2106,21 @@ def _native_field_map(source: str) -> dict[str, str]:
"organisation_name": "customer_name",
"title": "purchase_name",
"record_date": "publish_date",
"amount": "max_price_amount",
"status": "status",
}
if source == ParserLoadLog.Source.FNS_REPORTS:
return {
**common,
"id": "id",
"load_batch": "load_batch",
"external_id": "external_id",
"inn": "registry_organization__mn_inn",
"ogrn": "ogrn",
"organisation_name": "registry_organization__pn_name",
"title": "file_name",
"status": "status",
"created_at": "created_at",
"updated_at": "updated_at",
}
return {
**common,
@@ -2131,6 +2167,7 @@ def _native_search_q(source: str, search: str) -> Q:
Q(file_name__icontains=search)
| Q(external_id__icontains=search)
| Q(ogrn__icontains=search)
| Q(registry_organization__pn_name__icontains=search)
| Q(status__icontains=search)
)
return (
@@ -2155,6 +2192,21 @@ def _generic_search_q(search: str) -> Q:
)
def _apply_native_search(queryset, source: str, search: str):
if source != ParserLoadLog.Source.FNS_REPORTS:
return queryset.filter(_native_search_q(source, search))
return queryset.annotate(
registry_organization_inn_text=Cast(
"registry_organization__mn_inn",
output_field=CharField(),
),
).filter(
_native_search_q(source, search)
| Q(registry_organization_inn_text__icontains=search)
)
def _route_model_sources(descriptor) -> set[str]:
return {
item.source
@@ -2178,6 +2230,8 @@ def _result_sources_for_request(descriptor, params: dict) -> set[str]:
def _filter_native_result_queryset(source: str, params: dict, sources: set[str]):
queryset = NATIVE_RECORD_MODELS[source].objects.all()
if source == ParserLoadLog.Source.FNS_REPORTS:
queryset = queryset.select_related("registry_organization")
if not sources:
queryset = queryset.none()
field_map = _native_field_map(source)
@@ -2189,8 +2243,12 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str])
if params.get("record_date") and field_map.get("record_date"):
queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]})
if params.get("search"):
queryset = queryset.filter(_native_search_q(source, params["search"]))
ordering = _safe_ordering(params.get("ordering") or "-created_at", field_map)
queryset = _apply_native_search(queryset, source, params["search"])
ordering = _safe_ordering(
params.get("ordering") or "-created_at",
field_map,
case_insensitive_fields=CASE_INSENSITIVE_RESULT_ORDERING_FIELDS,
)
return queryset.order_by(*(ordering or ["-created_at"]))
@@ -2215,11 +2273,17 @@ def _filter_generic_result_queryset(sources: set[str], params: dict):
"organisation_name": "organisation_name",
"title": "title",
"record_date": "record_date",
"amount": "amount",
"status": "status",
"url": "url",
"created_at": "created_at",
"updated_at": "updated_at",
}
ordering = _safe_ordering(params.get("ordering") or "-created_at", field_map)
ordering = _safe_ordering(
params.get("ordering") or "-created_at",
field_map,
case_insensitive_fields=CASE_INSENSITIVE_RESULT_ORDERING_FIELDS,
)
return queryset.order_by(*(ordering or ["-created_at"]))

View File

@@ -35,7 +35,7 @@ from apps.parsers.urls import (
zakupki_urlpatterns,
)
from django.urls import include, path
from registers.urls import registers_urlpatterns
from registers.urls import registers_urlpatterns, stat_urlpatterns
app_name = "api_v1"
@@ -66,6 +66,8 @@ urlpatterns = [
path("fns/", include((fns_urlpatterns, "fns"))),
# Результаты новых источников без перекрытия старых API выше
path("", include("apps.parsers.api_result_urls", namespace="parser_results")),
# Сводные frontend-счетчики
path("stat/", include((stat_urlpatterns, "stat"))),
# Управление parser Celery задачами и dashboard data
path("parsers/", include("apps.parsers.urls")),
# Агрегированные карточки источников для фронтенда

View File

@@ -10,6 +10,7 @@ from registers.views import (
RegisterUploadView,
RegisterViewSet,
RegistryOrganizationListView,
RegistryStatsView,
)
app_name = "registers"
@@ -45,4 +46,8 @@ registers_v2_urlpatterns = [
path("", include(router.urls)),
]
stat_urlpatterns = [
path("organizations/", RegistryStatsView.as_view(), name="organization-summary"),
]
urlpatterns = []

View File

@@ -3,6 +3,7 @@
from __future__ import annotations
from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag
from apps.core.response import api_response
from django.db.models import Count, Q
from django.shortcuts import get_object_or_404
from drf_yasg import openapi
@@ -17,7 +18,7 @@ from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework.viewsets import ReadOnlyModelViewSet
from registers.models import Organization, Register
from registers.models import Organization, Register, RegistryMembershipPeriod
from registers.pagination import RegistersPagination
from registers.serializers import (
FixedRegisterFileUploadSerializer,
@@ -44,6 +45,44 @@ REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG = {
"rosatom-opk": "Реестр госкорпорации Росатом ОПК",
}
REGISTRY_STAT_CARD_DEFINITIONS = (
{
"slug": "opk",
"title": "Организации, входящие в реестр предприятий ОПК",
"registry_name": "Реестр предприятий ОПК",
},
{
"slug": "rosatom",
"title": "Организации, входящие в реестр госкорпорации Росатом",
"registry_name": "Реестр госкорпорации Росатом",
},
{
"slug": "roscosmos",
"title": "Организации, входящие в реестр госкорпорации Роскосмос",
"registry_name": "Реестр госкорпорации Роскосмос",
},
{
"slug": "rosatom-opk",
"title": "Организации, входящие в реестр госкорпорации Росатом ОПК",
"registry_name": "Реестр госкорпорации Росатом ОПК",
},
{
"slug": "rosatom-goz",
"title": "Организации, входящие в реестр госкорпорации Росатом ГОЗ",
"registry_name": "Реестр госкорпорации Росатом ГОЗ",
},
{
"slug": "roscosmos-goz",
"title": "Организации, входящие в реестр госкорпорации Роскосмос ГОЗ",
"registry_name": "Реестр госкорпорации Роскосмос ГОЗ",
},
{
"slug": "roscosmos-opk",
"title": "Организации, входящие в реестр госкорпорации Роскосмос ОПК",
"registry_name": "Реестр госкорпорации Роскосмос ОПК",
},
)
def _start_snapshot_refresh_task() -> None:
refresh_all_organization_data_snapshots.delay()
@@ -217,6 +256,70 @@ class OrganizationViewSet(ReadOnlyModelViewSet):
return super().retrieve(request, *args, **kwargs)
class RegistryStatsView(APIView):
"""Сводные счетчики организаций для frontend stats cards."""
permission_classes = [IsAuthenticated]
def get(self, request):
registry_names = [
item["registry_name"] for item in REGISTRY_STAT_CARD_DEFINITIONS
]
active_counts_by_name = {
row["registry__name"]: row["organizations_count"]
for row in (
RegistryMembershipPeriod.objects.filter(
ended_at__isnull=True,
registry__name__in=registry_names,
)
.values("registry__name")
.annotate(
organizations_count=Count("organization_id", distinct=True),
)
)
}
total_organizations = Organization.objects.count()
active_registry_organizations = (
RegistryMembershipPeriod.objects.filter(ended_at__isnull=True)
.order_by()
.values("organization_id")
.distinct()
.count()
)
cards = [
{
"slug": "total",
"title": "Общее количество организаций",
"registry_name": None,
"organizations_count": total_organizations,
"order": 0,
}
]
for order, definition in enumerate(REGISTRY_STAT_CARD_DEFINITIONS, start=10):
cards.append(
{
"slug": definition["slug"],
"title": definition["title"],
"registry_name": definition["registry_name"],
"organizations_count": active_counts_by_name.get(
definition["registry_name"],
0,
),
"order": order,
}
)
return api_response(
{
"total_organizations": total_organizations,
"active_registry_organizations": active_registry_organizations,
"counts": {item["slug"]: item["organizations_count"] for item in cards},
"cards": cards,
}
)
class RegistryOrganizationListView(ListAPIView):
"""API списка организаций конкретного реестра."""

View File

@@ -697,6 +697,112 @@ class ParsersViewSetTest(APITestCase):
self.assertEqual(unified_response.data["data"][0]["id"], report.id)
self.assertEqual(unified_response.data["data"][0]["title"], report.file_name)
def test_fns_financial_results_searches_and_orders_by_registry_organization(self):
alpha_org = RegisterOrganizationFactory(
pn_name='alpha "ФНС"',
mn_ogrn=1027700000001,
mn_inn=7701000001,
)
beta_org = RegisterOrganizationFactory(
pn_name='Beta "ФНС"',
mn_ogrn=1027700000002,
mn_inn=7701000002,
)
alpha_report = FinancialReport.objects.create(
external_id="fns-alpha",
ogrn=str(alpha_org.mn_ogrn),
registry_organization=alpha_org,
file_name=f"fin_fns-alpha_{alpha_org.mn_ogrn}.xlsx",
file_hash=fake.sha256(raw_output=False),
load_batch=1,
status=FinancialReport.Status.SUCCESS,
source=FinancialReport.SourceType.API,
)
beta_report = FinancialReport.objects.create(
external_id="fns-beta",
ogrn=str(beta_org.mn_ogrn),
registry_organization=beta_org,
file_name=f"fin_fns-beta_{beta_org.mn_ogrn}.xlsx",
file_hash=fake.sha256(raw_output=False),
load_batch=1,
status=FinancialReport.Status.SUCCESS,
source=FinancialReport.SourceType.API,
)
self.client.force_authenticate(self.user)
name_response = self.client.get(
"/api/v1/parsers/results/fns_financial/",
{"search": "alpha"},
)
inn_response = self.client.get(
"/api/v1/parsers/results/fns_financial/",
{"search": str(beta_org.mn_inn)},
)
ordered_response = self.client.get(
"/api/v1/parsers/results/fns_financial/",
{"ordering": "organisation_name"},
)
self.assertEqual(name_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in name_response.data["data"]],
[alpha_report.id],
)
self.assertEqual(
name_response.data["data"][0]["organisation_name"],
alpha_org.pn_name,
)
self.assertEqual(name_response.data["data"][0]["inn"], str(alpha_org.mn_inn))
self.assertEqual(inn_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in inn_response.data["data"]],
[beta_report.id],
)
self.assertEqual(ordered_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in ordered_response.data["data"]],
[alpha_report.id, beta_report.id],
)
def test_eis_results_order_text_case_insensitively_and_by_amount(self):
low_amount = GenericParserRecord.objects.create(
load_batch=1,
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
external_id="amount-low",
organisation_name="alpha customer",
title="Low amount",
amount="10.00",
)
high_amount = GenericParserRecord.objects.create(
load_batch=1,
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
external_id="amount-high",
organisation_name="Beta customer",
title="High amount",
amount="20.00",
)
self.client.force_authenticate(self.user)
amount_response = self.client.get(
"/api/v1/eis/procurements-44fz/",
{"ordering": "amount"},
)
name_response = self.client.get(
"/api/v1/eis/procurements-44fz/",
{"ordering": "-organisation_name"},
)
self.assertEqual(amount_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in amount_response.data["data"]],
[low_amount.id, high_amount.id],
)
self.assertEqual(name_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in name_response.data["data"]],
[high_amount.id, low_amount.id],
)
def test_system_logs_admin_only(self):
log = ParserLoadLogFactory()
url_logs = reverse("api_v1:system:parser-logs-list")

View File

@@ -6,7 +6,7 @@ import io
from datetime import date
from unittest.mock import patch
from apps.registers.models import Organization, RegistryMembershipPeriod
from apps.registers.models import Organization, Register, RegistryMembershipPeriod
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import IntegrityError
from django.urls import reverse
@@ -198,6 +198,55 @@ class RegistersViewsTest(APITestCase):
"/api/v2/registers/rosatom-goz/upload/",
)
def test_stat_organizations_endpoint_returns_registry_cards(self):
opk_registry, _ = Register.objects.get_or_create(name="Реестр предприятий ОПК")
rosatom_registry, _ = Register.objects.get_or_create(
name="Реестр госкорпорации Росатом"
)
roscosmos_goz_registry, _ = Register.objects.get_or_create(
name="Реестр госкорпорации Роскосмос ГОЗ"
)
opk_organization = OrganizationFactory()
shared_organization = OrganizationFactory()
inactive_organization = OrganizationFactory()
RegistryMembershipPeriodFactory(
registry=opk_registry,
organization=opk_organization,
)
RegistryMembershipPeriodFactory(
registry=rosatom_registry,
organization=opk_organization,
)
RegistryMembershipPeriodFactory(
registry=rosatom_registry,
organization=shared_organization,
)
RegistryMembershipPeriodFactory(
registry=roscosmos_goz_registry,
organization=inactive_organization,
ended_at=date(2026, 6, 1),
)
response = self.client.get("/api/v1/stat/organizations/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
payload = response.data["data"]
self.assertEqual(payload["total_organizations"], 3)
self.assertEqual(payload["active_registry_organizations"], 2)
self.assertEqual(payload["counts"]["total"], 3)
self.assertEqual(payload["counts"]["opk"], 1)
self.assertEqual(payload["counts"]["rosatom"], 2)
self.assertEqual(payload["counts"]["roscosmos-goz"], 0)
cards_by_slug = {item["slug"]: item for item in payload["cards"]}
self.assertEqual(
cards_by_slug["total"]["title"],
"Общее количество организаций",
)
self.assertEqual(
cards_by_slug["rosatom"]["registry_name"],
"Реестр госкорпорации Росатом",
)
def test_v2_registry_slug_upload_does_not_refresh_snapshots_after_import_error(self):
rows = [
{