fix: reuse registry source calculations in main dashboard #48

Merged
avm merged 1 commits from feature/mostovik-dashboard-main-performance into main 2026-10-01 14:24:52 +03:00
3 changed files with 49 additions and 12 deletions

View File

@@ -0,0 +1,15 @@
# Повторные расчеты dashboard
`/api/v1/parsers/dashboard/` использует один расчет соответствий организаций
источникам для покрытия и аналитики обогащения. Последние логи также читаются
один раз на запрос. Между запросами результаты не кешируются.
На одном снимке данных внутреннего main число SQL-запросов уменьшилось с 51
до 37, время обработчика — с 1,76 до 1,14 секунды. Полные JSON-ответы до и после
совпали. Сравнение выполнено в отдельном процессе, в транзакции REPEATABLE READ,
READ ONLY, с фиксированным временем. Сеть и аутентификация не измерялись.
В этой версии main нет дорогого поиска последних фоновых задач из dev.
Выборка карточек источников этим патчем не меняется. Миграций, новых зависимостей
и настроек нет. Регрессия в `test_views.py` проверяет отсутствие повторных
обходов записей источников вместе с прежними значениями аналитики.

View File

@@ -1986,8 +1986,9 @@ def _registry_source_matrix_rows(
return matrix_rows return matrix_rows
def _registry_data_coverage() -> dict: def _registry_data_coverage(
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes() registry_organization_ids: set[str], source_matches: dict[str, set[str]]
) -> dict:
total_organizations = len(registry_organization_ids) total_organizations = len(registry_organization_ids)
descriptor_titles: dict[str, str] = {} descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values(): for descriptor in PARSER_SOURCES.values():
@@ -1998,9 +1999,7 @@ def _registry_data_coverage() -> dict:
for source, _label in ParserLoadLog.Source.choices: for source, _label in ParserLoadLog.Source.choices:
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES: if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
continue continue
organizations_count = len( organizations_count = len(source_matches.get(source, set()))
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
)
coverage_percent = ( coverage_percent = (
round(organizations_count / total_organizations * 100, 1) round(organizations_count / total_organizations * 100, 1)
if total_organizations if total_organizations
@@ -2021,10 +2020,14 @@ def _registry_data_coverage() -> dict:
} }
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict: def _registry_enrichment_analytics(
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes() jobs,
schedules: list[dict],
registry_organization_ids: set[str],
source_matches: dict[str, set[str]],
latest_loads: list[ParserLoadLog],
) -> dict:
total_organizations = len(registry_organization_ids) total_organizations = len(registry_organization_ids)
source_matches = _source_registry_matches(by_inn, by_ogrn)
descriptor_titles: dict[str, str] = {} descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values(): for descriptor in PARSER_SOURCES.values():
descriptor_titles.setdefault(descriptor.source, descriptor.title) descriptor_titles.setdefault(descriptor.source, descriptor.title)
@@ -2113,7 +2116,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
enrichment_sources=enrichment_sources, enrichment_sources=enrichment_sources,
) )
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
recent_success = sum( recent_success = sum(
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS 1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
) )
@@ -2948,6 +2950,9 @@ class ParserDashboardDataView(APIView):
_periodic_task_to_dict(task) _periodic_task_to_dict(task)
for task in _parser_periodic_tasks_for_user(request.user) for task in _parser_periodic_tasks_for_user(request.user)
] ]
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
source_matches = _source_registry_matches(by_inn, by_ogrn)
return api_response( return api_response(
{ {
"sources": sources, "sources": sources,
@@ -2960,13 +2965,18 @@ class ParserDashboardDataView(APIView):
"schedules": ParserScheduleSerializer(schedules, many=True).data, "schedules": ParserScheduleSerializer(schedules, many=True).data,
"jobs": BackgroundJobListSerializer(jobs, many=True).data, "jobs": BackgroundJobListSerializer(jobs, many=True).data,
"source_counts": source_counts, "source_counts": source_counts,
"registry_data_coverage": _registry_data_coverage(), "registry_data_coverage": _registry_data_coverage(
registry_ids, source_matches
),
"registry_enrichment_analytics": _registry_enrichment_analytics( "registry_enrichment_analytics": _registry_enrichment_analytics(
jobs, jobs,
schedules, schedules,
registry_ids,
source_matches,
load_logs,
), ),
"load_logs": ParserLoadLogSerializer( "load_logs": ParserLoadLogSerializer(
ParserLoadLog.objects.all().order_by("-created_at")[:30], load_logs,
many=True, many=True,
).data, ).data,
} }

View File

@@ -20,6 +20,8 @@ from apps.parsers.models import (
ProcurementRecord, ProcurementRecord,
) )
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import connection
from django.test.utils import CaptureQueriesContext
from django.urls import reverse from django.urls import reverse
from openpyxl import Workbook from openpyxl import Workbook
from organizations.models import Organization, OrganizationSourceRecord from organizations.models import Organization, OrganizationSourceRecord
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
) )
self.client.force_authenticate(self.user) self.client.force_authenticate(self.user)
response = self.client.get("/api/v1/parsers/dashboard/") with CaptureQueriesContext(connection) as queries:
response = self.client.get("/api/v1/parsers/dashboard/")
source_scans = [
query["sql"]
for query in queries
if "SELECT DISTINCT" in query["sql"]
and "organizations_source_record" in query["sql"]
]
self.assertTrue(source_scans)
self.assertEqual(len(source_scans), len(set(source_scans)))
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
coverage = response.data["data"]["registry_data_coverage"] coverage = response.data["data"]["registry_data_coverage"]