fix: reuse source coverage calculations on main dashboard
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 1m3s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 1s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 1s
CI/CD Pipeline / Quality Gate (push) Successful in 49s
CI/CD Pipeline / Build and Push Images (push) Successful in 1s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s

This commit was merged in pull request #48.
This commit is contained in:
Aleksandr Meshchryakov
2026-10-01 13:21:52 +02:00
parent bad64fe001
commit e712494c3f
3 changed files with 49 additions and 12 deletions

View File

@@ -0,0 +1,15 @@
# Повторные расчеты dashboard
`/api/v1/parsers/dashboard/` использует один расчет соответствий организаций
источникам для покрытия и аналитики обогащения. Последние логи также читаются
один раз на запрос. Между запросами результаты не кешируются.
На одном снимке данных внутреннего main число SQL-запросов уменьшилось с 51
до 37, время обработчика — с 1,76 до 1,14 секунды. Полные JSON-ответы до и после
совпали. Сравнение выполнено в отдельном процессе, в транзакции REPEATABLE READ,
READ ONLY, с фиксированным временем. Сеть и аутентификация не измерялись.
В этой версии main нет дорогого поиска последних фоновых задач из dev.
Выборка карточек источников этим патчем не меняется. Миграций, новых зависимостей
и настроек нет. Регрессия в `test_views.py` проверяет отсутствие повторных
обходов записей источников вместе с прежними значениями аналитики.

View File

@@ -1986,8 +1986,9 @@ def _registry_source_matrix_rows(
return matrix_rows
def _registry_data_coverage() -> dict:
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
def _registry_data_coverage(
registry_organization_ids: set[str], source_matches: dict[str, set[str]]
) -> dict:
total_organizations = len(registry_organization_ids)
descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values():
@@ -1998,9 +1999,7 @@ def _registry_data_coverage() -> dict:
for source, _label in ParserLoadLog.Source.choices:
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
continue
organizations_count = len(
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
)
organizations_count = len(source_matches.get(source, set()))
coverage_percent = (
round(organizations_count / total_organizations * 100, 1)
if total_organizations
@@ -2021,10 +2020,14 @@ def _registry_data_coverage() -> dict:
}
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
def _registry_enrichment_analytics(
jobs,
schedules: list[dict],
registry_organization_ids: set[str],
source_matches: dict[str, set[str]],
latest_loads: list[ParserLoadLog],
) -> dict:
total_organizations = len(registry_organization_ids)
source_matches = _source_registry_matches(by_inn, by_ogrn)
descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values():
descriptor_titles.setdefault(descriptor.source, descriptor.title)
@@ -2113,7 +2116,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
enrichment_sources=enrichment_sources,
)
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
recent_success = sum(
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
)
@@ -2948,6 +2950,9 @@ class ParserDashboardDataView(APIView):
_periodic_task_to_dict(task)
for task in _parser_periodic_tasks_for_user(request.user)
]
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
source_matches = _source_registry_matches(by_inn, by_ogrn)
return api_response(
{
"sources": sources,
@@ -2960,13 +2965,18 @@ class ParserDashboardDataView(APIView):
"schedules": ParserScheduleSerializer(schedules, many=True).data,
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
"source_counts": source_counts,
"registry_data_coverage": _registry_data_coverage(),
"registry_data_coverage": _registry_data_coverage(
registry_ids, source_matches
),
"registry_enrichment_analytics": _registry_enrichment_analytics(
jobs,
schedules,
registry_ids,
source_matches,
load_logs,
),
"load_logs": ParserLoadLogSerializer(
ParserLoadLog.objects.all().order_by("-created_at")[:30],
load_logs,
many=True,
).data,
}

View File

@@ -20,6 +20,8 @@ from apps.parsers.models import (
ProcurementRecord,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import connection
from django.test.utils import CaptureQueriesContext
from django.urls import reverse
from openpyxl import Workbook
from organizations.models import Organization, OrganizationSourceRecord
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
)
self.client.force_authenticate(self.user)
response = self.client.get("/api/v1/parsers/dashboard/")
with CaptureQueriesContext(connection) as queries:
response = self.client.get("/api/v1/parsers/dashboard/")
source_scans = [
query["sql"]
for query in queries
if "SELECT DISTINCT" in query["sql"]
and "organizations_source_record" in query["sql"]
]
self.assertTrue(source_scans)
self.assertEqual(len(source_scans), len(set(source_scans)))
self.assertEqual(response.status_code, status.HTTP_200_OK)
coverage = response.data["data"]["registry_data_coverage"]