fix: reuse source coverage calculations on main dashboard
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 1m3s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 1s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 1s
CI/CD Pipeline / Quality Gate (push) Successful in 49s
CI/CD Pipeline / Build and Push Images (push) Successful in 1s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 1m3s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 1s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 1s
CI/CD Pipeline / Quality Gate (push) Successful in 49s
CI/CD Pipeline / Build and Push Images (push) Successful in 1s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
This commit was merged in pull request #48.
This commit is contained in:
15
docs/dashboard-performance.md
Normal file
15
docs/dashboard-performance.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# Повторные расчеты dashboard
|
||||
|
||||
`/api/v1/parsers/dashboard/` использует один расчет соответствий организаций
|
||||
источникам для покрытия и аналитики обогащения. Последние логи также читаются
|
||||
один раз на запрос. Между запросами результаты не кешируются.
|
||||
|
||||
На одном снимке данных внутреннего main число SQL-запросов уменьшилось с 51
|
||||
до 37, время обработчика — с 1,76 до 1,14 секунды. Полные JSON-ответы до и после
|
||||
совпали. Сравнение выполнено в отдельном процессе, в транзакции REPEATABLE READ,
|
||||
READ ONLY, с фиксированным временем. Сеть и аутентификация не измерялись.
|
||||
|
||||
В этой версии main нет дорогого поиска последних фоновых задач из dev.
|
||||
Выборка карточек источников этим патчем не меняется. Миграций, новых зависимостей
|
||||
и настроек нет. Регрессия в `test_views.py` проверяет отсутствие повторных
|
||||
обходов записей источников вместе с прежними значениями аналитики.
|
||||
@@ -1986,8 +1986,9 @@ def _registry_source_matrix_rows(
|
||||
return matrix_rows
|
||||
|
||||
|
||||
def _registry_data_coverage() -> dict:
|
||||
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||
def _registry_data_coverage(
|
||||
registry_organization_ids: set[str], source_matches: dict[str, set[str]]
|
||||
) -> dict:
|
||||
total_organizations = len(registry_organization_ids)
|
||||
descriptor_titles: dict[str, str] = {}
|
||||
for descriptor in PARSER_SOURCES.values():
|
||||
@@ -1998,9 +1999,7 @@ def _registry_data_coverage() -> dict:
|
||||
for source, _label in ParserLoadLog.Source.choices:
|
||||
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
|
||||
continue
|
||||
organizations_count = len(
|
||||
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
|
||||
)
|
||||
organizations_count = len(source_matches.get(source, set()))
|
||||
coverage_percent = (
|
||||
round(organizations_count / total_organizations * 100, 1)
|
||||
if total_organizations
|
||||
@@ -2021,10 +2020,14 @@ def _registry_data_coverage() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
|
||||
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||
def _registry_enrichment_analytics(
|
||||
jobs,
|
||||
schedules: list[dict],
|
||||
registry_organization_ids: set[str],
|
||||
source_matches: dict[str, set[str]],
|
||||
latest_loads: list[ParserLoadLog],
|
||||
) -> dict:
|
||||
total_organizations = len(registry_organization_ids)
|
||||
source_matches = _source_registry_matches(by_inn, by_ogrn)
|
||||
descriptor_titles: dict[str, str] = {}
|
||||
for descriptor in PARSER_SOURCES.values():
|
||||
descriptor_titles.setdefault(descriptor.source, descriptor.title)
|
||||
@@ -2113,7 +2116,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
|
||||
enrichment_sources=enrichment_sources,
|
||||
)
|
||||
|
||||
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
|
||||
recent_success = sum(
|
||||
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
|
||||
)
|
||||
@@ -2948,6 +2950,9 @@ class ParserDashboardDataView(APIView):
|
||||
_periodic_task_to_dict(task)
|
||||
for task in _parser_periodic_tasks_for_user(request.user)
|
||||
]
|
||||
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
|
||||
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||
source_matches = _source_registry_matches(by_inn, by_ogrn)
|
||||
return api_response(
|
||||
{
|
||||
"sources": sources,
|
||||
@@ -2960,13 +2965,18 @@ class ParserDashboardDataView(APIView):
|
||||
"schedules": ParserScheduleSerializer(schedules, many=True).data,
|
||||
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
|
||||
"source_counts": source_counts,
|
||||
"registry_data_coverage": _registry_data_coverage(),
|
||||
"registry_data_coverage": _registry_data_coverage(
|
||||
registry_ids, source_matches
|
||||
),
|
||||
"registry_enrichment_analytics": _registry_enrichment_analytics(
|
||||
jobs,
|
||||
schedules,
|
||||
registry_ids,
|
||||
source_matches,
|
||||
load_logs,
|
||||
),
|
||||
"load_logs": ParserLoadLogSerializer(
|
||||
ParserLoadLog.objects.all().order_by("-created_at")[:30],
|
||||
load_logs,
|
||||
many=True,
|
||||
).data,
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ from apps.parsers.models import (
|
||||
ProcurementRecord,
|
||||
)
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from django.urls import reverse
|
||||
from openpyxl import Workbook
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
|
||||
)
|
||||
self.client.force_authenticate(self.user)
|
||||
|
||||
response = self.client.get("/api/v1/parsers/dashboard/")
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
response = self.client.get("/api/v1/parsers/dashboard/")
|
||||
|
||||
source_scans = [
|
||||
query["sql"]
|
||||
for query in queries
|
||||
if "SELECT DISTINCT" in query["sql"]
|
||||
and "organizations_source_record" in query["sql"]
|
||||
]
|
||||
self.assertTrue(source_scans)
|
||||
self.assertEqual(len(source_scans), len(set(source_scans)))
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
coverage = response.data["data"]["registry_data_coverage"]
|
||||
|
||||
Reference in New Issue
Block a user