diff --git a/docs/dashboard-performance.md b/docs/dashboard-performance.md new file mode 100644 index 0000000..482190b --- /dev/null +++ b/docs/dashboard-performance.md @@ -0,0 +1,52 @@ +# Производительность главной страницы + +Главная страница использует `/api/v1/sources/` и `/api/v1/parsers/dashboard/`. +Счетчики, состав задач, порядок запусков и поля ответов остаются прежними. + +## Изменения + +- PostgreSQL выбирает последнюю фоновую задачу для каждого `task_name` один раз + через `DISTINCT ON`. Сортировка сохраняется: `created_at DESC, id DESC`. + Выбор последних задач выполняется до фильтрации карточки; более старые + активные задачи и завершенные части составного запуска остаются видимыми. + Для SQLite сохранен совместимый вариант с подзапросом. +- Аналитика повторно использует соответствия организаций источникам и последние + логи в пределах одного HTTP-запроса. Между пользователями результаты не кешируются. +- Метаданные снимка используют сохраненный `snapshot_organizations_count`, + включая ноль. Для старых снимков без допустимого итога выполняется один + групповой подсчет уникальных опубликованных организаций. Отклоненные строки + и строки без организации в итог не входят. Подсчет нужен только для выбранной + последней версии снимка каждого запуска. + +Новых настроек, миграций и правил инвалидирования кеша нет. + +## Проверка на данных dev, 1 октября 2026 + +| Обработчик | Кеш агрегатов | До | После | +| --- | --- | ---: | ---: | +| `/sources/` | Пустой | 13,200 с | 3,453 с | +| `/sources/` | Заполненный | 10,191 с | 0,243 с | +| `/parsers/dashboard/` | Пустой | 4,165 с | 2,864 с | +| `/parsers/dashboard/` | Заполненный | 4,246 с | 2,880 с | + +Это время обработчиков в отдельном процессе, без сети и аутентификационного +middleware. Сравнение выполнено на одном снимке БД в транзакции +`REPEATABLE READ, READ ONLY`, с фиксированным временем и отдельным кешем в памяти. +Полные JSON-ответы совпали во всех четырех сравнениях. Учетные данные +пользователей не использовались. Это диагностическое сравнение, не нагрузочный +тест и не подтверждение выкатки. + +Оставшееся время первого запроса связано с подсчетом записей источников. +Dashboard по-прежнему рассчитывает актуальные счетчики при каждом запросе. + +## Регрессионные проверки + +- `test_refresh_progress.py`: порядок по времени и UUID, старые активные задачи, + параллельные и составные запуски, глобальный выбор последней задачи. +- `test_snapshot_metadata.py`: сохраненные и старые счетчики, нулевой итог, + повторные организации, неопубликованные строки, выбор последнего снимка. +- `test_views.py`: значения аналитики и отсутствие повторных обходов источников. +- `test_refresh_concurrency.py`: проверка блокировок на PostgreSQL. + +Проверки PostgreSQL запускаются с `--ds=settings.test_postgres --migrations` +и отдельной тестовой БД. SQLite не проверяет `DISTINCT ON` и блокировки PostgreSQL. diff --git a/src/apps/parsers/snapshot_metadata.py b/src/apps/parsers/snapshot_metadata.py index 4679fc8..2777220 100644 --- a/src/apps/parsers/snapshot_metadata.py +++ b/src/apps/parsers/snapshot_metadata.py @@ -1,33 +1,41 @@ """Public snapshot provenance shared by cards, dashboard and parser history.""" -from apps.parsers.models import ParserSourceArtifact -from django.db.models import Count, Q +from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord +from django.db.models import Count def attach_snapshot_artifacts(logs) -> None: - """Attach the latest artifact for each load in one query, without raw data.""" + """Attach provenance, counting staged rows only for legacy snapshots.""" logs = list(logs) if not logs: return artifacts = {} - for artifact in ( - ParserSourceArtifact.objects.filter( - source__in={log.source for log in logs}, - load_batch__in={log.batch_id for log in logs}, - ) - .annotate( - published_organizations_count=Count( - "staged_records__organization_id", - distinct=True, - filter=Q(staged_records__disposition="published"), - ) - ) - .order_by("-created_at", "-uid") - ): + for artifact in ParserSourceArtifact.objects.filter( + source__in={log.source for log in logs}, + load_batch__in={log.batch_id for log in logs}, + ).order_by("-created_at", "-uid"): + artifacts.setdefault((artifact.source, artifact.load_batch), artifact) + legacy_artifacts = [] + for artifact in artifacts.values(): total = artifact.metadata.get("snapshot_organizations_count") if isinstance(total, int) and total >= 0: artifact.published_organizations_count = total - artifacts.setdefault((artifact.source, artifact.load_batch), artifact) + else: + legacy_artifacts.append(artifact.pk) + counts = {} + if legacy_artifacts: + counts = dict( + ParserStagedRecord.objects.filter( + artifact_id__in=legacy_artifacts, disposition="published" + ) + .order_by() + .values("artifact_id") + .annotate(total=Count("organization_id", distinct=True)) + .values_list("artifact_id", "total") + ) + for artifact in artifacts.values(): + if artifact.pk in legacy_artifacts: + artifact.published_organizations_count = counts.get(artifact.pk, 0) for log in logs: log._source_artifact_cache = artifacts.get((log.source, log.batch_id)) diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index f655948..53668a2 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -27,7 +27,7 @@ from apps.parsers.snapshot_metadata import ( from apps.parsers.source_registry import get_source_by_model_source from django.conf import settings from django.core.cache import cache -from django.db import transaction +from django.db import connections, transaction from django.db.models import Count, Max, OuterRef, Q, Subquery from django.http import Http404 from django.utils import timezone @@ -1033,19 +1033,28 @@ class SourceCardService: for task_name in definition.task_names: slugs_by_task_name.setdefault(task_name, []).append(definition.slug) slugs = {definition.slug for definition in definitions} - latest = ( - BackgroundJobService.get_queryset() - .filter(task_name=OuterRef("task_name")) - .order_by("-created_at", "-id") - .values("pk")[:1] - ) + all_jobs = BackgroundJobService.get_queryset() + if connections[all_jobs.db].features.can_distinct_on_fields: + # Find each task's latest job once, rather than once per history row. + latest = all_jobs.order_by("task_name", "-created_at", "-id").distinct( + "task_name" + ) + else: + latest_for_task = ( + all_jobs.filter(task_name=OuterRef("task_name")) + .order_by("-created_at", "-id") + .values("pk")[:1] + ) + latest = all_jobs.filter(pk=Subquery(latest_for_task)).order_by() jobs = list( BackgroundJobService.get_queryset() .filter( Q(task_name__in=slugs_by_task_name) | Q(meta__source_card__in=list(slugs)) ) - .filter(cls._fresh_active_job_filter() | Q(pk=Subquery(latest))) + .filter( + cls._fresh_active_job_filter() | Q(pk__in=Subquery(latest.values("pk"))) + ) .order_by("-created_at", "-id") ) active_by_slug: dict[str, list[dict[str, Any]]] = {} diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 0b877dc..72b3d39 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -2242,8 +2242,9 @@ def _registry_source_matrix_rows( return matrix_rows -def _registry_data_coverage() -> dict: - registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes() +def _registry_data_coverage( + registry_organization_ids: set[str], source_matches: dict[str, set[str]] +) -> dict: total_organizations = len(registry_organization_ids) descriptor_titles: dict[str, str] = {} for descriptor in PARSER_SOURCES.values(): @@ -2254,9 +2255,7 @@ def _registry_data_coverage() -> dict: for source, _label in ParserLoadLog.Source.choices: if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES: continue - organizations_count = len( - _matched_source_record_registry_organization_ids(source, by_inn, by_ogrn) - ) + organizations_count = len(source_matches.get(source, set())) coverage_percent = ( round(organizations_count / total_organizations * 100, 1) if total_organizations @@ -2277,10 +2276,14 @@ def _registry_data_coverage() -> dict: } -def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict: - registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes() +def _registry_enrichment_analytics( + jobs, + schedules: list[dict], + registry_organization_ids: set[str], + source_matches: dict[str, set[str]], + latest_loads: list[ParserLoadLog], +) -> dict: total_organizations = len(registry_organization_ids) - source_matches = _source_registry_matches(by_inn, by_ogrn) descriptor_titles: dict[str, str] = {} for descriptor in PARSER_SOURCES.values(): descriptor_titles.setdefault(descriptor.source, descriptor.title) @@ -2369,7 +2372,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict: enrichment_sources=enrichment_sources, ) - latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30]) recent_success = sum( 1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS ) @@ -3459,6 +3461,8 @@ class ParserDashboardDataView(APIView): ] load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30]) attach_snapshot_artifacts(load_logs) + registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes() + source_matches = _source_registry_matches(by_inn, by_ogrn) return api_response( { "sources": sources, @@ -3471,10 +3475,15 @@ class ParserDashboardDataView(APIView): "schedules": ParserScheduleSerializer(schedules, many=True).data, "jobs": BackgroundJobListSerializer(jobs, many=True).data, "source_counts": source_counts, - "registry_data_coverage": _registry_data_coverage(), + "registry_data_coverage": _registry_data_coverage( + registry_ids, source_matches + ), "registry_enrichment_analytics": _registry_enrichment_analytics( jobs, schedules, + registry_ids, + source_matches, + load_logs, ), "load_logs": ParserLoadLogSerializer( load_logs, diff --git a/tests/apps/parsers/test_refresh_progress.py b/tests/apps/parsers/test_refresh_progress.py index 88d0c8e..43f5bbe 100644 --- a/tests/apps/parsers/test_refresh_progress.py +++ b/tests/apps/parsers/test_refresh_progress.py @@ -2,6 +2,7 @@ from datetime import timedelta from unittest.mock import MagicMock +from uuid import UUID import pytest from apps.core.models import BackgroundJob, JobStatus @@ -145,3 +146,53 @@ def test_latest_task_does_not_reactivate_a_stale_job(status, time_field): **{time_field: timezone.now() - timedelta(minutes=15)} ) assert SourceCardService.get_card(definition.slug)["active_tasks"] == [] + + +@pytest.mark.django_db +def test_latest_job_uses_created_time_then_uuid_and_keeps_older_active_jobs(): + definition = SourceCardService.get_definition("arbitration-cases") + jobs = [ + BackgroundJob.objects.create( + id=UUID(int=index), + task_id=f"ordered-{index}", + task_name=definition.task_names[0], + status=JobStatus.SUCCESS, + ) + for index in (1, 2, 3, 4) + ] + now = timezone.now() + BackgroundJob.objects.filter(pk__in=[job.pk for job in jobs]).update( + created_at=now - timedelta(minutes=2) + ) + # UUID 4 is older; UUID 2 wins the timestamp tie with UUID 1. + BackgroundJob.objects.filter(pk__in=[jobs[0].pk, jobs[1].pk]).update(created_at=now) + BackgroundJob.objects.filter(pk=jobs[2].pk).update(status=JobStatus.STARTED) + active, refresh = SourceCardService._build_task_maps((definition,)) + assert [job["task_id"] for job in active[definition.slug]] == ["ordered-3"] + assert [job["task_id"] for job in refresh[definition.slug]] == [ + "ordered-2", + "ordered-3", + ] + + +@pytest.mark.django_db +def test_latest_job_is_chosen_before_filtering_source_card_metadata(): + definition = SourceCardService.get_definition("arbitration-cases") + older = BackgroundJob.objects.create( + task_id="old-card-job", + task_name="custom.shared.task", + status=JobStatus.SUCCESS, + meta={"source_card": definition.slug}, + ) + newer = BackgroundJob.objects.create( + task_id="new-other-card-job", + task_name=older.task_name, + status=JobStatus.SUCCESS, + meta={"source_card": "bankruptcy-procedures"}, + ) + BackgroundJob.objects.filter(pk=older.pk).update( + created_at=newer.created_at - timedelta(minutes=1) + ) + active, refresh = SourceCardService._build_task_maps((definition,)) + assert active == {} + assert refresh == {} diff --git a/tests/apps/parsers/test_snapshot_metadata.py b/tests/apps/parsers/test_snapshot_metadata.py new file mode 100644 index 0000000..128b825 --- /dev/null +++ b/tests/apps/parsers/test_snapshot_metadata.py @@ -0,0 +1,61 @@ +"""Snapshot counts avoid rescanning staged data when a saved total exists.""" + +from datetime import timedelta + +import pytest +from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord +from apps.parsers.snapshot_metadata import attach_snapshot_artifacts +from django.db import connection +from django.test.utils import CaptureQueriesContext +from organizations.models import Organization + + +@pytest.mark.django_db +@pytest.mark.parametrize("saved_total", [0, 7, None, -1, "7"]) +def test_saved_total_or_distinct_published_organizations(saved_total): + log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1) + artifact = ParserSourceArtifact.objects.create( + source=log.source, + load_batch=log.batch_id, + metadata={"snapshot_organizations_count": saved_total}, + ) + organization = Organization.objects.create(name="Published fixture") + for row, disposition in enumerate(["published", "published", "quarantined"]): + ParserStagedRecord.objects.create( + artifact=artifact, + row_number=row + 1, + organization=organization, + disposition=disposition, + ) + ParserStagedRecord.objects.create( + artifact=artifact, row_number=4, disposition="published", organization=None + ) + with CaptureQueriesContext(connection) as queries: + attach_snapshot_artifacts([log, log]) + cached = log._source_artifact_cache + assert cached.pk == artifact.pk + has_saved_total = isinstance(saved_total, int) and saved_total >= 0 + assert cached.published_organizations_count == ( + saved_total if has_saved_total else 1 + ) + staged_queries = [q for q in queries if "parsers_staged_record" in q["sql"]] + assert len(staged_queries) == (0 if has_saved_total else 1) + + +@pytest.mark.django_db +def test_latest_artifact_and_missing_load_keep_their_own_counts(): + log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1) + missing = ParserLoadLog.objects.create(source=log.source, batch_id=2) + old = ParserSourceArtifact.objects.create(source=log.source, load_batch=1) + latest = ParserSourceArtifact.objects.create( + source=log.source, load_batch=1, metadata={"snapshot_organizations_count": 9} + ) + ParserSourceArtifact.objects.filter(pk=old.pk).update( + created_at=latest.created_at - timedelta(seconds=1) + ) + with CaptureQueriesContext(connection) as queries: + attach_snapshot_artifacts([log, missing]) + assert log._source_artifact_cache.pk == latest.pk + assert log._source_artifact_cache.published_organizations_count == 9 + assert missing._source_artifact_cache is None + assert not any("parsers_staged_record" in q["sql"] for q in queries) diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 85c21b1..e3ca321 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -20,6 +20,8 @@ from apps.parsers.models import ( ProcurementRecord, ) from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import connection +from django.test.utils import CaptureQueriesContext from django.urls import reverse from openpyxl import Workbook from organizations.models import Organization, OrganizationSourceRecord @@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase): ) self.client.force_authenticate(self.user) - response = self.client.get("/api/v1/parsers/dashboard/") + with CaptureQueriesContext(connection) as queries: + response = self.client.get("/api/v1/parsers/dashboard/") + + source_scans = [ + query["sql"] + for query in queries + if "SELECT DISTINCT" in query["sql"] + and "organizations_source_record" in query["sql"] + ] + self.assertTrue(source_scans) + self.assertEqual(len(source_scans), len(set(source_scans))) self.assertEqual(response.status_code, status.HTTP_200_OK) coverage = response.data["data"]["registry_data_coverage"]