fix: avoid repeated task and source scans on dashboard
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit was merged in pull request #47.
This commit is contained in:
Aleksandr Meshchryakov
2026-10-01 13:16:11 +02:00
parent 887442e8e3
commit fedd2e00a3
7 changed files with 239 additions and 37 deletions

View File

@@ -0,0 +1,52 @@
# Производительность главной страницы
Главная страница использует `/api/v1/sources/` и `/api/v1/parsers/dashboard/`.
Счетчики, состав задач, порядок запусков и поля ответов остаются прежними.
## Изменения
- PostgreSQL выбирает последнюю фоновую задачу для каждого `task_name` один раз
через `DISTINCT ON`. Сортировка сохраняется: `created_at DESC, id DESC`.
Выбор последних задач выполняется до фильтрации карточки; более старые
активные задачи и завершенные части составного запуска остаются видимыми.
Для SQLite сохранен совместимый вариант с подзапросом.
- Аналитика повторно использует соответствия организаций источникам и последние
логи в пределах одного HTTP-запроса. Между пользователями результаты не кешируются.
- Метаданные снимка используют сохраненный `snapshot_organizations_count`,
включая ноль. Для старых снимков без допустимого итога выполняется один
групповой подсчет уникальных опубликованных организаций. Отклоненные строки
и строки без организации в итог не входят. Подсчет нужен только для выбранной
последней версии снимка каждого запуска.
Новых настроек, миграций и правил инвалидирования кеша нет.
## Проверка на данных dev, 1 октября 2026
| Обработчик | Кеш агрегатов | До | После |
| --- | --- | ---: | ---: |
| `/sources/` | Пустой | 13,200 с | 3,453 с |
| `/sources/` | Заполненный | 10,191 с | 0,243 с |
| `/parsers/dashboard/` | Пустой | 4,165 с | 2,864 с |
| `/parsers/dashboard/` | Заполненный | 4,246 с | 2,880 с |
Это время обработчиков в отдельном процессе, без сети и аутентификационного
middleware. Сравнение выполнено на одном снимке БД в транзакции
`REPEATABLE READ, READ ONLY`, с фиксированным временем и отдельным кешем в памяти.
Полные JSON-ответы совпали во всех четырех сравнениях. Учетные данные
пользователей не использовались. Это диагностическое сравнение, не нагрузочный
тест и не подтверждение выкатки.
Оставшееся время первого запроса связано с подсчетом записей источников.
Dashboard по-прежнему рассчитывает актуальные счетчики при каждом запросе.
## Регрессионные проверки
- `test_refresh_progress.py`: порядок по времени и UUID, старые активные задачи,
параллельные и составные запуски, глобальный выбор последней задачи.
- `test_snapshot_metadata.py`: сохраненные и старые счетчики, нулевой итог,
повторные организации, неопубликованные строки, выбор последнего снимка.
- `test_views.py`: значения аналитики и отсутствие повторных обходов источников.
- `test_refresh_concurrency.py`: проверка блокировок на PostgreSQL.
Проверки PostgreSQL запускаются с `--ds=settings.test_postgres --migrations`
и отдельной тестовой БД. SQLite не проверяет `DISTINCT ON` и блокировки PostgreSQL.

View File

@@ -1,33 +1,41 @@
"""Public snapshot provenance shared by cards, dashboard and parser history."""
from apps.parsers.models import ParserSourceArtifact
from django.db.models import Count, Q
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from django.db.models import Count
def attach_snapshot_artifacts(logs) -> None:
"""Attach the latest artifact for each load in one query, without raw data."""
"""Attach provenance, counting staged rows only for legacy snapshots."""
logs = list(logs)
if not logs:
return
artifacts = {}
for artifact in (
ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
)
.annotate(
published_organizations_count=Count(
"staged_records__organization_id",
distinct=True,
filter=Q(staged_records__disposition="published"),
)
)
.order_by("-created_at", "-uid")
):
for artifact in ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
).order_by("-created_at", "-uid"):
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
legacy_artifacts = []
for artifact in artifacts.values():
total = artifact.metadata.get("snapshot_organizations_count")
if isinstance(total, int) and total >= 0:
artifact.published_organizations_count = total
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
else:
legacy_artifacts.append(artifact.pk)
counts = {}
if legacy_artifacts:
counts = dict(
ParserStagedRecord.objects.filter(
artifact_id__in=legacy_artifacts, disposition="published"
)
.order_by()
.values("artifact_id")
.annotate(total=Count("organization_id", distinct=True))
.values_list("artifact_id", "total")
)
for artifact in artifacts.values():
if artifact.pk in legacy_artifacts:
artifact.published_organizations_count = counts.get(artifact.pk, 0)
for log in logs:
log._source_artifact_cache = artifacts.get((log.source, log.batch_id))

View File

@@ -27,7 +27,7 @@ from apps.parsers.snapshot_metadata import (
from apps.parsers.source_registry import get_source_by_model_source
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from django.db import connections, transaction
from django.db.models import Count, Max, OuterRef, Q, Subquery
from django.http import Http404
from django.utils import timezone
@@ -1033,19 +1033,28 @@ class SourceCardService:
for task_name in definition.task_names:
slugs_by_task_name.setdefault(task_name, []).append(definition.slug)
slugs = {definition.slug for definition in definitions}
latest = (
BackgroundJobService.get_queryset()
.filter(task_name=OuterRef("task_name"))
.order_by("-created_at", "-id")
.values("pk")[:1]
)
all_jobs = BackgroundJobService.get_queryset()
if connections[all_jobs.db].features.can_distinct_on_fields:
# Find each task's latest job once, rather than once per history row.
latest = all_jobs.order_by("task_name", "-created_at", "-id").distinct(
"task_name"
)
else:
latest_for_task = (
all_jobs.filter(task_name=OuterRef("task_name"))
.order_by("-created_at", "-id")
.values("pk")[:1]
)
latest = all_jobs.filter(pk=Subquery(latest_for_task)).order_by()
jobs = list(
BackgroundJobService.get_queryset()
.filter(
Q(task_name__in=slugs_by_task_name)
| Q(meta__source_card__in=list(slugs))
)
.filter(cls._fresh_active_job_filter() | Q(pk=Subquery(latest)))
.filter(
cls._fresh_active_job_filter() | Q(pk__in=Subquery(latest.values("pk")))
)
.order_by("-created_at", "-id")
)
active_by_slug: dict[str, list[dict[str, Any]]] = {}

View File

@@ -2242,8 +2242,9 @@ def _registry_source_matrix_rows(
return matrix_rows
def _registry_data_coverage() -> dict:
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
def _registry_data_coverage(
registry_organization_ids: set[str], source_matches: dict[str, set[str]]
) -> dict:
total_organizations = len(registry_organization_ids)
descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values():
@@ -2254,9 +2255,7 @@ def _registry_data_coverage() -> dict:
for source, _label in ParserLoadLog.Source.choices:
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
continue
organizations_count = len(
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
)
organizations_count = len(source_matches.get(source, set()))
coverage_percent = (
round(organizations_count / total_organizations * 100, 1)
if total_organizations
@@ -2277,10 +2276,14 @@ def _registry_data_coverage() -> dict:
}
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
def _registry_enrichment_analytics(
jobs,
schedules: list[dict],
registry_organization_ids: set[str],
source_matches: dict[str, set[str]],
latest_loads: list[ParserLoadLog],
) -> dict:
total_organizations = len(registry_organization_ids)
source_matches = _source_registry_matches(by_inn, by_ogrn)
descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values():
descriptor_titles.setdefault(descriptor.source, descriptor.title)
@@ -2369,7 +2372,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
enrichment_sources=enrichment_sources,
)
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
recent_success = sum(
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
)
@@ -3459,6 +3461,8 @@ class ParserDashboardDataView(APIView):
]
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
attach_snapshot_artifacts(load_logs)
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
source_matches = _source_registry_matches(by_inn, by_ogrn)
return api_response(
{
"sources": sources,
@@ -3471,10 +3475,15 @@ class ParserDashboardDataView(APIView):
"schedules": ParserScheduleSerializer(schedules, many=True).data,
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
"source_counts": source_counts,
"registry_data_coverage": _registry_data_coverage(),
"registry_data_coverage": _registry_data_coverage(
registry_ids, source_matches
),
"registry_enrichment_analytics": _registry_enrichment_analytics(
jobs,
schedules,
registry_ids,
source_matches,
load_logs,
),
"load_logs": ParserLoadLogSerializer(
load_logs,

View File

@@ -2,6 +2,7 @@
from datetime import timedelta
from unittest.mock import MagicMock
from uuid import UUID
import pytest
from apps.core.models import BackgroundJob, JobStatus
@@ -145,3 +146,53 @@ def test_latest_task_does_not_reactivate_a_stale_job(status, time_field):
**{time_field: timezone.now() - timedelta(minutes=15)}
)
assert SourceCardService.get_card(definition.slug)["active_tasks"] == []
@pytest.mark.django_db
def test_latest_job_uses_created_time_then_uuid_and_keeps_older_active_jobs():
definition = SourceCardService.get_definition("arbitration-cases")
jobs = [
BackgroundJob.objects.create(
id=UUID(int=index),
task_id=f"ordered-{index}",
task_name=definition.task_names[0],
status=JobStatus.SUCCESS,
)
for index in (1, 2, 3, 4)
]
now = timezone.now()
BackgroundJob.objects.filter(pk__in=[job.pk for job in jobs]).update(
created_at=now - timedelta(minutes=2)
)
# UUID 4 is older; UUID 2 wins the timestamp tie with UUID 1.
BackgroundJob.objects.filter(pk__in=[jobs[0].pk, jobs[1].pk]).update(created_at=now)
BackgroundJob.objects.filter(pk=jobs[2].pk).update(status=JobStatus.STARTED)
active, refresh = SourceCardService._build_task_maps((definition,))
assert [job["task_id"] for job in active[definition.slug]] == ["ordered-3"]
assert [job["task_id"] for job in refresh[definition.slug]] == [
"ordered-2",
"ordered-3",
]
@pytest.mark.django_db
def test_latest_job_is_chosen_before_filtering_source_card_metadata():
definition = SourceCardService.get_definition("arbitration-cases")
older = BackgroundJob.objects.create(
task_id="old-card-job",
task_name="custom.shared.task",
status=JobStatus.SUCCESS,
meta={"source_card": definition.slug},
)
newer = BackgroundJob.objects.create(
task_id="new-other-card-job",
task_name=older.task_name,
status=JobStatus.SUCCESS,
meta={"source_card": "bankruptcy-procedures"},
)
BackgroundJob.objects.filter(pk=older.pk).update(
created_at=newer.created_at - timedelta(minutes=1)
)
active, refresh = SourceCardService._build_task_maps((definition,))
assert active == {}
assert refresh == {}

View File

@@ -0,0 +1,61 @@
"""Snapshot counts avoid rescanning staged data when a saved total exists."""
from datetime import timedelta
import pytest
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
from apps.parsers.snapshot_metadata import attach_snapshot_artifacts
from django.db import connection
from django.test.utils import CaptureQueriesContext
from organizations.models import Organization
@pytest.mark.django_db
@pytest.mark.parametrize("saved_total", [0, 7, None, -1, "7"])
def test_saved_total_or_distinct_published_organizations(saved_total):
log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1)
artifact = ParserSourceArtifact.objects.create(
source=log.source,
load_batch=log.batch_id,
metadata={"snapshot_organizations_count": saved_total},
)
organization = Organization.objects.create(name="Published fixture")
for row, disposition in enumerate(["published", "published", "quarantined"]):
ParserStagedRecord.objects.create(
artifact=artifact,
row_number=row + 1,
organization=organization,
disposition=disposition,
)
ParserStagedRecord.objects.create(
artifact=artifact, row_number=4, disposition="published", organization=None
)
with CaptureQueriesContext(connection) as queries:
attach_snapshot_artifacts([log, log])
cached = log._source_artifact_cache
assert cached.pk == artifact.pk
has_saved_total = isinstance(saved_total, int) and saved_total >= 0
assert cached.published_organizations_count == (
saved_total if has_saved_total else 1
)
staged_queries = [q for q in queries if "parsers_staged_record" in q["sql"]]
assert len(staged_queries) == (0 if has_saved_total else 1)
@pytest.mark.django_db
def test_latest_artifact_and_missing_load_keep_their_own_counts():
log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1)
missing = ParserLoadLog.objects.create(source=log.source, batch_id=2)
old = ParserSourceArtifact.objects.create(source=log.source, load_batch=1)
latest = ParserSourceArtifact.objects.create(
source=log.source, load_batch=1, metadata={"snapshot_organizations_count": 9}
)
ParserSourceArtifact.objects.filter(pk=old.pk).update(
created_at=latest.created_at - timedelta(seconds=1)
)
with CaptureQueriesContext(connection) as queries:
attach_snapshot_artifacts([log, missing])
assert log._source_artifact_cache.pk == latest.pk
assert log._source_artifact_cache.published_organizations_count == 9
assert missing._source_artifact_cache is None
assert not any("parsers_staged_record" in q["sql"] for q in queries)

View File

@@ -20,6 +20,8 @@ from apps.parsers.models import (
ProcurementRecord,
)
from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import connection
from django.test.utils import CaptureQueriesContext
from django.urls import reverse
from openpyxl import Workbook
from organizations.models import Organization, OrganizationSourceRecord
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
)
self.client.force_authenticate(self.user)
response = self.client.get("/api/v1/parsers/dashboard/")
with CaptureQueriesContext(connection) as queries:
response = self.client.get("/api/v1/parsers/dashboard/")
source_scans = [
query["sql"]
for query in queries
if "SELECT DISTINCT" in query["sql"]
and "organizations_source_record" in query["sql"]
]
self.assertTrue(source_scans)
self.assertEqual(len(source_scans), len(set(source_scans)))
self.assertEqual(response.status_code, status.HTTP_200_OK)
coverage = response.data["data"]["registry_data_coverage"]