fix: avoid repeated task and source scans on dashboard
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit was merged in pull request #47.
This commit is contained in:
Aleksandr Meshchryakov
2026-10-01 13:16:11 +02:00
parent 887442e8e3
commit fedd2e00a3
7 changed files with 239 additions and 37 deletions

View File

@@ -1,33 +1,41 @@
"""Public snapshot provenance shared by cards, dashboard and parser history."""
from apps.parsers.models import ParserSourceArtifact
from django.db.models import Count, Q
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from django.db.models import Count
def attach_snapshot_artifacts(logs) -> None:
"""Attach the latest artifact for each load in one query, without raw data."""
"""Attach provenance, counting staged rows only for legacy snapshots."""
logs = list(logs)
if not logs:
return
artifacts = {}
for artifact in (
ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
)
.annotate(
published_organizations_count=Count(
"staged_records__organization_id",
distinct=True,
filter=Q(staged_records__disposition="published"),
)
)
.order_by("-created_at", "-uid")
):
for artifact in ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
).order_by("-created_at", "-uid"):
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
legacy_artifacts = []
for artifact in artifacts.values():
total = artifact.metadata.get("snapshot_organizations_count")
if isinstance(total, int) and total >= 0:
artifact.published_organizations_count = total
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
else:
legacy_artifacts.append(artifact.pk)
counts = {}
if legacy_artifacts:
counts = dict(
ParserStagedRecord.objects.filter(
artifact_id__in=legacy_artifacts, disposition="published"
)
.order_by()
.values("artifact_id")
.annotate(total=Count("organization_id", distinct=True))
.values_list("artifact_id", "total")
)
for artifact in artifacts.values():
if artifact.pk in legacy_artifacts:
artifact.published_organizations_count = counts.get(artifact.pk, 0)
for log in logs:
log._source_artifact_cache = artifacts.get((log.source, log.batch_id))

View File

@@ -27,7 +27,7 @@ from apps.parsers.snapshot_metadata import (
from apps.parsers.source_registry import get_source_by_model_source
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from django.db import connections, transaction
from django.db.models import Count, Max, OuterRef, Q, Subquery
from django.http import Http404
from django.utils import timezone
@@ -1033,19 +1033,28 @@ class SourceCardService:
for task_name in definition.task_names:
slugs_by_task_name.setdefault(task_name, []).append(definition.slug)
slugs = {definition.slug for definition in definitions}
latest = (
BackgroundJobService.get_queryset()
.filter(task_name=OuterRef("task_name"))
.order_by("-created_at", "-id")
.values("pk")[:1]
)
all_jobs = BackgroundJobService.get_queryset()
if connections[all_jobs.db].features.can_distinct_on_fields:
# Find each task's latest job once, rather than once per history row.
latest = all_jobs.order_by("task_name", "-created_at", "-id").distinct(
"task_name"
)
else:
latest_for_task = (
all_jobs.filter(task_name=OuterRef("task_name"))
.order_by("-created_at", "-id")
.values("pk")[:1]
)
latest = all_jobs.filter(pk=Subquery(latest_for_task)).order_by()
jobs = list(
BackgroundJobService.get_queryset()
.filter(
Q(task_name__in=slugs_by_task_name)
| Q(meta__source_card__in=list(slugs))
)
.filter(cls._fresh_active_job_filter() | Q(pk=Subquery(latest)))
.filter(
cls._fresh_active_job_filter() | Q(pk__in=Subquery(latest.values("pk")))
)
.order_by("-created_at", "-id")
)
active_by_slug: dict[str, list[dict[str, Any]]] = {}

View File

@@ -2242,8 +2242,9 @@ def _registry_source_matrix_rows(
return matrix_rows
def _registry_data_coverage() -> dict:
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
def _registry_data_coverage(
registry_organization_ids: set[str], source_matches: dict[str, set[str]]
) -> dict:
total_organizations = len(registry_organization_ids)
descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values():
@@ -2254,9 +2255,7 @@ def _registry_data_coverage() -> dict:
for source, _label in ParserLoadLog.Source.choices:
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
continue
organizations_count = len(
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
)
organizations_count = len(source_matches.get(source, set()))
coverage_percent = (
round(organizations_count / total_organizations * 100, 1)
if total_organizations
@@ -2277,10 +2276,14 @@ def _registry_data_coverage() -> dict:
}
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
def _registry_enrichment_analytics(
jobs,
schedules: list[dict],
registry_organization_ids: set[str],
source_matches: dict[str, set[str]],
latest_loads: list[ParserLoadLog],
) -> dict:
total_organizations = len(registry_organization_ids)
source_matches = _source_registry_matches(by_inn, by_ogrn)
descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values():
descriptor_titles.setdefault(descriptor.source, descriptor.title)
@@ -2369,7 +2372,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
enrichment_sources=enrichment_sources,
)
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
recent_success = sum(
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
)
@@ -3459,6 +3461,8 @@ class ParserDashboardDataView(APIView):
]
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
attach_snapshot_artifacts(load_logs)
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
source_matches = _source_registry_matches(by_inn, by_ogrn)
return api_response(
{
"sources": sources,
@@ -3471,10 +3475,15 @@ class ParserDashboardDataView(APIView):
"schedules": ParserScheduleSerializer(schedules, many=True).data,
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
"source_counts": source_counts,
"registry_data_coverage": _registry_data_coverage(),
"registry_data_coverage": _registry_data_coverage(
registry_ids, source_matches
),
"registry_enrichment_analytics": _registry_enrichment_analytics(
jobs,
schedules,
registry_ids,
source_matches,
load_logs,
),
"load_logs": ParserLoadLogSerializer(
load_logs,