fix: avoid repeated task and source scans on dashboard
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit was merged in pull request #47.
This commit is contained in:
@@ -1,33 +1,41 @@
|
||||
"""Public snapshot provenance shared by cards, dashboard and parser history."""
|
||||
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from django.db.models import Count, Q
|
||||
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
||||
from django.db.models import Count
|
||||
|
||||
|
||||
def attach_snapshot_artifacts(logs) -> None:
|
||||
"""Attach the latest artifact for each load in one query, without raw data."""
|
||||
"""Attach provenance, counting staged rows only for legacy snapshots."""
|
||||
logs = list(logs)
|
||||
if not logs:
|
||||
return
|
||||
artifacts = {}
|
||||
for artifact in (
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source__in={log.source for log in logs},
|
||||
load_batch__in={log.batch_id for log in logs},
|
||||
)
|
||||
.annotate(
|
||||
published_organizations_count=Count(
|
||||
"staged_records__organization_id",
|
||||
distinct=True,
|
||||
filter=Q(staged_records__disposition="published"),
|
||||
)
|
||||
)
|
||||
.order_by("-created_at", "-uid")
|
||||
):
|
||||
for artifact in ParserSourceArtifact.objects.filter(
|
||||
source__in={log.source for log in logs},
|
||||
load_batch__in={log.batch_id for log in logs},
|
||||
).order_by("-created_at", "-uid"):
|
||||
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
|
||||
legacy_artifacts = []
|
||||
for artifact in artifacts.values():
|
||||
total = artifact.metadata.get("snapshot_organizations_count")
|
||||
if isinstance(total, int) and total >= 0:
|
||||
artifact.published_organizations_count = total
|
||||
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
|
||||
else:
|
||||
legacy_artifacts.append(artifact.pk)
|
||||
counts = {}
|
||||
if legacy_artifacts:
|
||||
counts = dict(
|
||||
ParserStagedRecord.objects.filter(
|
||||
artifact_id__in=legacy_artifacts, disposition="published"
|
||||
)
|
||||
.order_by()
|
||||
.values("artifact_id")
|
||||
.annotate(total=Count("organization_id", distinct=True))
|
||||
.values_list("artifact_id", "total")
|
||||
)
|
||||
for artifact in artifacts.values():
|
||||
if artifact.pk in legacy_artifacts:
|
||||
artifact.published_organizations_count = counts.get(artifact.pk, 0)
|
||||
for log in logs:
|
||||
log._source_artifact_cache = artifacts.get((log.source, log.batch_id))
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ from apps.parsers.snapshot_metadata import (
|
||||
from apps.parsers.source_registry import get_source_by_model_source
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db import transaction
|
||||
from django.db import connections, transaction
|
||||
from django.db.models import Count, Max, OuterRef, Q, Subquery
|
||||
from django.http import Http404
|
||||
from django.utils import timezone
|
||||
@@ -1033,19 +1033,28 @@ class SourceCardService:
|
||||
for task_name in definition.task_names:
|
||||
slugs_by_task_name.setdefault(task_name, []).append(definition.slug)
|
||||
slugs = {definition.slug for definition in definitions}
|
||||
latest = (
|
||||
BackgroundJobService.get_queryset()
|
||||
.filter(task_name=OuterRef("task_name"))
|
||||
.order_by("-created_at", "-id")
|
||||
.values("pk")[:1]
|
||||
)
|
||||
all_jobs = BackgroundJobService.get_queryset()
|
||||
if connections[all_jobs.db].features.can_distinct_on_fields:
|
||||
# Find each task's latest job once, rather than once per history row.
|
||||
latest = all_jobs.order_by("task_name", "-created_at", "-id").distinct(
|
||||
"task_name"
|
||||
)
|
||||
else:
|
||||
latest_for_task = (
|
||||
all_jobs.filter(task_name=OuterRef("task_name"))
|
||||
.order_by("-created_at", "-id")
|
||||
.values("pk")[:1]
|
||||
)
|
||||
latest = all_jobs.filter(pk=Subquery(latest_for_task)).order_by()
|
||||
jobs = list(
|
||||
BackgroundJobService.get_queryset()
|
||||
.filter(
|
||||
Q(task_name__in=slugs_by_task_name)
|
||||
| Q(meta__source_card__in=list(slugs))
|
||||
)
|
||||
.filter(cls._fresh_active_job_filter() | Q(pk=Subquery(latest)))
|
||||
.filter(
|
||||
cls._fresh_active_job_filter() | Q(pk__in=Subquery(latest.values("pk")))
|
||||
)
|
||||
.order_by("-created_at", "-id")
|
||||
)
|
||||
active_by_slug: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
@@ -2242,8 +2242,9 @@ def _registry_source_matrix_rows(
|
||||
return matrix_rows
|
||||
|
||||
|
||||
def _registry_data_coverage() -> dict:
|
||||
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||
def _registry_data_coverage(
|
||||
registry_organization_ids: set[str], source_matches: dict[str, set[str]]
|
||||
) -> dict:
|
||||
total_organizations = len(registry_organization_ids)
|
||||
descriptor_titles: dict[str, str] = {}
|
||||
for descriptor in PARSER_SOURCES.values():
|
||||
@@ -2254,9 +2255,7 @@ def _registry_data_coverage() -> dict:
|
||||
for source, _label in ParserLoadLog.Source.choices:
|
||||
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
|
||||
continue
|
||||
organizations_count = len(
|
||||
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
|
||||
)
|
||||
organizations_count = len(source_matches.get(source, set()))
|
||||
coverage_percent = (
|
||||
round(organizations_count / total_organizations * 100, 1)
|
||||
if total_organizations
|
||||
@@ -2277,10 +2276,14 @@ def _registry_data_coverage() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
|
||||
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||
def _registry_enrichment_analytics(
|
||||
jobs,
|
||||
schedules: list[dict],
|
||||
registry_organization_ids: set[str],
|
||||
source_matches: dict[str, set[str]],
|
||||
latest_loads: list[ParserLoadLog],
|
||||
) -> dict:
|
||||
total_organizations = len(registry_organization_ids)
|
||||
source_matches = _source_registry_matches(by_inn, by_ogrn)
|
||||
descriptor_titles: dict[str, str] = {}
|
||||
for descriptor in PARSER_SOURCES.values():
|
||||
descriptor_titles.setdefault(descriptor.source, descriptor.title)
|
||||
@@ -2369,7 +2372,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
|
||||
enrichment_sources=enrichment_sources,
|
||||
)
|
||||
|
||||
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
|
||||
recent_success = sum(
|
||||
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
|
||||
)
|
||||
@@ -3459,6 +3461,8 @@ class ParserDashboardDataView(APIView):
|
||||
]
|
||||
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
|
||||
attach_snapshot_artifacts(load_logs)
|
||||
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||
source_matches = _source_registry_matches(by_inn, by_ogrn)
|
||||
return api_response(
|
||||
{
|
||||
"sources": sources,
|
||||
@@ -3471,10 +3475,15 @@ class ParserDashboardDataView(APIView):
|
||||
"schedules": ParserScheduleSerializer(schedules, many=True).data,
|
||||
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
|
||||
"source_counts": source_counts,
|
||||
"registry_data_coverage": _registry_data_coverage(),
|
||||
"registry_data_coverage": _registry_data_coverage(
|
||||
registry_ids, source_matches
|
||||
),
|
||||
"registry_enrichment_analytics": _registry_enrichment_analytics(
|
||||
jobs,
|
||||
schedules,
|
||||
registry_ids,
|
||||
source_matches,
|
||||
load_logs,
|
||||
),
|
||||
"load_logs": ParserLoadLogSerializer(
|
||||
load_logs,
|
||||
|
||||
Reference in New Issue
Block a user