fix: avoid repeated task and source scans on dashboard
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit was merged in pull request #47.
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
from datetime import timedelta
|
||||
from unittest.mock import MagicMock
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
@@ -145,3 +146,53 @@ def test_latest_task_does_not_reactivate_a_stale_job(status, time_field):
|
||||
**{time_field: timezone.now() - timedelta(minutes=15)}
|
||||
)
|
||||
assert SourceCardService.get_card(definition.slug)["active_tasks"] == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_latest_job_uses_created_time_then_uuid_and_keeps_older_active_jobs():
|
||||
definition = SourceCardService.get_definition("arbitration-cases")
|
||||
jobs = [
|
||||
BackgroundJob.objects.create(
|
||||
id=UUID(int=index),
|
||||
task_id=f"ordered-{index}",
|
||||
task_name=definition.task_names[0],
|
||||
status=JobStatus.SUCCESS,
|
||||
)
|
||||
for index in (1, 2, 3, 4)
|
||||
]
|
||||
now = timezone.now()
|
||||
BackgroundJob.objects.filter(pk__in=[job.pk for job in jobs]).update(
|
||||
created_at=now - timedelta(minutes=2)
|
||||
)
|
||||
# UUID 4 is older; UUID 2 wins the timestamp tie with UUID 1.
|
||||
BackgroundJob.objects.filter(pk__in=[jobs[0].pk, jobs[1].pk]).update(created_at=now)
|
||||
BackgroundJob.objects.filter(pk=jobs[2].pk).update(status=JobStatus.STARTED)
|
||||
active, refresh = SourceCardService._build_task_maps((definition,))
|
||||
assert [job["task_id"] for job in active[definition.slug]] == ["ordered-3"]
|
||||
assert [job["task_id"] for job in refresh[definition.slug]] == [
|
||||
"ordered-2",
|
||||
"ordered-3",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_latest_job_is_chosen_before_filtering_source_card_metadata():
|
||||
definition = SourceCardService.get_definition("arbitration-cases")
|
||||
older = BackgroundJob.objects.create(
|
||||
task_id="old-card-job",
|
||||
task_name="custom.shared.task",
|
||||
status=JobStatus.SUCCESS,
|
||||
meta={"source_card": definition.slug},
|
||||
)
|
||||
newer = BackgroundJob.objects.create(
|
||||
task_id="new-other-card-job",
|
||||
task_name=older.task_name,
|
||||
status=JobStatus.SUCCESS,
|
||||
meta={"source_card": "bankruptcy-procedures"},
|
||||
)
|
||||
BackgroundJob.objects.filter(pk=older.pk).update(
|
||||
created_at=newer.created_at - timedelta(minutes=1)
|
||||
)
|
||||
active, refresh = SourceCardService._build_task_maps((definition,))
|
||||
assert active == {}
|
||||
assert refresh == {}
|
||||
|
||||
61
tests/apps/parsers/test_snapshot_metadata.py
Normal file
61
tests/apps/parsers/test_snapshot_metadata.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Snapshot counts avoid rescanning staged data when a saved total exists."""
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
|
||||
from apps.parsers.snapshot_metadata import attach_snapshot_artifacts
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from organizations.models import Organization
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("saved_total", [0, 7, None, -1, "7"])
|
||||
def test_saved_total_or_distinct_published_organizations(saved_total):
|
||||
log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1)
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source=log.source,
|
||||
load_batch=log.batch_id,
|
||||
metadata={"snapshot_organizations_count": saved_total},
|
||||
)
|
||||
organization = Organization.objects.create(name="Published fixture")
|
||||
for row, disposition in enumerate(["published", "published", "quarantined"]):
|
||||
ParserStagedRecord.objects.create(
|
||||
artifact=artifact,
|
||||
row_number=row + 1,
|
||||
organization=organization,
|
||||
disposition=disposition,
|
||||
)
|
||||
ParserStagedRecord.objects.create(
|
||||
artifact=artifact, row_number=4, disposition="published", organization=None
|
||||
)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
attach_snapshot_artifacts([log, log])
|
||||
cached = log._source_artifact_cache
|
||||
assert cached.pk == artifact.pk
|
||||
has_saved_total = isinstance(saved_total, int) and saved_total >= 0
|
||||
assert cached.published_organizations_count == (
|
||||
saved_total if has_saved_total else 1
|
||||
)
|
||||
staged_queries = [q for q in queries if "parsers_staged_record" in q["sql"]]
|
||||
assert len(staged_queries) == (0 if has_saved_total else 1)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_latest_artifact_and_missing_load_keep_their_own_counts():
|
||||
log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1)
|
||||
missing = ParserLoadLog.objects.create(source=log.source, batch_id=2)
|
||||
old = ParserSourceArtifact.objects.create(source=log.source, load_batch=1)
|
||||
latest = ParserSourceArtifact.objects.create(
|
||||
source=log.source, load_batch=1, metadata={"snapshot_organizations_count": 9}
|
||||
)
|
||||
ParserSourceArtifact.objects.filter(pk=old.pk).update(
|
||||
created_at=latest.created_at - timedelta(seconds=1)
|
||||
)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
attach_snapshot_artifacts([log, missing])
|
||||
assert log._source_artifact_cache.pk == latest.pk
|
||||
assert log._source_artifact_cache.published_organizations_count == 9
|
||||
assert missing._source_artifact_cache is None
|
||||
assert not any("parsers_staged_record" in q["sql"] for q in queries)
|
||||
@@ -20,6 +20,8 @@ from apps.parsers.models import (
|
||||
ProcurementRecord,
|
||||
)
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from django.urls import reverse
|
||||
from openpyxl import Workbook
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
|
||||
)
|
||||
self.client.force_authenticate(self.user)
|
||||
|
||||
response = self.client.get("/api/v1/parsers/dashboard/")
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
response = self.client.get("/api/v1/parsers/dashboard/")
|
||||
|
||||
source_scans = [
|
||||
query["sql"]
|
||||
for query in queries
|
||||
if "SELECT DISTINCT" in query["sql"]
|
||||
and "organizations_source_record" in query["sql"]
|
||||
]
|
||||
self.assertTrue(source_scans)
|
||||
self.assertEqual(len(source_scans), len(set(source_scans)))
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
coverage = response.data["data"]["registry_data_coverage"]
|
||||
|
||||
Reference in New Issue
Block a user