Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Successful in 3m16s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Mostovik Backend CI/CD / Tests and lint (push) Failing after 3s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
62 lines
2.7 KiB
Python
62 lines
2.7 KiB
Python
"""Snapshot counts avoid rescanning staged data when a saved total exists."""
|
|
|
|
from datetime import timedelta
|
|
|
|
import pytest
|
|
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
|
|
from apps.parsers.snapshot_metadata import attach_snapshot_artifacts
|
|
from django.db import connection
|
|
from django.test.utils import CaptureQueriesContext
|
|
from organizations.models import Organization
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.parametrize("saved_total", [0, 7, None, -1, "7"])
|
|
def test_saved_total_or_distinct_published_organizations(saved_total):
|
|
log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1)
|
|
artifact = ParserSourceArtifact.objects.create(
|
|
source=log.source,
|
|
load_batch=log.batch_id,
|
|
metadata={"snapshot_organizations_count": saved_total},
|
|
)
|
|
organization = Organization.objects.create(name="Published fixture")
|
|
for row, disposition in enumerate(["published", "published", "quarantined"]):
|
|
ParserStagedRecord.objects.create(
|
|
artifact=artifact,
|
|
row_number=row + 1,
|
|
organization=organization,
|
|
disposition=disposition,
|
|
)
|
|
ParserStagedRecord.objects.create(
|
|
artifact=artifact, row_number=4, disposition="published", organization=None
|
|
)
|
|
with CaptureQueriesContext(connection) as queries:
|
|
attach_snapshot_artifacts([log, log])
|
|
cached = log._source_artifact_cache
|
|
assert cached.pk == artifact.pk
|
|
has_saved_total = isinstance(saved_total, int) and saved_total >= 0
|
|
assert cached.published_organizations_count == (
|
|
saved_total if has_saved_total else 1
|
|
)
|
|
staged_queries = [q for q in queries if "parsers_staged_record" in q["sql"]]
|
|
assert len(staged_queries) == (0 if has_saved_total else 1)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_latest_artifact_and_missing_load_keep_their_own_counts():
|
|
log = ParserLoadLog.objects.create(source="sro_membership_check", batch_id=1)
|
|
missing = ParserLoadLog.objects.create(source=log.source, batch_id=2)
|
|
old = ParserSourceArtifact.objects.create(source=log.source, load_batch=1)
|
|
latest = ParserSourceArtifact.objects.create(
|
|
source=log.source, load_batch=1, metadata={"snapshot_organizations_count": 9}
|
|
)
|
|
ParserSourceArtifact.objects.filter(pk=old.pk).update(
|
|
created_at=latest.created_at - timedelta(seconds=1)
|
|
)
|
|
with CaptureQueriesContext(connection) as queries:
|
|
attach_snapshot_artifacts([log, missing])
|
|
assert log._source_artifact_cache.pk == latest.pk
|
|
assert log._source_artifact_cache.published_organizations_count == 9
|
|
assert missing._source_artifact_cache is None
|
|
assert not any("parsers_staged_record" in q["sql"] for q in queries)
|