feat: publish quota-safe Checko section snapshots

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:29:58 +02:00
parent a0f623effe
commit 650a47f6c9
9 changed files with 658 additions and 2 deletions

View File

@@ -0,0 +1,276 @@
"""Real DB scoped publication with a synthetic, quota-aware transport boundary."""
import importlib
from datetime import date
from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch
import pytest
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.models import (
CheckoCollectionAttempt,
ParserSourceArtifact,
ParserStagedRecord,
)
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
SOURCE = "checko_messages"
def adapter():
return importlib.import_module("apps.parsers.checko_section_snapshots")
def record(org, suffix="one"):
return SourceRecordInput(
external_id=f"{org.inn}:{suffix}",
title="Сообщение",
organization_name=org.name,
organization_uid=org.uid,
inn=org.inn,
ogrn=org.ogrn,
record_type="company_message",
payload={"upstream": {"GUID": suffix}},
)
class CheckoSectionSnapshotTests(TestCase):
def setUp(self):
self.tmp = TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
settings = override_settings(
MEDIA_ROOT=self.tmp.name + "/public",
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
)
settings.enable()
self.addCleanup(settings.disable)
self.orgs = [
Organization.objects.create(
name=f"АО Тест {n}",
inn=inn,
ogrn=ogrn,
okpo=f"1234567{n}",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
for n, (inn, ogrn) in enumerate(
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
)
]
self.orgs.sort(key=lambda org: org.inn)
def run_refresh(self, rows=None, **kwargs):
module = adapter()
collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs])
client = Mock()
client.__enter__ = Mock(return_value=client)
client.__exit__ = Mock(return_value=False)
with patch.object(module, "_client", return_value=client), patch.object(
module, "collect_checko_section", collector
):
result = module.refresh_checko_section(
source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs
)
return result, collector
def seed(self, org):
OrganizationSourceIngestionService.save_records(
source=SOURCE, load_batch=1, records=[record(org, "old")]
)
def existing(self, org):
return list(
OrganizationSourceRecord.objects.filter(
source=SOURCE, extension__organization=org
).values_list("uid", "external_id", "load_batch")
)
def test_claimed_organization_keeps_previous_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
assert self.existing(self.orgs[0]) == before
assert collector.call_count == 1
assert artifact.metadata["completeness"] == "partial"
assert (
artifact.metadata["quota_skipped"]
== artifact.metadata["organizations_preserved"]
== 1
)
assert result.published == 2
assert artifact.metadata["updated_records_count"] == 1
assert artifact.metadata["retained_records_count"] == 1
def test_full_counter_partition_and_private_artifact(self):
(artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []])
assert artifact.metadata["completeness"] == "full"
assert artifact.metadata["organizations_total"] == 2
assert (
artifact.metadata["organizations_updated"]
== artifact.metadata["organizations_empty"]
== 1
)
assert (
artifact.metadata["organizations_failed"]
== artifact.metadata["quota_skipped"]
== 0
)
assert result.published == 1
assert len(artifact.sha256) == 64
assert artifact.size_bytes > 0
with pytest.raises(ValueError):
_ = artifact.file.url
def test_empty_success_clears_only_its_organization(self):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[1])
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
self.run_refresh(rows=[[]])
assert not self.existing(self.orgs[0])
assert self.existing(self.orgs[1]) == before
def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[0])
def interrupted():
for n in range(501):
yield record(self.orgs[0], str(n))
raise SnapshotValidationError("pagination_failed")
(artifact, _), _ = self.run_refresh(
rows=[interrupted(), [record(self.orgs[1])]]
)
assert self.existing(self.orgs[0]) == before
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["organizations_preserved"] == 1
assert not ParserStagedRecord.objects.filter(
artifact=artifact, organization=self.orgs[0]
).exists()
attempt = CheckoCollectionAttempt.objects.get(
organization=self.orgs[0], source=SOURCE
)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(
organization_id=str(self.orgs[0].uid), source=SOURCE
)
is None
)
def test_no_completed_scope_rejects_without_publication(self):
for org in self.orgs:
self.seed(org)
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
before = [self.existing(org) for org in self.orgs]
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_refresh()
artifact = ParserSourceArtifact.objects.get(source=SOURCE)
assert artifact.status == artifact.Status.REJECTED
assert artifact.metadata["quota_skipped"] == 2
assert before == [self.existing(org) for org in self.orgs]
def test_publication_failure_rolls_back_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
def reject(*args):
raise SnapshotValidationError("fixture_publication_failed")
with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"):
self.run_refresh(on_publish=reject)
assert self.existing(self.orgs[0]) == before
assert not self.existing(self.orgs[1])
assert (
ParserSourceArtifact.objects.get(source=SOURCE).status
== ParserSourceArtifact.Status.REJECTED
)
assert CheckoCollectionAttempt.objects.count() == 2
def test_uid_stable_across_months(self):
self.run_refresh(load_batch=1)
before = [self.existing(org)[0][0] for org in self.orgs]
# Advance the calendar; never delete or reset a monthly claim.
with patch(
"apps.parsers.checko_collection.collection_period_month",
return_value=date(2026, 11, 1),
):
self.run_refresh(load_batch=2)
assert before == [self.existing(org)[0][0] for org in self.orgs]
def test_sources_have_independent_nonreleased_claims(self):
assert set(adapter().CHECKO_SECTIONS) <= set(
CheckoCollectionAttempt.Source.values
)
for source in adapter().CHECKO_SECTIONS:
attempt = claim_monthly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
assert attempt is not None
finish_collection(attempt, records_count=0, error="fixture")
assert (
claim_monthly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
is None
)
def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self):
self.orgs[1].directory_imported_at = None
self.orgs[1].save()
Organization.objects.create(
name="Outside",
inn="7710140679",
ogrn="1027700425174",
okpo="87654321",
directory_imported_at=timezone.now(),
)
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
assert collector.call_count == 1
assert artifact.metadata["organizations_total"] == 1
assert Organization.objects.count() == 3
def test_invalid_canonical_identifiers_preserved_without_http(self):
self.orgs[1].okpo = "1234567890"
self.orgs[1].save()
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
assert collector.call_count == 1
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["completeness"] == "partial"
@pytest.mark.django_db(transaction=True)
def test_concurrent_monthly_claim_one_winner():
from django.db import connection
from tests.apps.parsers.test_refresh_concurrency import _parallel
if connection.vendor != "postgresql":
pytest.skip("Requires PostgreSQL concurrent unique claims")
org = Organization.objects.create(
name="Claim",
inn="7707083893",
ogrn="1027700132195",
okpo="12345678",
directory_imported_at=timezone.now(),
)
def claim():
return (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
is not None
)
assert sorted(_parallel([claim, claim])) == [False, True]
assert CheckoCollectionAttempt.objects.count() == 1