feat: publish quota-safe Checko section snapshots
This commit is contained in:
276
tests/apps/parsers/test_checko_section_snapshots.py
Normal file
276
tests/apps/parsers/test_checko_section_snapshots.py
Normal file
@@ -0,0 +1,276 @@
|
||||
"""Real DB scoped publication with a synthetic, quota-aware transport boundary."""
|
||||
|
||||
import importlib
|
||||
from datetime import date
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
||||
from apps.parsers.models import (
|
||||
CheckoCollectionAttempt,
|
||||
ParserSourceArtifact,
|
||||
ParserStagedRecord,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from django.test import TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from organizations.source_ingestion import (
|
||||
OrganizationSourceIngestionService,
|
||||
SourceRecordInput,
|
||||
)
|
||||
|
||||
SOURCE = "checko_messages"
|
||||
|
||||
|
||||
def adapter():
|
||||
return importlib.import_module("apps.parsers.checko_section_snapshots")
|
||||
|
||||
|
||||
def record(org, suffix="one"):
|
||||
return SourceRecordInput(
|
||||
external_id=f"{org.inn}:{suffix}",
|
||||
title="Сообщение",
|
||||
organization_name=org.name,
|
||||
organization_uid=org.uid,
|
||||
inn=org.inn,
|
||||
ogrn=org.ogrn,
|
||||
record_type="company_message",
|
||||
payload={"upstream": {"GUID": suffix}},
|
||||
)
|
||||
|
||||
|
||||
class CheckoSectionSnapshotTests(TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
settings = override_settings(
|
||||
MEDIA_ROOT=self.tmp.name + "/public",
|
||||
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
|
||||
)
|
||||
settings.enable()
|
||||
self.addCleanup(settings.disable)
|
||||
self.orgs = [
|
||||
Organization.objects.create(
|
||||
name=f"АО Тест {n}",
|
||||
inn=inn,
|
||||
ogrn=ogrn,
|
||||
okpo=f"1234567{n}",
|
||||
directory_imported_at=timezone.now(),
|
||||
opk_registry_membership=True,
|
||||
)
|
||||
for n, (inn, ogrn) in enumerate(
|
||||
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
|
||||
)
|
||||
]
|
||||
|
||||
self.orgs.sort(key=lambda org: org.inn)
|
||||
|
||||
def run_refresh(self, rows=None, **kwargs):
|
||||
module = adapter()
|
||||
collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs])
|
||||
client = Mock()
|
||||
client.__enter__ = Mock(return_value=client)
|
||||
client.__exit__ = Mock(return_value=False)
|
||||
with patch.object(module, "_client", return_value=client), patch.object(
|
||||
module, "collect_checko_section", collector
|
||||
):
|
||||
result = module.refresh_checko_section(
|
||||
source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs
|
||||
)
|
||||
return result, collector
|
||||
|
||||
def seed(self, org):
|
||||
OrganizationSourceIngestionService.save_records(
|
||||
source=SOURCE, load_batch=1, records=[record(org, "old")]
|
||||
)
|
||||
|
||||
def existing(self, org):
|
||||
return list(
|
||||
OrganizationSourceRecord.objects.filter(
|
||||
source=SOURCE, extension__organization=org
|
||||
).values_list("uid", "external_id", "load_batch")
|
||||
)
|
||||
|
||||
def test_claimed_organization_keeps_previous_records(self):
|
||||
self.seed(self.orgs[0])
|
||||
before = self.existing(self.orgs[0])
|
||||
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
|
||||
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
|
||||
assert self.existing(self.orgs[0]) == before
|
||||
assert collector.call_count == 1
|
||||
assert artifact.metadata["completeness"] == "partial"
|
||||
assert (
|
||||
artifact.metadata["quota_skipped"]
|
||||
== artifact.metadata["organizations_preserved"]
|
||||
== 1
|
||||
)
|
||||
assert result.published == 2
|
||||
assert artifact.metadata["updated_records_count"] == 1
|
||||
assert artifact.metadata["retained_records_count"] == 1
|
||||
|
||||
def test_full_counter_partition_and_private_artifact(self):
|
||||
(artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []])
|
||||
assert artifact.metadata["completeness"] == "full"
|
||||
assert artifact.metadata["organizations_total"] == 2
|
||||
assert (
|
||||
artifact.metadata["organizations_updated"]
|
||||
== artifact.metadata["organizations_empty"]
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
artifact.metadata["organizations_failed"]
|
||||
== artifact.metadata["quota_skipped"]
|
||||
== 0
|
||||
)
|
||||
assert result.published == 1
|
||||
assert len(artifact.sha256) == 64
|
||||
assert artifact.size_bytes > 0
|
||||
with pytest.raises(ValueError):
|
||||
_ = artifact.file.url
|
||||
|
||||
def test_empty_success_clears_only_its_organization(self):
|
||||
for org in self.orgs:
|
||||
self.seed(org)
|
||||
before = self.existing(self.orgs[1])
|
||||
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
|
||||
self.run_refresh(rows=[[]])
|
||||
assert not self.existing(self.orgs[0])
|
||||
assert self.existing(self.orgs[1]) == before
|
||||
|
||||
def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self):
|
||||
for org in self.orgs:
|
||||
self.seed(org)
|
||||
before = self.existing(self.orgs[0])
|
||||
|
||||
def interrupted():
|
||||
for n in range(501):
|
||||
yield record(self.orgs[0], str(n))
|
||||
raise SnapshotValidationError("pagination_failed")
|
||||
|
||||
(artifact, _), _ = self.run_refresh(
|
||||
rows=[interrupted(), [record(self.orgs[1])]]
|
||||
)
|
||||
assert self.existing(self.orgs[0]) == before
|
||||
assert artifact.metadata["organizations_failed"] == 1
|
||||
assert artifact.metadata["organizations_preserved"] == 1
|
||||
assert not ParserStagedRecord.objects.filter(
|
||||
artifact=artifact, organization=self.orgs[0]
|
||||
).exists()
|
||||
attempt = CheckoCollectionAttempt.objects.get(
|
||||
organization=self.orgs[0], source=SOURCE
|
||||
)
|
||||
assert attempt.status == attempt.Status.FAILED
|
||||
assert (
|
||||
claim_monthly_collection(
|
||||
organization_id=str(self.orgs[0].uid), source=SOURCE
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_no_completed_scope_rejects_without_publication(self):
|
||||
for org in self.orgs:
|
||||
self.seed(org)
|
||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
before = [self.existing(org) for org in self.orgs]
|
||||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||||
self.run_refresh()
|
||||
artifact = ParserSourceArtifact.objects.get(source=SOURCE)
|
||||
assert artifact.status == artifact.Status.REJECTED
|
||||
assert artifact.metadata["quota_skipped"] == 2
|
||||
assert before == [self.existing(org) for org in self.orgs]
|
||||
|
||||
def test_publication_failure_rolls_back_records(self):
|
||||
self.seed(self.orgs[0])
|
||||
before = self.existing(self.orgs[0])
|
||||
|
||||
def reject(*args):
|
||||
raise SnapshotValidationError("fixture_publication_failed")
|
||||
|
||||
with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"):
|
||||
self.run_refresh(on_publish=reject)
|
||||
assert self.existing(self.orgs[0]) == before
|
||||
assert not self.existing(self.orgs[1])
|
||||
assert (
|
||||
ParserSourceArtifact.objects.get(source=SOURCE).status
|
||||
== ParserSourceArtifact.Status.REJECTED
|
||||
)
|
||||
assert CheckoCollectionAttempt.objects.count() == 2
|
||||
|
||||
def test_uid_stable_across_months(self):
|
||||
self.run_refresh(load_batch=1)
|
||||
before = [self.existing(org)[0][0] for org in self.orgs]
|
||||
# Advance the calendar; never delete or reset a monthly claim.
|
||||
with patch(
|
||||
"apps.parsers.checko_collection.collection_period_month",
|
||||
return_value=date(2026, 11, 1),
|
||||
):
|
||||
self.run_refresh(load_batch=2)
|
||||
assert before == [self.existing(org)[0][0] for org in self.orgs]
|
||||
|
||||
def test_sources_have_independent_nonreleased_claims(self):
|
||||
assert set(adapter().CHECKO_SECTIONS) <= set(
|
||||
CheckoCollectionAttempt.Source.values
|
||||
)
|
||||
for source in adapter().CHECKO_SECTIONS:
|
||||
attempt = claim_monthly_collection(
|
||||
organization_id=str(self.orgs[0].uid), source=source
|
||||
)
|
||||
assert attempt is not None
|
||||
finish_collection(attempt, records_count=0, error="fixture")
|
||||
assert (
|
||||
claim_monthly_collection(
|
||||
organization_id=str(self.orgs[0].uid), source=source
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self):
|
||||
self.orgs[1].directory_imported_at = None
|
||||
self.orgs[1].save()
|
||||
Organization.objects.create(
|
||||
name="Outside",
|
||||
inn="7710140679",
|
||||
ogrn="1027700425174",
|
||||
okpo="87654321",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
|
||||
assert collector.call_count == 1
|
||||
assert artifact.metadata["organizations_total"] == 1
|
||||
assert Organization.objects.count() == 3
|
||||
|
||||
def test_invalid_canonical_identifiers_preserved_without_http(self):
|
||||
self.orgs[1].okpo = "1234567890"
|
||||
self.orgs[1].save()
|
||||
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
|
||||
assert collector.call_count == 1
|
||||
assert artifact.metadata["organizations_failed"] == 1
|
||||
assert artifact.metadata["completeness"] == "partial"
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_concurrent_monthly_claim_one_winner():
|
||||
from django.db import connection
|
||||
|
||||
from tests.apps.parsers.test_refresh_concurrency import _parallel
|
||||
|
||||
if connection.vendor != "postgresql":
|
||||
pytest.skip("Requires PostgreSQL concurrent unique claims")
|
||||
org = Organization.objects.create(
|
||||
name="Claim",
|
||||
inn="7707083893",
|
||||
ogrn="1027700132195",
|
||||
okpo="12345678",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
|
||||
def claim():
|
||||
return (
|
||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
is not None
|
||||
)
|
||||
|
||||
assert sorted(_parallel([claim, claim])) == [False, True]
|
||||
assert CheckoCollectionAttempt.objects.count() == 1
|
||||
Reference in New Issue
Block a user