"""Real DB scoped publication with a synthetic, quota-aware transport boundary.""" import importlib from datetime import date from tempfile import TemporaryDirectory from unittest.mock import Mock, patch import pytest from apps.parsers.checko_collection import claim_monthly_collection, finish_collection from apps.parsers.models import ( CheckoCollectionAttempt, ParserSourceArtifact, ParserStagedRecord, ) from apps.parsers.registry_snapshots import SnapshotValidationError from django.test import TestCase, override_settings from django.utils import timezone from organizations.models import Organization, OrganizationSourceRecord from organizations.source_ingestion import ( OrganizationSourceIngestionService, SourceRecordInput, ) SOURCE = "checko_messages" def adapter(): return importlib.import_module("apps.parsers.checko_section_snapshots") def record(org, suffix="one"): return SourceRecordInput( external_id=f"{org.inn}:{suffix}", title="Сообщение", organization_name=org.name, organization_uid=org.uid, inn=org.inn, ogrn=org.ogrn, record_type="company_message", payload={"upstream": {"GUID": suffix}}, ) class CheckoSectionSnapshotTests(TestCase): def setUp(self): self.tmp = TemporaryDirectory() self.addCleanup(self.tmp.cleanup) settings = override_settings( MEDIA_ROOT=self.tmp.name + "/public", PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private", ) settings.enable() self.addCleanup(settings.disable) self.orgs = [ Organization.objects.create( name=f"АО Тест {n}", inn=inn, ogrn=ogrn, okpo=f"1234567{n}", directory_imported_at=timezone.now(), opk_registry_membership=True, ) for n, (inn, ogrn) in enumerate( [("7707083893", "1027700132195"), ("7702070139", "1027700070518")] ) ] self.orgs.sort(key=lambda org: org.inn) def run_refresh(self, rows=None, **kwargs): module = adapter() collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs]) client = Mock() client.__enter__ = Mock(return_value=client) client.__exit__ = Mock(return_value=False) with patch.object(module, "_client", return_value=client), patch.object( module, "collect_checko_section", collector ): result = module.refresh_checko_section( source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs ) return result, collector def seed(self, org): OrganizationSourceIngestionService.save_records( source=SOURCE, load_batch=1, records=[record(org, "old")] ) def existing(self, org): return list( OrganizationSourceRecord.objects.filter( source=SOURCE, extension__organization=org ).values_list("uid", "external_id", "load_batch") ) def test_claimed_organization_keeps_previous_records(self): self.seed(self.orgs[0]) before = self.existing(self.orgs[0]) claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE) (artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]]) assert self.existing(self.orgs[0]) == before assert collector.call_count == 1 assert artifact.metadata["completeness"] == "partial" assert ( artifact.metadata["quota_skipped"] == artifact.metadata["organizations_preserved"] == 1 ) assert result.published == 2 assert artifact.metadata["updated_records_count"] == 1 assert artifact.metadata["retained_records_count"] == 1 def test_full_counter_partition_and_private_artifact(self): (artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []]) assert artifact.metadata["completeness"] == "full" assert artifact.metadata["organizations_total"] == 2 assert ( artifact.metadata["organizations_updated"] == artifact.metadata["organizations_empty"] == 1 ) assert ( artifact.metadata["organizations_failed"] == artifact.metadata["quota_skipped"] == 0 ) assert result.published == 1 assert len(artifact.sha256) == 64 assert artifact.size_bytes > 0 with pytest.raises(ValueError): _ = artifact.file.url def test_empty_success_clears_only_its_organization(self): for org in self.orgs: self.seed(org) before = self.existing(self.orgs[1]) claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE) self.run_refresh(rows=[[]]) assert not self.existing(self.orgs[0]) assert self.existing(self.orgs[1]) == before def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self): for org in self.orgs: self.seed(org) before = self.existing(self.orgs[0]) def interrupted(): for n in range(501): yield record(self.orgs[0], str(n)) raise SnapshotValidationError("pagination_failed") (artifact, _), _ = self.run_refresh( rows=[interrupted(), [record(self.orgs[1])]] ) assert self.existing(self.orgs[0]) == before assert artifact.metadata["organizations_failed"] == 1 assert artifact.metadata["organizations_preserved"] == 1 assert not ParserStagedRecord.objects.filter( artifact=artifact, organization=self.orgs[0] ).exists() attempt = CheckoCollectionAttempt.objects.get( organization=self.orgs[0], source=SOURCE ) assert attempt.status == attempt.Status.FAILED assert ( claim_monthly_collection( organization_id=str(self.orgs[0].uid), source=SOURCE ) is None ) def test_no_completed_scope_rejects_without_publication(self): for org in self.orgs: self.seed(org) claim_monthly_collection(organization_id=str(org.uid), source=SOURCE) before = [self.existing(org) for org in self.orgs] with pytest.raises(SnapshotValidationError, match="no_completed_organizations"): self.run_refresh() artifact = ParserSourceArtifact.objects.get(source=SOURCE) assert artifact.status == artifact.Status.REJECTED assert artifact.metadata["quota_skipped"] == 2 assert before == [self.existing(org) for org in self.orgs] def test_overlapping_provider_pages_preserve_previous_four_records(self): from copy import deepcopy from apps.parsers.checko_sections import collect_checko_section from tests.apps.parsers.test_checko_sections import client_with, fixture org = self.orgs[0] OrganizationSourceIngestionService.save_records( source=SOURCE, load_batch=1, records=[record(org, str(n)) for n in range(4)], ) before = self.existing(org) first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json") shared = deepcopy(first["data"]["Записи"][0]) first["data"]["Записи"].append(dict(shared, GUID="other-first")) second["data"]["Записи"].insert(0, shared) for response in (first, second): response["company"].update({"ИНН": org.inn, "ОГРН": org.ogrn}) response["data"]["ЗапВсего"] = 4 for message in response["data"]["Записи"]: message["Публикатор"] = org.ogrn message["Участники"] = [org.inn] rows = collect_checko_section( client_with(first, second), organization=org, source=SOURCE, ) (artifact, _), _ = self.run_refresh(rows=[rows, [record(self.orgs[1])]]) assert self.existing(org) == before assert artifact.metadata["organizations_failed"] == 1 assert artifact.metadata["completeness"] == "partial" assert not ParserStagedRecord.objects.filter( artifact=artifact, organization=org ).exists() attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE) assert attempt.status == attempt.Status.FAILED assert ( claim_monthly_collection(organization_id=str(org.uid), source=SOURCE) is None ) def test_publication_failure_rolls_back_records(self): self.seed(self.orgs[0]) before = self.existing(self.orgs[0]) def reject(*args): raise SnapshotValidationError("fixture_publication_failed") with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"): self.run_refresh(on_publish=reject) assert self.existing(self.orgs[0]) == before assert not self.existing(self.orgs[1]) assert ( ParserSourceArtifact.objects.get(source=SOURCE).status == ParserSourceArtifact.Status.REJECTED ) assert CheckoCollectionAttempt.objects.count() == 2 def test_uid_stable_across_months(self): self.run_refresh(load_batch=1) before = [self.existing(org)[0][0] for org in self.orgs] # Advance the calendar; never delete or reset a monthly claim. with patch( "apps.parsers.checko_collection.collection_period_month", return_value=date(2026, 11, 1), ): self.run_refresh(load_batch=2) assert before == [self.existing(org)[0][0] for org in self.orgs] def test_sources_have_independent_nonreleased_claims(self): assert set(adapter().CHECKO_SECTIONS) <= set( CheckoCollectionAttempt.Source.values ) for source in adapter().CHECKO_SECTIONS: attempt = claim_monthly_collection( organization_id=str(self.orgs[0].uid), source=source ) assert attempt is not None finish_collection(attempt, records_count=0, error="fixture") assert ( claim_monthly_collection( organization_id=str(self.orgs[0].uid), source=source ) is None ) def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self): self.orgs[1].directory_imported_at = None self.orgs[1].save() Organization.objects.create( name="Outside", inn="7710140679", ogrn="1027700425174", okpo="87654321", directory_imported_at=timezone.now(), ) (artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]]) assert collector.call_count == 1 assert artifact.metadata["organizations_total"] == 1 assert Organization.objects.count() == 3 def test_invalid_canonical_identifiers_preserved_without_http(self): self.orgs[1].okpo = "1234567890" self.orgs[1].save() (artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]]) assert collector.call_count == 1 assert artifact.metadata["organizations_failed"] == 1 assert artifact.metadata["completeness"] == "partial" @pytest.mark.django_db(transaction=True) def test_concurrent_monthly_claim_one_winner(): from django.db import connection from tests.apps.parsers.test_refresh_concurrency import _parallel if connection.vendor != "postgresql": pytest.skip("Requires PostgreSQL concurrent unique claims") org = Organization.objects.create( name="Claim", inn="7707083893", ogrn="1027700132195", okpo="12345678", directory_imported_at=timezone.now(), ) def claim(): return ( claim_monthly_collection(organization_id=str(org.uid), source=SOURCE) is not None ) assert sorted(_parallel([claim, claim])) == [False, True] assert CheckoCollectionAttempt.objects.count() == 1