Files
mostovik-backend/tests/apps/parsers/test_checko_section_snapshots.py
2026-10-29 17:46:26 +01:00

320 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Real DB scoped publication with a synthetic, quota-aware transport boundary."""
import importlib
from datetime import date
from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch
import pytest
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.models import (
CheckoCollectionAttempt,
ParserSourceArtifact,
ParserStagedRecord,
)
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
SOURCE = "checko_messages"
def adapter():
return importlib.import_module("apps.parsers.checko_section_snapshots")
def record(org, suffix="one"):
return SourceRecordInput(
external_id=f"{org.inn}:{suffix}",
title="Сообщение",
organization_name=org.name,
organization_uid=org.uid,
inn=org.inn,
ogrn=org.ogrn,
record_type="company_message",
payload={"upstream": {"GUID": suffix}},
)
class CheckoSectionSnapshotTests(TestCase):
def setUp(self):
self.tmp = TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
settings = override_settings(
MEDIA_ROOT=self.tmp.name + "/public",
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
)
settings.enable()
self.addCleanup(settings.disable)
self.orgs = [
Organization.objects.create(
name=f"АО Тест {n}",
inn=inn,
ogrn=ogrn,
okpo=f"1234567{n}",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
for n, (inn, ogrn) in enumerate(
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
)
]
self.orgs.sort(key=lambda org: org.inn)
def run_refresh(self, rows=None, **kwargs):
module = adapter()
collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs])
client = Mock()
client.__enter__ = Mock(return_value=client)
client.__exit__ = Mock(return_value=False)
with patch.object(module, "_client", return_value=client), patch.object(
module, "collect_checko_section", collector
):
result = module.refresh_checko_section(
source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs
)
return result, collector
def seed(self, org):
OrganizationSourceIngestionService.save_records(
source=SOURCE, load_batch=1, records=[record(org, "old")]
)
def existing(self, org):
return list(
OrganizationSourceRecord.objects.filter(
source=SOURCE, extension__organization=org
).values_list("uid", "external_id", "load_batch")
)
def test_claimed_organization_keeps_previous_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
claim_weekly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
assert self.existing(self.orgs[0]) == before
assert collector.call_count == 1
assert artifact.metadata["completeness"] == "partial"
assert (
artifact.metadata["quota_skipped"]
== artifact.metadata["organizations_preserved"]
== 1
)
assert result.published == 2
assert artifact.metadata["updated_records_count"] == 1
assert artifact.metadata["retained_records_count"] == 1
def test_full_counter_partition_and_private_artifact(self):
(artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []])
assert artifact.metadata["completeness"] == "full"
assert artifact.metadata["organizations_total"] == 2
assert (
artifact.metadata["organizations_updated"]
== artifact.metadata["organizations_empty"]
== 1
)
assert (
artifact.metadata["organizations_failed"]
== artifact.metadata["quota_skipped"]
== 0
)
assert result.published == 1
assert len(artifact.sha256) == 64
assert artifact.size_bytes > 0
with pytest.raises(ValueError):
_ = artifact.file.url
def test_empty_success_clears_only_its_organization(self):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[1])
claim_weekly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
self.run_refresh(rows=[[]])
assert not self.existing(self.orgs[0])
assert self.existing(self.orgs[1]) == before
def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[0])
def interrupted():
for n in range(501):
yield record(self.orgs[0], str(n))
raise SnapshotValidationError("pagination_failed")
(artifact, _), _ = self.run_refresh(
rows=[interrupted(), [record(self.orgs[1])]]
)
assert self.existing(self.orgs[0]) == before
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["organizations_preserved"] == 1
assert not ParserStagedRecord.objects.filter(
artifact=artifact, organization=self.orgs[0]
).exists()
attempt = CheckoCollectionAttempt.objects.get(
organization=self.orgs[0], source=SOURCE
)
assert attempt.status == attempt.Status.FAILED
assert (
claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=SOURCE
)
is None
)
def test_no_completed_scope_rejects_without_publication(self):
for org in self.orgs:
self.seed(org)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
before = [self.existing(org) for org in self.orgs]
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_refresh()
artifact = ParserSourceArtifact.objects.get(source=SOURCE)
assert artifact.status == artifact.Status.REJECTED
assert artifact.metadata["quota_skipped"] == 2
assert before == [self.existing(org) for org in self.orgs]
def test_overlapping_provider_pages_preserve_previous_four_records(self):
from copy import deepcopy
from apps.parsers.checko_sections import collect_checko_section
from tests.apps.parsers.test_checko_sections import client_with, fixture
org = self.orgs[0]
OrganizationSourceIngestionService.save_records(
source=SOURCE,
load_batch=1,
records=[record(org, str(n)) for n in range(4)],
)
before = self.existing(org)
first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json")
shared = deepcopy(first["data"]["Записи"][0])
first["data"]["Записи"].append(dict(shared, GUID="other-first"))
second["data"]["Записи"].insert(0, shared)
for response in (first, second):
response["company"].update({"ИНН": org.inn, "ОГРН": org.ogrn})
response["data"]["ЗапВсего"] = 4
for message in response["data"]["Записи"]:
message["Публикатор"] = org.ogrn
message["Участники"] = [org.inn]
rows = collect_checko_section(
client_with(first, second),
organization=org,
source=SOURCE,
)
(artifact, _), _ = self.run_refresh(rows=[rows, [record(self.orgs[1])]])
assert self.existing(org) == before
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["completeness"] == "partial"
assert not ParserStagedRecord.objects.filter(
artifact=artifact, organization=org
).exists()
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
assert attempt.status == attempt.Status.FAILED
assert (
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
is None
)
def test_publication_failure_rolls_back_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
def reject(*args):
raise SnapshotValidationError("fixture_publication_failed")
with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"):
self.run_refresh(on_publish=reject)
assert self.existing(self.orgs[0]) == before
assert not self.existing(self.orgs[1])
assert (
ParserSourceArtifact.objects.get(source=SOURCE).status
== ParserSourceArtifact.Status.REJECTED
)
assert CheckoCollectionAttempt.objects.count() == 2
def test_uid_stable_across_months(self):
self.run_refresh(load_batch=1)
before = [self.existing(org)[0][0] for org in self.orgs]
# Advance the calendar; never delete or reset a monthly claim.
with patch(
"apps.parsers.checko_collection.collection_period_week",
return_value=date(2026, 11, 1),
):
self.run_refresh(load_batch=2)
assert before == [self.existing(org)[0][0] for org in self.orgs]
def test_sources_have_independent_nonreleased_claims(self):
assert set(adapter().CHECKO_SECTIONS) <= set(
CheckoCollectionAttempt.Source.values
)
for source in adapter().CHECKO_SECTIONS:
attempt = claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
assert attempt is not None
finish_collection(attempt, records_count=0, error="fixture")
assert (
claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
is None
)
def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self):
self.orgs[1].directory_imported_at = None
self.orgs[1].save()
Organization.objects.create(
name="Outside",
inn="7710140679",
ogrn="1027700425174",
okpo="87654321",
directory_imported_at=timezone.now(),
)
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
assert collector.call_count == 1
assert artifact.metadata["organizations_total"] == 1
assert Organization.objects.count() == 3
def test_invalid_canonical_identifiers_preserved_without_http(self):
self.orgs[1].okpo = "1234567890"
self.orgs[1].save()
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
assert collector.call_count == 1
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["completeness"] == "partial"
@pytest.mark.django_db(transaction=True)
def test_concurrent_monthly_claim_one_winner():
from django.db import connection
from tests.apps.parsers.test_refresh_concurrency import _parallel
if connection.vendor != "postgresql":
pytest.skip("Requires PostgreSQL concurrent unique claims")
org = Organization.objects.create(
name="Claim",
inn="7707083893",
ogrn="1027700132195",
okpo="12345678",
directory_imported_at=timezone.now(),
)
def claim():
return (
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
is not None
)
assert sorted(_parallel([claim, claim])) == [False, True]
assert CheckoCollectionAttempt.objects.count() == 1