320 lines
12 KiB
Python
320 lines
12 KiB
Python
"""Real DB scoped publication with a synthetic, quota-aware transport boundary."""
|
||
|
||
import importlib
|
||
from datetime import date
|
||
from tempfile import TemporaryDirectory
|
||
from unittest.mock import Mock, patch
|
||
|
||
import pytest
|
||
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||
from apps.parsers.models import (
|
||
CheckoCollectionAttempt,
|
||
ParserSourceArtifact,
|
||
ParserStagedRecord,
|
||
)
|
||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||
from django.test import TestCase, override_settings
|
||
from django.utils import timezone
|
||
from organizations.models import Organization, OrganizationSourceRecord
|
||
from organizations.source_ingestion import (
|
||
OrganizationSourceIngestionService,
|
||
SourceRecordInput,
|
||
)
|
||
|
||
SOURCE = "checko_messages"
|
||
|
||
|
||
def adapter():
|
||
return importlib.import_module("apps.parsers.checko_section_snapshots")
|
||
|
||
|
||
def record(org, suffix="one"):
|
||
return SourceRecordInput(
|
||
external_id=f"{org.inn}:{suffix}",
|
||
title="Сообщение",
|
||
organization_name=org.name,
|
||
organization_uid=org.uid,
|
||
inn=org.inn,
|
||
ogrn=org.ogrn,
|
||
record_type="company_message",
|
||
payload={"upstream": {"GUID": suffix}},
|
||
)
|
||
|
||
|
||
class CheckoSectionSnapshotTests(TestCase):
|
||
def setUp(self):
|
||
self.tmp = TemporaryDirectory()
|
||
self.addCleanup(self.tmp.cleanup)
|
||
settings = override_settings(
|
||
MEDIA_ROOT=self.tmp.name + "/public",
|
||
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
|
||
)
|
||
settings.enable()
|
||
self.addCleanup(settings.disable)
|
||
self.orgs = [
|
||
Organization.objects.create(
|
||
name=f"АО Тест {n}",
|
||
inn=inn,
|
||
ogrn=ogrn,
|
||
okpo=f"1234567{n}",
|
||
directory_imported_at=timezone.now(),
|
||
opk_registry_membership=True,
|
||
)
|
||
for n, (inn, ogrn) in enumerate(
|
||
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
|
||
)
|
||
]
|
||
|
||
self.orgs.sort(key=lambda org: org.inn)
|
||
|
||
def run_refresh(self, rows=None, **kwargs):
|
||
module = adapter()
|
||
collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs])
|
||
client = Mock()
|
||
client.__enter__ = Mock(return_value=client)
|
||
client.__exit__ = Mock(return_value=False)
|
||
with patch.object(module, "_client", return_value=client), patch.object(
|
||
module, "collect_checko_section", collector
|
||
):
|
||
result = module.refresh_checko_section(
|
||
source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs
|
||
)
|
||
return result, collector
|
||
|
||
def seed(self, org):
|
||
OrganizationSourceIngestionService.save_records(
|
||
source=SOURCE, load_batch=1, records=[record(org, "old")]
|
||
)
|
||
|
||
def existing(self, org):
|
||
return list(
|
||
OrganizationSourceRecord.objects.filter(
|
||
source=SOURCE, extension__organization=org
|
||
).values_list("uid", "external_id", "load_batch")
|
||
)
|
||
|
||
def test_claimed_organization_keeps_previous_records(self):
|
||
self.seed(self.orgs[0])
|
||
before = self.existing(self.orgs[0])
|
||
claim_weekly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
|
||
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
|
||
assert self.existing(self.orgs[0]) == before
|
||
assert collector.call_count == 1
|
||
assert artifact.metadata["completeness"] == "partial"
|
||
assert (
|
||
artifact.metadata["quota_skipped"]
|
||
== artifact.metadata["organizations_preserved"]
|
||
== 1
|
||
)
|
||
assert result.published == 2
|
||
assert artifact.metadata["updated_records_count"] == 1
|
||
assert artifact.metadata["retained_records_count"] == 1
|
||
|
||
def test_full_counter_partition_and_private_artifact(self):
|
||
(artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []])
|
||
assert artifact.metadata["completeness"] == "full"
|
||
assert artifact.metadata["organizations_total"] == 2
|
||
assert (
|
||
artifact.metadata["organizations_updated"]
|
||
== artifact.metadata["organizations_empty"]
|
||
== 1
|
||
)
|
||
assert (
|
||
artifact.metadata["organizations_failed"]
|
||
== artifact.metadata["quota_skipped"]
|
||
== 0
|
||
)
|
||
assert result.published == 1
|
||
assert len(artifact.sha256) == 64
|
||
assert artifact.size_bytes > 0
|
||
with pytest.raises(ValueError):
|
||
_ = artifact.file.url
|
||
|
||
def test_empty_success_clears_only_its_organization(self):
|
||
for org in self.orgs:
|
||
self.seed(org)
|
||
before = self.existing(self.orgs[1])
|
||
claim_weekly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
|
||
self.run_refresh(rows=[[]])
|
||
assert not self.existing(self.orgs[0])
|
||
assert self.existing(self.orgs[1]) == before
|
||
|
||
def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self):
|
||
for org in self.orgs:
|
||
self.seed(org)
|
||
before = self.existing(self.orgs[0])
|
||
|
||
def interrupted():
|
||
for n in range(501):
|
||
yield record(self.orgs[0], str(n))
|
||
raise SnapshotValidationError("pagination_failed")
|
||
|
||
(artifact, _), _ = self.run_refresh(
|
||
rows=[interrupted(), [record(self.orgs[1])]]
|
||
)
|
||
assert self.existing(self.orgs[0]) == before
|
||
assert artifact.metadata["organizations_failed"] == 1
|
||
assert artifact.metadata["organizations_preserved"] == 1
|
||
assert not ParserStagedRecord.objects.filter(
|
||
artifact=artifact, organization=self.orgs[0]
|
||
).exists()
|
||
attempt = CheckoCollectionAttempt.objects.get(
|
||
organization=self.orgs[0], source=SOURCE
|
||
)
|
||
assert attempt.status == attempt.Status.FAILED
|
||
assert (
|
||
claim_weekly_collection(
|
||
organization_id=str(self.orgs[0].uid), source=SOURCE
|
||
)
|
||
is None
|
||
)
|
||
|
||
def test_no_completed_scope_rejects_without_publication(self):
|
||
for org in self.orgs:
|
||
self.seed(org)
|
||
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||
before = [self.existing(org) for org in self.orgs]
|
||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||
self.run_refresh()
|
||
artifact = ParserSourceArtifact.objects.get(source=SOURCE)
|
||
assert artifact.status == artifact.Status.REJECTED
|
||
assert artifact.metadata["quota_skipped"] == 2
|
||
assert before == [self.existing(org) for org in self.orgs]
|
||
|
||
def test_overlapping_provider_pages_preserve_previous_four_records(self):
|
||
from copy import deepcopy
|
||
|
||
from apps.parsers.checko_sections import collect_checko_section
|
||
|
||
from tests.apps.parsers.test_checko_sections import client_with, fixture
|
||
|
||
org = self.orgs[0]
|
||
OrganizationSourceIngestionService.save_records(
|
||
source=SOURCE,
|
||
load_batch=1,
|
||
records=[record(org, str(n)) for n in range(4)],
|
||
)
|
||
before = self.existing(org)
|
||
first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json")
|
||
shared = deepcopy(first["data"]["Записи"][0])
|
||
first["data"]["Записи"].append(dict(shared, GUID="other-first"))
|
||
second["data"]["Записи"].insert(0, shared)
|
||
for response in (first, second):
|
||
response["company"].update({"ИНН": org.inn, "ОГРН": org.ogrn})
|
||
response["data"]["ЗапВсего"] = 4
|
||
for message in response["data"]["Записи"]:
|
||
message["Публикатор"] = org.ogrn
|
||
message["Участники"] = [org.inn]
|
||
rows = collect_checko_section(
|
||
client_with(first, second),
|
||
organization=org,
|
||
source=SOURCE,
|
||
)
|
||
(artifact, _), _ = self.run_refresh(rows=[rows, [record(self.orgs[1])]])
|
||
assert self.existing(org) == before
|
||
assert artifact.metadata["organizations_failed"] == 1
|
||
assert artifact.metadata["completeness"] == "partial"
|
||
assert not ParserStagedRecord.objects.filter(
|
||
artifact=artifact, organization=org
|
||
).exists()
|
||
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
|
||
assert attempt.status == attempt.Status.FAILED
|
||
assert (
|
||
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||
is None
|
||
)
|
||
|
||
def test_publication_failure_rolls_back_records(self):
|
||
self.seed(self.orgs[0])
|
||
before = self.existing(self.orgs[0])
|
||
|
||
def reject(*args):
|
||
raise SnapshotValidationError("fixture_publication_failed")
|
||
|
||
with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"):
|
||
self.run_refresh(on_publish=reject)
|
||
assert self.existing(self.orgs[0]) == before
|
||
assert not self.existing(self.orgs[1])
|
||
assert (
|
||
ParserSourceArtifact.objects.get(source=SOURCE).status
|
||
== ParserSourceArtifact.Status.REJECTED
|
||
)
|
||
assert CheckoCollectionAttempt.objects.count() == 2
|
||
|
||
def test_uid_stable_across_months(self):
|
||
self.run_refresh(load_batch=1)
|
||
before = [self.existing(org)[0][0] for org in self.orgs]
|
||
# Advance the calendar; never delete or reset a monthly claim.
|
||
with patch(
|
||
"apps.parsers.checko_collection.collection_period_week",
|
||
return_value=date(2026, 11, 1),
|
||
):
|
||
self.run_refresh(load_batch=2)
|
||
assert before == [self.existing(org)[0][0] for org in self.orgs]
|
||
|
||
def test_sources_have_independent_nonreleased_claims(self):
|
||
assert set(adapter().CHECKO_SECTIONS) <= set(
|
||
CheckoCollectionAttempt.Source.values
|
||
)
|
||
for source in adapter().CHECKO_SECTIONS:
|
||
attempt = claim_weekly_collection(
|
||
organization_id=str(self.orgs[0].uid), source=source
|
||
)
|
||
assert attempt is not None
|
||
finish_collection(attempt, records_count=0, error="fixture")
|
||
assert (
|
||
claim_weekly_collection(
|
||
organization_id=str(self.orgs[0].uid), source=source
|
||
)
|
||
is None
|
||
)
|
||
|
||
def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self):
|
||
self.orgs[1].directory_imported_at = None
|
||
self.orgs[1].save()
|
||
Organization.objects.create(
|
||
name="Outside",
|
||
inn="7710140679",
|
||
ogrn="1027700425174",
|
||
okpo="87654321",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
|
||
assert collector.call_count == 1
|
||
assert artifact.metadata["organizations_total"] == 1
|
||
assert Organization.objects.count() == 3
|
||
|
||
def test_invalid_canonical_identifiers_preserved_without_http(self):
|
||
self.orgs[1].okpo = "1234567890"
|
||
self.orgs[1].save()
|
||
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
|
||
assert collector.call_count == 1
|
||
assert artifact.metadata["organizations_failed"] == 1
|
||
assert artifact.metadata["completeness"] == "partial"
|
||
|
||
|
||
@pytest.mark.django_db(transaction=True)
|
||
def test_concurrent_monthly_claim_one_winner():
|
||
from django.db import connection
|
||
|
||
from tests.apps.parsers.test_refresh_concurrency import _parallel
|
||
|
||
if connection.vendor != "postgresql":
|
||
pytest.skip("Requires PostgreSQL concurrent unique claims")
|
||
org = Organization.objects.create(
|
||
name="Claim",
|
||
inn="7707083893",
|
||
ogrn="1027700132195",
|
||
okpo="12345678",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
|
||
def claim():
|
||
return (
|
||
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||
is not None
|
||
)
|
||
|
||
assert sorted(_parallel([claim, claim])) == [False, True]
|
||
assert CheckoCollectionAttempt.objects.count() == 1
|