feat: publish quota-safe Checko section snapshots

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:29:58 +02:00
parent a0f623effe
commit 650a47f6c9
9 changed files with 658 additions and 2 deletions

View File

@@ -8,6 +8,9 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag
from django.utils.deconstruct import deconstructible
PRIVATE_ARTIFACT_PREFIXES = (
"parser-artifacts/checko_leasing/",
"parser-artifacts/checko_messages/",
"parser-artifacts/checko_licenses/",
"parser-artifacts/sro_membership_check/",
"parser-artifacts/integrated_structure_documents/",
"parser-artifacts/fedresurs_company_profile/",

View File

@@ -0,0 +1,215 @@
"""Monthly, canonical and scoped Checko collection; incomplete organizations survive."""
from __future__ import annotations
import json
import re
from tempfile import TemporaryFile
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
from apps.parsers.clients.checko import CheckoClient
from apps.parsers.fssp_csv import valid_legal_inn
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from apps.parsers.registry_snapshots import (
SNAPSHOT_CHUNK_SIZE,
OwnOrganizationIndex,
SnapshotValidationError,
publish_snapshot,
reject_snapshot,
save_artifact_file,
stage_records,
)
from django.conf import settings
from django.db.models import Q
from organizations.models import Organization
def _client():
api_key = getattr(settings, "CHECKO_API_KEY", "")
if not api_key:
raise SnapshotValidationError("checko_api_not_configured")
proxies = None
if getattr(settings, "CHECKO_USE_RUNTIME_PROXIES", False) and getattr(
settings, "PARSER_USE_RUNTIME_PROXIES", False
):
from apps.parsers.services import ProxyService
proxies = ProxyService.get_runtime_proxies_or_none()
return CheckoClient(api_key=api_key, proxies=proxies, timeout=30)
def _organizations():
# Same existing eligibility as paid collectors, with authoritative directory required.
scope = (
Q(opk_registry_membership=True)
| Q(gk_name__icontains="Росатом")
| Q(gk_name__icontains="Роскосмос")
| Q(gk_code__iexact="rosatom")
| Q(gk_code__iexact="roscosmos")
| Q(gk_code__iexact="roskosmos")
)
return Organization.objects.filter(
scope, directory_imported_at__isnull=False
).order_by("inn", "ogrn", "uid")
def _eligible_identity(org, index):
if (
not valid_legal_inn(org.inn)
or not re.fullmatch(r"[0-9]{13}", org.ogrn or "")
or not re.fullmatch(r"[0-9]{8}", org.okpo or "")
):
return False
resolved, reason = index.resolve(inn=org.inn, ogrn=org.ogrn)
return not reason and resolved.uid == org.uid
def _collect_organization(client, org, source, artifact, row_number, raw_file):
pending = []
count = 0
for row in collect_checko_section(client, organization=org, source=source):
# Adapter binds exact identifiers; never let a row escape its claimed scope.
if (
row.inn != org.inn
or row.ogrn != org.ogrn
or row.record_type != CHECKO_SECTIONS[source]
):
raise SnapshotValidationError("checko_record_outside_claim_scope")
count += 1
row_number += 1
normalized = {
"title": row.title,
"record_date": row.record_date,
"amount": str(row.amount) if row.amount is not None else None,
"status": row.status,
"url": row.url,
"payload": row.payload,
}
pending.append(
ParserStagedRecord(
artifact=artifact,
row_number=row_number,
external_id=row.external_id,
record_type=row.record_type,
organization=org,
normalized_data=normalized,
)
)
raw_file.write(
(
json.dumps(
{
"organization_uid": str(org.uid),
"external_id": row.external_id,
"record": normalized,
},
ensure_ascii=False,
sort_keys=True,
)
+ "\n"
).encode()
)
if len(pending) == SNAPSHOT_CHUNK_SIZE:
stage_records(pending)
stage_records(pending)
return count, row_number
def _collect_scope(client, organizations, source, artifact, raw_file, on_progress):
index = OwnOrganizationIndex()
success, preserved = [], []
counts = {
"organizations_total": organizations.count(),
"organizations_updated": 0,
"organizations_empty": 0,
"organizations_preserved": 0,
"organizations_failed": 0,
"quota_skipped": 0,
}
row_number = 0
for number, org in enumerate(organizations.iterator(), 1):
if not _eligible_identity(org, index):
counts["organizations_failed"] += 1
preserved.append(org.uid)
elif (
attempt := claim_monthly_collection(
organization_id=str(org.uid), source=source
)
) is None:
counts["quota_skipped"] += 1
preserved.append(org.uid)
else:
# Roll failed organization's raw observations back too; no partial data is public.
offset = raw_file.tell()
try:
count, row_number = _collect_organization(
client, org, source, artifact, row_number, raw_file
)
except SnapshotValidationError as exc:
ParserStagedRecord.objects.filter(
artifact=artifact, organization=org
).delete()
raw_file.seek(offset)
raw_file.truncate()
finish_collection(attempt, records_count=0, error=str(exc)[:200])
counts["organizations_failed"] += 1
preserved.append(org.uid)
else:
finish_collection(attempt, records_count=count)
counts["organizations_updated" if count else "organizations_empty"] += 1
success.append(org.uid)
if on_progress:
on_progress(
number, counts["organizations_total"], len(success), len(preserved)
)
counts["organizations_preserved"] = len(preserved)
counts["completeness"] = "full" if not preserved else "partial"
artifact.metadata = counts
artifact.parsed_count = artifact.staged_records.count()
return success, preserved
def refresh_checko_section(
*,
source: str,
load_batch: int,
uploaded_by_id: int | None = None,
on_publish=None,
on_progress=None,
):
"""Publish only fully exhausted organization collections; never release a claim."""
if source not in CHECKO_SECTIONS:
raise SnapshotValidationError("unknown_checko_section")
organizations = _organizations()
if not organizations.exists():
raise SnapshotValidationError("empty_active_directory")
artifact = ParserSourceArtifact.objects.create(
source=source,
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
version="api-v2.5/sections-v1",
content_type="application/x-ndjson",
original_name=f"{source}.jsonl",
)
try:
with _client() as client, TemporaryFile() as raw_file:
successful, preserved = _collect_scope(
client, organizations, source, artifact, raw_file, on_progress
)
save_artifact_file(artifact, raw_file)
if not successful:
raise SnapshotValidationError("no_completed_organizations")
artifact.status = artifact.Status.PARSED
artifact.save()
result = publish_snapshot(
artifact,
scope_organization_ids=successful,
preserve_organization_ids=preserved,
on_publish=on_publish,
)
return artifact, result
except Exception as exc:
artifact.save(update_fields=["metadata", "parsed_count"])
reject_snapshot(artifact, exc)
raise

View File

@@ -0,0 +1,38 @@
# Generated by Django 3.2.25 on 2026-10-05 14:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('parsers', '0043_fssp_disabled_schedule'),
]
operations = [
migrations.AlterField(
model_name='checkocollectionattempt',
name='source',
field=models.CharField(choices=[('arbitration', 'Арбитражные дела'), ('bankruptcy', 'Банкротства'), ('contracts', 'Контракты'), ('inspections', 'Проверки'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], max_length=32, verbose_name='источник'),
),
migrations.AlterField(
model_name='genericparserrecord',
name='source',
field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций'), ('hh', 'Вакансии HeadHunter'), ('superjob', 'Вакансии SuperJob')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'),
),
migrations.AlterField(
model_name='parserbatchsequence',
name='source',
field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], help_text='Источник данных', max_length=50, unique=True, verbose_name='источник'),
),
migrations.AlterField(
model_name='parserloadlog',
name='source',
field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'),
),
migrations.AlterField(
model_name='parsersourceartifact',
name='source',
field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], db_index=True, max_length=50),
),
]

View File

@@ -66,6 +66,10 @@ class ParserLoadLog(TimestampMixin, models.Model):
_("Исполнительные производства ФССП"),
)
CHECKO_LEASING = "checko_leasing", _("Лизинг")
CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций")
CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций")
class Status(models.TextChoices):
SUCCESS = "success", _("Успешно")
FAILED = "failed", _("Ошибка")
@@ -300,6 +304,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
PROCUREMENTS_223FZ = "procurements_223fz", _("Закупки 223-ФЗ")
UNFAIR_SUPPLIERS = "unfair_suppliers", _("Недобросовестные поставщики")
CHECKO_LEASING = "checko_leasing", _("Лизинг")
CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций")
CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций")
class Status(models.TextChoices):
IN_PROGRESS = "in_progress", _("В процессе")
SUCCESS = "success", _("Успешно")

View File

@@ -157,8 +157,8 @@ def _remove_obsolete_records(
)
retained_count = 0
if preserve_organization_ids:
protected = obsolete.filter(
extension__organization_id__in=preserve_organization_ids
protected = OrganizationSourceRecord.objects.filter(
source=source, extension__organization_id__in=preserve_organization_ids
)
retained_count = protected.exclude(external_id__in=retained_ids).count()
obsolete = obsolete.exclude(

View File

@@ -0,0 +1,55 @@
# Generated by Django 3.2.25 on 2026-10-05 14:28
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('organizations', '0016_fssp_sources'),
]
operations = [
migrations.CreateModel(
name='CompanyLicenseExtension',
fields=[
('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')),
],
options={
'verbose_name': 'лицензии организации',
'verbose_name_plural': 'лицензии организации',
'db_table': 'organizations_company_license_extension',
},
bases=('organizations.organizationsourceextension',),
),
migrations.CreateModel(
name='CompanyMessageExtension',
fields=[
('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')),
],
options={
'verbose_name': 'сообщения организации',
'verbose_name_plural': 'сообщения организации',
'db_table': 'organizations_company_message_extension',
},
bases=('organizations.organizationsourceextension',),
),
migrations.CreateModel(
name='LeasingExtension',
fields=[
('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')),
],
options={
'verbose_name': 'лизинг',
'verbose_name_plural': 'лизинг',
'db_table': 'organizations_leasing_extension',
},
bases=('organizations.organizationsourceextension',),
),
migrations.AlterField(
model_name='organizationsourceextension',
name='source_group',
field=models.CharField(choices=[('financial_indicators', 'Финансово-экономические показатели'), ('government_procurements', 'Государственные закупки'), ('industrial_production', 'Производители и продукция России'), ('planned_inspections', 'Плановые проверки'), ('bankruptcy', 'Сведения о процедурах банкротства'), ('defense_suppliers', 'Недобросовестные поставщики ГОЗ'), ('arbitration', 'Арбитражные дела'), ('security_registries', 'Реестры по информационной безопасности'), ('vacancies', 'Вакансии'), ('electronic_document_exchange', 'Электронный документооборот'), ('media_mentions', 'Упоминания в СМИ'), ('sanctions', 'Санкции'), ('government_support', 'Государственная поддержка'), ('sro_membership', 'Членство в СРО'), ('budget_process_registry', 'Реестр участников бюджетного процесса'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('enforcement_proceedings', 'Исполнительные производства'), ('leasing', 'Лизинг'), ('company_messages', 'Сообщения организаций'), ('company_licenses', 'Лицензии организаций')], db_index=True, max_length=64, verbose_name='группа источников'),
),
]

View File

@@ -57,6 +57,10 @@ class SourceGroup(models.TextChoices):
_("Исполнительные производства"),
)
LEASING = "leasing", _("Лизинг")
COMPANY_MESSAGES = "company_messages", _("Сообщения организаций")
COMPANY_LICENSES = "company_licenses", _("Лицензии организаций")
class SourceExtensionStatus(models.TextChoices):
"""Lifecycle status for an organization source extension."""
@@ -1033,3 +1037,36 @@ class OrganizationDataSnapshot(models.Model):
)
super().save(*args, **kwargs)
class LeasingExtension(OrganizationSourceExtension):
"""Verified external leasing records for one canonical organization."""
source_group_value = SourceGroup.LEASING
class Meta:
db_table = "organizations_leasing_extension"
verbose_name = _("лизинг")
verbose_name_plural = _("лизинг")
class CompanyMessageExtension(OrganizationSourceExtension):
"""Verified external company_message records for one canonical organization."""
source_group_value = SourceGroup.COMPANY_MESSAGES
class Meta:
db_table = "organizations_company_message_extension"
verbose_name = _("сообщения организации")
verbose_name_plural = _("сообщения организации")
class CompanyLicenseExtension(OrganizationSourceExtension):
"""Verified external company_license records for one canonical organization."""
source_group_value = SourceGroup.COMPANY_LICENSES
class Meta:
db_table = "organizations_company_license_extension"
verbose_name = _("лицензии организации")
verbose_name_plural = _("лицензии организации")

View File

@@ -10,6 +10,8 @@ from organizations.models import (
ArbitrationExtension,
BankruptcyExtension,
BudgetProcessRegistryExtension,
CompanyLicenseExtension,
CompanyMessageExtension,
DefenseSupplierExtension,
ElectronicDocumentExchangeExtension,
EnforcementProceedingExtension,
@@ -19,6 +21,7 @@ from organizations.models import (
GovernmentSupportExtension,
IndustrialProductionExtension,
IntegratedStructureExtension,
LeasingExtension,
MediaMentionExtension,
OrganizationSourceExtension,
PlannedInspectionExtension,
@@ -42,6 +45,27 @@ class SourceGroupDescriptor:
SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = {
ParserLoadLog.Source.CHECKO_LICENSES: SourceGroupDescriptor(
source=ParserLoadLog.Source.CHECKO_LICENSES,
source_group=SourceGroup.COMPANY_LICENSES,
record_type="company_license",
title="Лицензии организаций",
extension_model=CompanyLicenseExtension,
),
ParserLoadLog.Source.CHECKO_MESSAGES: SourceGroupDescriptor(
source=ParserLoadLog.Source.CHECKO_MESSAGES,
source_group=SourceGroup.COMPANY_MESSAGES,
record_type="company_message",
title="Сообщения организаций",
extension_model=CompanyMessageExtension,
),
ParserLoadLog.Source.CHECKO_LEASING: SourceGroupDescriptor(
source=ParserLoadLog.Source.CHECKO_LEASING,
source_group=SourceGroup.LEASING,
record_type="leasing_contract",
title="Лизинг",
extension_model=LeasingExtension,
),
ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS: SourceGroupDescriptor(
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
source_group=SourceGroup.ENFORCEMENT_PROCEEDINGS,

View File

@@ -0,0 +1,276 @@
"""Real DB scoped publication with a synthetic, quota-aware transport boundary."""
import importlib
from datetime import date
from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch
import pytest
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.models import (
CheckoCollectionAttempt,
ParserSourceArtifact,
ParserStagedRecord,
)
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
SOURCE = "checko_messages"
def adapter():
return importlib.import_module("apps.parsers.checko_section_snapshots")
def record(org, suffix="one"):
return SourceRecordInput(
external_id=f"{org.inn}:{suffix}",
title="Сообщение",
organization_name=org.name,
organization_uid=org.uid,
inn=org.inn,
ogrn=org.ogrn,
record_type="company_message",
payload={"upstream": {"GUID": suffix}},
)
class CheckoSectionSnapshotTests(TestCase):
def setUp(self):
self.tmp = TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
settings = override_settings(
MEDIA_ROOT=self.tmp.name + "/public",
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
)
settings.enable()
self.addCleanup(settings.disable)
self.orgs = [
Organization.objects.create(
name=f"АО Тест {n}",
inn=inn,
ogrn=ogrn,
okpo=f"1234567{n}",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
for n, (inn, ogrn) in enumerate(
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
)
]
self.orgs.sort(key=lambda org: org.inn)
def run_refresh(self, rows=None, **kwargs):
module = adapter()
collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs])
client = Mock()
client.__enter__ = Mock(return_value=client)
client.__exit__ = Mock(return_value=False)
with patch.object(module, "_client", return_value=client), patch.object(
module, "collect_checko_section", collector
):
result = module.refresh_checko_section(
source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs
)
return result, collector
def seed(self, org):
OrganizationSourceIngestionService.save_records(
source=SOURCE, load_batch=1, records=[record(org, "old")]
)
def existing(self, org):
return list(
OrganizationSourceRecord.objects.filter(
source=SOURCE, extension__organization=org
).values_list("uid", "external_id", "load_batch")
)
def test_claimed_organization_keeps_previous_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
assert self.existing(self.orgs[0]) == before
assert collector.call_count == 1
assert artifact.metadata["completeness"] == "partial"
assert (
artifact.metadata["quota_skipped"]
== artifact.metadata["organizations_preserved"]
== 1
)
assert result.published == 2
assert artifact.metadata["updated_records_count"] == 1
assert artifact.metadata["retained_records_count"] == 1
def test_full_counter_partition_and_private_artifact(self):
(artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []])
assert artifact.metadata["completeness"] == "full"
assert artifact.metadata["organizations_total"] == 2
assert (
artifact.metadata["organizations_updated"]
== artifact.metadata["organizations_empty"]
== 1
)
assert (
artifact.metadata["organizations_failed"]
== artifact.metadata["quota_skipped"]
== 0
)
assert result.published == 1
assert len(artifact.sha256) == 64
assert artifact.size_bytes > 0
with pytest.raises(ValueError):
_ = artifact.file.url
def test_empty_success_clears_only_its_organization(self):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[1])
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
self.run_refresh(rows=[[]])
assert not self.existing(self.orgs[0])
assert self.existing(self.orgs[1]) == before
def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[0])
def interrupted():
for n in range(501):
yield record(self.orgs[0], str(n))
raise SnapshotValidationError("pagination_failed")
(artifact, _), _ = self.run_refresh(
rows=[interrupted(), [record(self.orgs[1])]]
)
assert self.existing(self.orgs[0]) == before
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["organizations_preserved"] == 1
assert not ParserStagedRecord.objects.filter(
artifact=artifact, organization=self.orgs[0]
).exists()
attempt = CheckoCollectionAttempt.objects.get(
organization=self.orgs[0], source=SOURCE
)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(
organization_id=str(self.orgs[0].uid), source=SOURCE
)
is None
)
def test_no_completed_scope_rejects_without_publication(self):
for org in self.orgs:
self.seed(org)
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
before = [self.existing(org) for org in self.orgs]
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_refresh()
artifact = ParserSourceArtifact.objects.get(source=SOURCE)
assert artifact.status == artifact.Status.REJECTED
assert artifact.metadata["quota_skipped"] == 2
assert before == [self.existing(org) for org in self.orgs]
def test_publication_failure_rolls_back_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
def reject(*args):
raise SnapshotValidationError("fixture_publication_failed")
with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"):
self.run_refresh(on_publish=reject)
assert self.existing(self.orgs[0]) == before
assert not self.existing(self.orgs[1])
assert (
ParserSourceArtifact.objects.get(source=SOURCE).status
== ParserSourceArtifact.Status.REJECTED
)
assert CheckoCollectionAttempt.objects.count() == 2
def test_uid_stable_across_months(self):
self.run_refresh(load_batch=1)
before = [self.existing(org)[0][0] for org in self.orgs]
# Advance the calendar; never delete or reset a monthly claim.
with patch(
"apps.parsers.checko_collection.collection_period_month",
return_value=date(2026, 11, 1),
):
self.run_refresh(load_batch=2)
assert before == [self.existing(org)[0][0] for org in self.orgs]
def test_sources_have_independent_nonreleased_claims(self):
assert set(adapter().CHECKO_SECTIONS) <= set(
CheckoCollectionAttempt.Source.values
)
for source in adapter().CHECKO_SECTIONS:
attempt = claim_monthly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
assert attempt is not None
finish_collection(attempt, records_count=0, error="fixture")
assert (
claim_monthly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
is None
)
def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self):
self.orgs[1].directory_imported_at = None
self.orgs[1].save()
Organization.objects.create(
name="Outside",
inn="7710140679",
ogrn="1027700425174",
okpo="87654321",
directory_imported_at=timezone.now(),
)
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
assert collector.call_count == 1
assert artifact.metadata["organizations_total"] == 1
assert Organization.objects.count() == 3
def test_invalid_canonical_identifiers_preserved_without_http(self):
self.orgs[1].okpo = "1234567890"
self.orgs[1].save()
(artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]])
assert collector.call_count == 1
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["completeness"] == "partial"
@pytest.mark.django_db(transaction=True)
def test_concurrent_monthly_claim_one_winner():
from django.db import connection
from tests.apps.parsers.test_refresh_concurrency import _parallel
if connection.vendor != "postgresql":
pytest.skip("Requires PostgreSQL concurrent unique claims")
org = Organization.objects.create(
name="Claim",
inn="7707083893",
ogrn="1027700132195",
okpo="12345678",
directory_imported_at=timezone.now(),
)
def claim():
return (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
is not None
)
assert sorted(_parallel([claim, claim])) == [False, True]
assert CheckoCollectionAttempt.objects.count() == 1