From 650a47f6c949d6c234fde9091e6b997372a74f20 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 16:29:58 +0200 Subject: [PATCH] feat: publish quota-safe Checko section snapshots --- src/apps/parsers/artifact_storage.py | 3 + src/apps/parsers/checko_section_snapshots.py | 215 ++++++++++++++ .../migrations/0044_checko_sections.py | 38 +++ src/apps/parsers/models.py | 8 + src/apps/parsers/registry_snapshots.py | 4 +- .../migrations/0017_checko_sections.py | 55 ++++ src/organizations/models.py | 37 +++ src/organizations/source_groups.py | 24 ++ .../parsers/test_checko_section_snapshots.py | 276 ++++++++++++++++++ 9 files changed, 658 insertions(+), 2 deletions(-) create mode 100644 src/apps/parsers/checko_section_snapshots.py create mode 100644 src/apps/parsers/migrations/0044_checko_sections.py create mode 100644 src/organizations/migrations/0017_checko_sections.py create mode 100644 tests/apps/parsers/test_checko_section_snapshots.py diff --git a/src/apps/parsers/artifact_storage.py b/src/apps/parsers/artifact_storage.py index e5e492f..bc5ff60 100644 --- a/src/apps/parsers/artifact_storage.py +++ b/src/apps/parsers/artifact_storage.py @@ -8,6 +8,9 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag from django.utils.deconstruct import deconstructible PRIVATE_ARTIFACT_PREFIXES = ( + "parser-artifacts/checko_leasing/", + "parser-artifacts/checko_messages/", + "parser-artifacts/checko_licenses/", "parser-artifacts/sro_membership_check/", "parser-artifacts/integrated_structure_documents/", "parser-artifacts/fedresurs_company_profile/", diff --git a/src/apps/parsers/checko_section_snapshots.py b/src/apps/parsers/checko_section_snapshots.py new file mode 100644 index 0000000..743786a --- /dev/null +++ b/src/apps/parsers/checko_section_snapshots.py @@ -0,0 +1,215 @@ +"""Monthly, canonical and scoped Checko collection; incomplete organizations survive.""" + +from __future__ import annotations + +import json +import re +from tempfile import TemporaryFile + +from apps.parsers.checko_collection import claim_monthly_collection, finish_collection +from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section +from apps.parsers.clients.checko import CheckoClient +from apps.parsers.fssp_csv import valid_legal_inn +from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord +from apps.parsers.registry_snapshots import ( + SNAPSHOT_CHUNK_SIZE, + OwnOrganizationIndex, + SnapshotValidationError, + publish_snapshot, + reject_snapshot, + save_artifact_file, + stage_records, +) +from django.conf import settings +from django.db.models import Q +from organizations.models import Organization + + +def _client(): + api_key = getattr(settings, "CHECKO_API_KEY", "") + if not api_key: + raise SnapshotValidationError("checko_api_not_configured") + proxies = None + if getattr(settings, "CHECKO_USE_RUNTIME_PROXIES", False) and getattr( + settings, "PARSER_USE_RUNTIME_PROXIES", False + ): + from apps.parsers.services import ProxyService + + proxies = ProxyService.get_runtime_proxies_or_none() + return CheckoClient(api_key=api_key, proxies=proxies, timeout=30) + + +def _organizations(): + # Same existing eligibility as paid collectors, with authoritative directory required. + scope = ( + Q(opk_registry_membership=True) + | Q(gk_name__icontains="Росатом") + | Q(gk_name__icontains="Роскосмос") + | Q(gk_code__iexact="rosatom") + | Q(gk_code__iexact="roscosmos") + | Q(gk_code__iexact="roskosmos") + ) + return Organization.objects.filter( + scope, directory_imported_at__isnull=False + ).order_by("inn", "ogrn", "uid") + + +def _eligible_identity(org, index): + if ( + not valid_legal_inn(org.inn) + or not re.fullmatch(r"[0-9]{13}", org.ogrn or "") + or not re.fullmatch(r"[0-9]{8}", org.okpo or "") + ): + return False + resolved, reason = index.resolve(inn=org.inn, ogrn=org.ogrn) + return not reason and resolved.uid == org.uid + + +def _collect_organization(client, org, source, artifact, row_number, raw_file): + pending = [] + count = 0 + for row in collect_checko_section(client, organization=org, source=source): + # Adapter binds exact identifiers; never let a row escape its claimed scope. + if ( + row.inn != org.inn + or row.ogrn != org.ogrn + or row.record_type != CHECKO_SECTIONS[source] + ): + raise SnapshotValidationError("checko_record_outside_claim_scope") + count += 1 + row_number += 1 + normalized = { + "title": row.title, + "record_date": row.record_date, + "amount": str(row.amount) if row.amount is not None else None, + "status": row.status, + "url": row.url, + "payload": row.payload, + } + pending.append( + ParserStagedRecord( + artifact=artifact, + row_number=row_number, + external_id=row.external_id, + record_type=row.record_type, + organization=org, + normalized_data=normalized, + ) + ) + raw_file.write( + ( + json.dumps( + { + "organization_uid": str(org.uid), + "external_id": row.external_id, + "record": normalized, + }, + ensure_ascii=False, + sort_keys=True, + ) + + "\n" + ).encode() + ) + if len(pending) == SNAPSHOT_CHUNK_SIZE: + stage_records(pending) + stage_records(pending) + return count, row_number + + +def _collect_scope(client, organizations, source, artifact, raw_file, on_progress): + index = OwnOrganizationIndex() + success, preserved = [], [] + counts = { + "organizations_total": organizations.count(), + "organizations_updated": 0, + "organizations_empty": 0, + "organizations_preserved": 0, + "organizations_failed": 0, + "quota_skipped": 0, + } + row_number = 0 + for number, org in enumerate(organizations.iterator(), 1): + if not _eligible_identity(org, index): + counts["organizations_failed"] += 1 + preserved.append(org.uid) + elif ( + attempt := claim_monthly_collection( + organization_id=str(org.uid), source=source + ) + ) is None: + counts["quota_skipped"] += 1 + preserved.append(org.uid) + else: + # Roll failed organization's raw observations back too; no partial data is public. + offset = raw_file.tell() + try: + count, row_number = _collect_organization( + client, org, source, artifact, row_number, raw_file + ) + except SnapshotValidationError as exc: + ParserStagedRecord.objects.filter( + artifact=artifact, organization=org + ).delete() + raw_file.seek(offset) + raw_file.truncate() + finish_collection(attempt, records_count=0, error=str(exc)[:200]) + counts["organizations_failed"] += 1 + preserved.append(org.uid) + else: + finish_collection(attempt, records_count=count) + counts["organizations_updated" if count else "organizations_empty"] += 1 + success.append(org.uid) + if on_progress: + on_progress( + number, counts["organizations_total"], len(success), len(preserved) + ) + counts["organizations_preserved"] = len(preserved) + counts["completeness"] = "full" if not preserved else "partial" + artifact.metadata = counts + artifact.parsed_count = artifact.staged_records.count() + return success, preserved + + +def refresh_checko_section( + *, + source: str, + load_batch: int, + uploaded_by_id: int | None = None, + on_publish=None, + on_progress=None, +): + """Publish only fully exhausted organization collections; never release a claim.""" + if source not in CHECKO_SECTIONS: + raise SnapshotValidationError("unknown_checko_section") + organizations = _organizations() + if not organizations.exists(): + raise SnapshotValidationError("empty_active_directory") + artifact = ParserSourceArtifact.objects.create( + source=source, + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + version="api-v2.5/sections-v1", + content_type="application/x-ndjson", + original_name=f"{source}.jsonl", + ) + try: + with _client() as client, TemporaryFile() as raw_file: + successful, preserved = _collect_scope( + client, organizations, source, artifact, raw_file, on_progress + ) + save_artifact_file(artifact, raw_file) + if not successful: + raise SnapshotValidationError("no_completed_organizations") + artifact.status = artifact.Status.PARSED + artifact.save() + result = publish_snapshot( + artifact, + scope_organization_ids=successful, + preserve_organization_ids=preserved, + on_publish=on_publish, + ) + return artifact, result + except Exception as exc: + artifact.save(update_fields=["metadata", "parsed_count"]) + reject_snapshot(artifact, exc) + raise diff --git a/src/apps/parsers/migrations/0044_checko_sections.py b/src/apps/parsers/migrations/0044_checko_sections.py new file mode 100644 index 0000000..e1330d1 --- /dev/null +++ b/src/apps/parsers/migrations/0044_checko_sections.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.25 on 2026-10-05 14:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0043_fssp_disabled_schedule'), + ] + + operations = [ + migrations.AlterField( + model_name='checkocollectionattempt', + name='source', + field=models.CharField(choices=[('arbitration', 'Арбитражные дела'), ('bankruptcy', 'Банкротства'), ('contracts', 'Контракты'), ('inspections', 'Проверки'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], max_length=32, verbose_name='источник'), + ), + migrations.AlterField( + model_name='genericparserrecord', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций'), ('hh', 'Вакансии HeadHunter'), ('superjob', 'Вакансии SuperJob')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parserbatchsequence', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], help_text='Источник данных', max_length=50, unique=True, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parserloadlog', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parsersourceartifact', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('fssp_enforcement_proceedings', 'Исполнительные производства ФССП'), ('checko_leasing', 'Лизинг'), ('checko_messages', 'Сообщения организаций'), ('checko_licenses', 'Лицензии организаций')], db_index=True, max_length=50), + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 65b67c8..3b14eaa 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -66,6 +66,10 @@ class ParserLoadLog(TimestampMixin, models.Model): _("Исполнительные производства ФССП"), ) + CHECKO_LEASING = "checko_leasing", _("Лизинг") + CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций") + CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций") + class Status(models.TextChoices): SUCCESS = "success", _("Успешно") FAILED = "failed", _("Ошибка") @@ -300,6 +304,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model): PROCUREMENTS_223FZ = "procurements_223fz", _("Закупки 223-ФЗ") UNFAIR_SUPPLIERS = "unfair_suppliers", _("Недобросовестные поставщики") + CHECKO_LEASING = "checko_leasing", _("Лизинг") + CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций") + CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций") + class Status(models.TextChoices): IN_PROGRESS = "in_progress", _("В процессе") SUCCESS = "success", _("Успешно") diff --git a/src/apps/parsers/registry_snapshots.py b/src/apps/parsers/registry_snapshots.py index d62b33d..97ea469 100644 --- a/src/apps/parsers/registry_snapshots.py +++ b/src/apps/parsers/registry_snapshots.py @@ -157,8 +157,8 @@ def _remove_obsolete_records( ) retained_count = 0 if preserve_organization_ids: - protected = obsolete.filter( - extension__organization_id__in=preserve_organization_ids + protected = OrganizationSourceRecord.objects.filter( + source=source, extension__organization_id__in=preserve_organization_ids ) retained_count = protected.exclude(external_id__in=retained_ids).count() obsolete = obsolete.exclude( diff --git a/src/organizations/migrations/0017_checko_sections.py b/src/organizations/migrations/0017_checko_sections.py new file mode 100644 index 0000000..e9d958e --- /dev/null +++ b/src/organizations/migrations/0017_checko_sections.py @@ -0,0 +1,55 @@ +# Generated by Django 3.2.25 on 2026-10-05 14:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('organizations', '0016_fssp_sources'), + ] + + operations = [ + migrations.CreateModel( + name='CompanyLicenseExtension', + fields=[ + ('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')), + ], + options={ + 'verbose_name': 'лицензии организации', + 'verbose_name_plural': 'лицензии организации', + 'db_table': 'organizations_company_license_extension', + }, + bases=('organizations.organizationsourceextension',), + ), + migrations.CreateModel( + name='CompanyMessageExtension', + fields=[ + ('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')), + ], + options={ + 'verbose_name': 'сообщения организации', + 'verbose_name_plural': 'сообщения организации', + 'db_table': 'organizations_company_message_extension', + }, + bases=('organizations.organizationsourceextension',), + ), + migrations.CreateModel( + name='LeasingExtension', + fields=[ + ('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')), + ], + options={ + 'verbose_name': 'лизинг', + 'verbose_name_plural': 'лизинг', + 'db_table': 'organizations_leasing_extension', + }, + bases=('organizations.organizationsourceextension',), + ), + migrations.AlterField( + model_name='organizationsourceextension', + name='source_group', + field=models.CharField(choices=[('financial_indicators', 'Финансово-экономические показатели'), ('government_procurements', 'Государственные закупки'), ('industrial_production', 'Производители и продукция России'), ('planned_inspections', 'Плановые проверки'), ('bankruptcy', 'Сведения о процедурах банкротства'), ('defense_suppliers', 'Недобросовестные поставщики ГОЗ'), ('arbitration', 'Арбитражные дела'), ('security_registries', 'Реестры по информационной безопасности'), ('vacancies', 'Вакансии'), ('electronic_document_exchange', 'Электронный документооборот'), ('media_mentions', 'Упоминания в СМИ'), ('sanctions', 'Санкции'), ('government_support', 'Государственная поддержка'), ('sro_membership', 'Членство в СРО'), ('budget_process_registry', 'Реестр участников бюджетного процесса'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('enforcement_proceedings', 'Исполнительные производства'), ('leasing', 'Лизинг'), ('company_messages', 'Сообщения организаций'), ('company_licenses', 'Лицензии организаций')], db_index=True, max_length=64, verbose_name='группа источников'), + ), + ] diff --git a/src/organizations/models.py b/src/organizations/models.py index 8761893..204c9b7 100644 --- a/src/organizations/models.py +++ b/src/organizations/models.py @@ -57,6 +57,10 @@ class SourceGroup(models.TextChoices): _("Исполнительные производства"), ) + LEASING = "leasing", _("Лизинг") + COMPANY_MESSAGES = "company_messages", _("Сообщения организаций") + COMPANY_LICENSES = "company_licenses", _("Лицензии организаций") + class SourceExtensionStatus(models.TextChoices): """Lifecycle status for an organization source extension.""" @@ -1033,3 +1037,36 @@ class OrganizationDataSnapshot(models.Model): ) super().save(*args, **kwargs) + + +class LeasingExtension(OrganizationSourceExtension): + """Verified external leasing records for one canonical organization.""" + + source_group_value = SourceGroup.LEASING + + class Meta: + db_table = "organizations_leasing_extension" + verbose_name = _("лизинг") + verbose_name_plural = _("лизинг") + + +class CompanyMessageExtension(OrganizationSourceExtension): + """Verified external company_message records for one canonical organization.""" + + source_group_value = SourceGroup.COMPANY_MESSAGES + + class Meta: + db_table = "organizations_company_message_extension" + verbose_name = _("сообщения организации") + verbose_name_plural = _("сообщения организации") + + +class CompanyLicenseExtension(OrganizationSourceExtension): + """Verified external company_license records for one canonical organization.""" + + source_group_value = SourceGroup.COMPANY_LICENSES + + class Meta: + db_table = "organizations_company_license_extension" + verbose_name = _("лицензии организации") + verbose_name_plural = _("лицензии организации") diff --git a/src/organizations/source_groups.py b/src/organizations/source_groups.py index 8343a90..8d73fac 100644 --- a/src/organizations/source_groups.py +++ b/src/organizations/source_groups.py @@ -10,6 +10,8 @@ from organizations.models import ( ArbitrationExtension, BankruptcyExtension, BudgetProcessRegistryExtension, + CompanyLicenseExtension, + CompanyMessageExtension, DefenseSupplierExtension, ElectronicDocumentExchangeExtension, EnforcementProceedingExtension, @@ -19,6 +21,7 @@ from organizations.models import ( GovernmentSupportExtension, IndustrialProductionExtension, IntegratedStructureExtension, + LeasingExtension, MediaMentionExtension, OrganizationSourceExtension, PlannedInspectionExtension, @@ -42,6 +45,27 @@ class SourceGroupDescriptor: SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = { + ParserLoadLog.Source.CHECKO_LICENSES: SourceGroupDescriptor( + source=ParserLoadLog.Source.CHECKO_LICENSES, + source_group=SourceGroup.COMPANY_LICENSES, + record_type="company_license", + title="Лицензии организаций", + extension_model=CompanyLicenseExtension, + ), + ParserLoadLog.Source.CHECKO_MESSAGES: SourceGroupDescriptor( + source=ParserLoadLog.Source.CHECKO_MESSAGES, + source_group=SourceGroup.COMPANY_MESSAGES, + record_type="company_message", + title="Сообщения организаций", + extension_model=CompanyMessageExtension, + ), + ParserLoadLog.Source.CHECKO_LEASING: SourceGroupDescriptor( + source=ParserLoadLog.Source.CHECKO_LEASING, + source_group=SourceGroup.LEASING, + record_type="leasing_contract", + title="Лизинг", + extension_model=LeasingExtension, + ), ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS: SourceGroupDescriptor( source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS, source_group=SourceGroup.ENFORCEMENT_PROCEEDINGS, diff --git a/tests/apps/parsers/test_checko_section_snapshots.py b/tests/apps/parsers/test_checko_section_snapshots.py new file mode 100644 index 0000000..d2ca297 --- /dev/null +++ b/tests/apps/parsers/test_checko_section_snapshots.py @@ -0,0 +1,276 @@ +"""Real DB scoped publication with a synthetic, quota-aware transport boundary.""" + +import importlib +from datetime import date +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +import pytest +from apps.parsers.checko_collection import claim_monthly_collection, finish_collection +from apps.parsers.models import ( + CheckoCollectionAttempt, + ParserSourceArtifact, + ParserStagedRecord, +) +from apps.parsers.registry_snapshots import SnapshotValidationError +from django.test import TestCase, override_settings +from django.utils import timezone +from organizations.models import Organization, OrganizationSourceRecord +from organizations.source_ingestion import ( + OrganizationSourceIngestionService, + SourceRecordInput, +) + +SOURCE = "checko_messages" + + +def adapter(): + return importlib.import_module("apps.parsers.checko_section_snapshots") + + +def record(org, suffix="one"): + return SourceRecordInput( + external_id=f"{org.inn}:{suffix}", + title="Сообщение", + organization_name=org.name, + organization_uid=org.uid, + inn=org.inn, + ogrn=org.ogrn, + record_type="company_message", + payload={"upstream": {"GUID": suffix}}, + ) + + +class CheckoSectionSnapshotTests(TestCase): + def setUp(self): + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + settings = override_settings( + MEDIA_ROOT=self.tmp.name + "/public", + PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private", + ) + settings.enable() + self.addCleanup(settings.disable) + self.orgs = [ + Organization.objects.create( + name=f"АО Тест {n}", + inn=inn, + ogrn=ogrn, + okpo=f"1234567{n}", + directory_imported_at=timezone.now(), + opk_registry_membership=True, + ) + for n, (inn, ogrn) in enumerate( + [("7707083893", "1027700132195"), ("7702070139", "1027700070518")] + ) + ] + + self.orgs.sort(key=lambda org: org.inn) + + def run_refresh(self, rows=None, **kwargs): + module = adapter() + collector = Mock(side_effect=rows or [[record(org)] for org in self.orgs]) + client = Mock() + client.__enter__ = Mock(return_value=client) + client.__exit__ = Mock(return_value=False) + with patch.object(module, "_client", return_value=client), patch.object( + module, "collect_checko_section", collector + ): + result = module.refresh_checko_section( + source=SOURCE, load_batch=kwargs.pop("load_batch", 2), **kwargs + ) + return result, collector + + def seed(self, org): + OrganizationSourceIngestionService.save_records( + source=SOURCE, load_batch=1, records=[record(org, "old")] + ) + + def existing(self, org): + return list( + OrganizationSourceRecord.objects.filter( + source=SOURCE, extension__organization=org + ).values_list("uid", "external_id", "load_batch") + ) + + def test_claimed_organization_keeps_previous_records(self): + self.seed(self.orgs[0]) + before = self.existing(self.orgs[0]) + claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE) + (artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]]) + assert self.existing(self.orgs[0]) == before + assert collector.call_count == 1 + assert artifact.metadata["completeness"] == "partial" + assert ( + artifact.metadata["quota_skipped"] + == artifact.metadata["organizations_preserved"] + == 1 + ) + assert result.published == 2 + assert artifact.metadata["updated_records_count"] == 1 + assert artifact.metadata["retained_records_count"] == 1 + + def test_full_counter_partition_and_private_artifact(self): + (artifact, result), _ = self.run_refresh(rows=[[record(self.orgs[0])], []]) + assert artifact.metadata["completeness"] == "full" + assert artifact.metadata["organizations_total"] == 2 + assert ( + artifact.metadata["organizations_updated"] + == artifact.metadata["organizations_empty"] + == 1 + ) + assert ( + artifact.metadata["organizations_failed"] + == artifact.metadata["quota_skipped"] + == 0 + ) + assert result.published == 1 + assert len(artifact.sha256) == 64 + assert artifact.size_bytes > 0 + with pytest.raises(ValueError): + _ = artifact.file.url + + def test_empty_success_clears_only_its_organization(self): + for org in self.orgs: + self.seed(org) + before = self.existing(self.orgs[1]) + claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE) + self.run_refresh(rows=[[]]) + assert not self.existing(self.orgs[0]) + assert self.existing(self.orgs[1]) == before + + def test_pagination_failure_discards_all_org_staging_and_keeps_claim(self): + for org in self.orgs: + self.seed(org) + before = self.existing(self.orgs[0]) + + def interrupted(): + for n in range(501): + yield record(self.orgs[0], str(n)) + raise SnapshotValidationError("pagination_failed") + + (artifact, _), _ = self.run_refresh( + rows=[interrupted(), [record(self.orgs[1])]] + ) + assert self.existing(self.orgs[0]) == before + assert artifact.metadata["organizations_failed"] == 1 + assert artifact.metadata["organizations_preserved"] == 1 + assert not ParserStagedRecord.objects.filter( + artifact=artifact, organization=self.orgs[0] + ).exists() + attempt = CheckoCollectionAttempt.objects.get( + organization=self.orgs[0], source=SOURCE + ) + assert attempt.status == attempt.Status.FAILED + assert ( + claim_monthly_collection( + organization_id=str(self.orgs[0].uid), source=SOURCE + ) + is None + ) + + def test_no_completed_scope_rejects_without_publication(self): + for org in self.orgs: + self.seed(org) + claim_monthly_collection(organization_id=str(org.uid), source=SOURCE) + before = [self.existing(org) for org in self.orgs] + with pytest.raises(SnapshotValidationError, match="no_completed_organizations"): + self.run_refresh() + artifact = ParserSourceArtifact.objects.get(source=SOURCE) + assert artifact.status == artifact.Status.REJECTED + assert artifact.metadata["quota_skipped"] == 2 + assert before == [self.existing(org) for org in self.orgs] + + def test_publication_failure_rolls_back_records(self): + self.seed(self.orgs[0]) + before = self.existing(self.orgs[0]) + + def reject(*args): + raise SnapshotValidationError("fixture_publication_failed") + + with pytest.raises(SnapshotValidationError, match="fixture_publication_failed"): + self.run_refresh(on_publish=reject) + assert self.existing(self.orgs[0]) == before + assert not self.existing(self.orgs[1]) + assert ( + ParserSourceArtifact.objects.get(source=SOURCE).status + == ParserSourceArtifact.Status.REJECTED + ) + assert CheckoCollectionAttempt.objects.count() == 2 + + def test_uid_stable_across_months(self): + self.run_refresh(load_batch=1) + before = [self.existing(org)[0][0] for org in self.orgs] + # Advance the calendar; never delete or reset a monthly claim. + with patch( + "apps.parsers.checko_collection.collection_period_month", + return_value=date(2026, 11, 1), + ): + self.run_refresh(load_batch=2) + assert before == [self.existing(org)[0][0] for org in self.orgs] + + def test_sources_have_independent_nonreleased_claims(self): + assert set(adapter().CHECKO_SECTIONS) <= set( + CheckoCollectionAttempt.Source.values + ) + for source in adapter().CHECKO_SECTIONS: + attempt = claim_monthly_collection( + organization_id=str(self.orgs[0].uid), source=source + ) + assert attempt is not None + finish_collection(attempt, records_count=0, error="fixture") + assert ( + claim_monthly_collection( + organization_id=str(self.orgs[0].uid), source=source + ) + is None + ) + + def test_noncanonical_and_ineligible_orgs_do_not_call_provider(self): + self.orgs[1].directory_imported_at = None + self.orgs[1].save() + Organization.objects.create( + name="Outside", + inn="7710140679", + ogrn="1027700425174", + okpo="87654321", + directory_imported_at=timezone.now(), + ) + (artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]]) + assert collector.call_count == 1 + assert artifact.metadata["organizations_total"] == 1 + assert Organization.objects.count() == 3 + + def test_invalid_canonical_identifiers_preserved_without_http(self): + self.orgs[1].okpo = "1234567890" + self.orgs[1].save() + (artifact, _), collector = self.run_refresh(rows=[[record(self.orgs[0])]]) + assert collector.call_count == 1 + assert artifact.metadata["organizations_failed"] == 1 + assert artifact.metadata["completeness"] == "partial" + + +@pytest.mark.django_db(transaction=True) +def test_concurrent_monthly_claim_one_winner(): + from django.db import connection + + from tests.apps.parsers.test_refresh_concurrency import _parallel + + if connection.vendor != "postgresql": + pytest.skip("Requires PostgreSQL concurrent unique claims") + org = Organization.objects.create( + name="Claim", + inn="7707083893", + ogrn="1027700132195", + okpo="12345678", + directory_imported_at=timezone.now(), + ) + + def claim(): + return ( + claim_monthly_collection(organization_id=str(org.uid), source=SOURCE) + is not None + ) + + assert sorted(_parallel([claim, claim])) == [False, True] + assert CheckoCollectionAttempt.objects.count() == 1