From 1884ecc5469f594959826083d50c67e7528c4e98 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 15:35:10 +0200 Subject: [PATCH] feat: publish complete FSSP snapshots atomically --- src/apps/parsers/artifact_storage.py | 1 + src/apps/parsers/fssp_snapshot.py | 366 ++++++++++++++++++ .../parsers/migrations/0042_fssp_sources.py | 204 ++++++++++ src/apps/parsers/models.py | 4 + .../migrations/0016_fssp_sources.py | 70 ++++ src/organizations/models.py | 15 + src/organizations/source_groups.py | 8 + src/settings/base.py | 3 + tests/apps/parsers/test_fssp_snapshot.py | 300 ++++++++++++++ 9 files changed, 971 insertions(+) create mode 100644 src/apps/parsers/fssp_snapshot.py create mode 100644 src/apps/parsers/migrations/0042_fssp_sources.py create mode 100644 src/organizations/migrations/0016_fssp_sources.py create mode 100644 tests/apps/parsers/test_fssp_snapshot.py diff --git a/src/apps/parsers/artifact_storage.py b/src/apps/parsers/artifact_storage.py index ee8221e..e5e492f 100644 --- a/src/apps/parsers/artifact_storage.py +++ b/src/apps/parsers/artifact_storage.py @@ -11,6 +11,7 @@ PRIVATE_ARTIFACT_PREFIXES = ( "parser-artifacts/sro_membership_check/", "parser-artifacts/integrated_structure_documents/", "parser-artifacts/fedresurs_company_profile/", + "parser-artifacts/fssp_enforcement_proceedings/", ) diff --git a/src/apps/parsers/fssp_snapshot.py b/src/apps/parsers/fssp_snapshot.py new file mode 100644 index 0000000..13765c6 --- /dev/null +++ b/src/apps/parsers/fssp_snapshot.py @@ -0,0 +1,366 @@ +"""Two-input FSSP staging and one atomic canonical publication.""" + +from __future__ import annotations + +import csv +import json +import re +import tempfile +from collections import Counter +from collections.abc import Callable, Iterable +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import date +from io import BytesIO, StringIO, TextIOWrapper +from typing import BinaryIO +from urllib.parse import urlparse + +from apps.parsers.fssp_csv import ( + ENDED_DATASET, + FSSP_DATASETS, + FSSP_SOURCE, + iter_fssp_records, + valid_legal_inn, +) +from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord +from apps.parsers.registry_http import ( + download_registry_archive, + limited_bytes, + registry_response, + registry_session, +) +from apps.parsers.registry_snapshots import ( + SNAPSHOT_CHUNK_SIZE, + OwnOrganizationIndex, + SnapshotResult, + SnapshotValidationError, + publish_snapshot, + reject_snapshot, + save_artifact_file, + stage_records, +) +from django.conf import settings +from django.db.models import Count + + +@dataclass(frozen=True) +class FsspInput: + dataset: str + handle: BinaryIO + original_name: str + published_at: date + artifact: ParserSourceArtifact | None = None + + +def _identity(row, index: OwnOrganizationIndex): + if not valid_legal_inn(row.inn): + return None, "invalid_debtor_inn" + if not row.payload["proceeding_number"] or len(row.external_id) > 255: + return None, "invalid_proceeding_number" + if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]: + return None, "missing_fssp_required_field" + return index.resolve(inn=row.inn, ogrn="") + + +def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress): + pending = [] + item.handle.seek(0) + text = TextIOWrapper(item.handle, encoding="utf-8-sig", newline="") + try: + rows = iter_fssp_records( + text, + dataset=item.dataset, + snapshot_date=item.published_at.isoformat(), + fetched_at=raw_artifact.created_at.isoformat(), + ) + for row in rows: + counts["parsed"] += 1 + raw_artifact.parsed_count += 1 + organization, reason = _identity(row, index) + if reason == "outside_directory": + counts["skipped"] += 1 + elif reason: + counts["quarantined"] += 1 + counts[reason] += 1 + if on_progress is not None and counts["parsed"] % 10000 == 0: + on_progress( + counts["parsed"], + 0, + counts["parsed"] - counts["skipped"], + counts["quarantined"], + ) + if reason == "outside_directory": + continue + provenance = { + "artifact_id": str(raw_artifact.uid), + "dataset": item.dataset, + "published_at": item.published_at.isoformat(), + "row_number": row.payload["source_row_number"], + "checksum_sha256": raw_artifact.sha256, + } + row.payload["provenance"] = provenance + pending.append( + ParserStagedRecord( + artifact=aggregate, + row_number=counts["parsed"], + organization=organization, + external_id=row.external_id, + record_type=row.record_type, + raw_data=row.payload["raw_columns"], + normalized_data={ + "title": row.title, + "record_date": row.record_date, + "amount": format(row.amount, ".2f") + if row.amount is not None + else None, + "status": row.status, + "url": row.url, + "payload": row.payload, + }, + disposition="quarantined" if reason else "staged", + reason=reason, + ) + ) + if len(pending) >= SNAPSHOT_CHUNK_SIZE: + stage_records(pending) + finally: + text.detach() + stage_records(pending) + raw_artifact.status = ParserSourceArtifact.Status.PARSED + raw_artifact.save() + + +def _deduplicate(artifact): + staged = artifact.staged_records.filter(disposition="staged") + duplicates = ( + staged.order_by().values("external_id").annotate(n=Count("uid")).filter(n__gt=1) + ) + for duplicate in duplicates.iterator(chunk_size=SNAPSHOT_CHUNK_SIZE): + if duplicate["n"] > 20: + raise SnapshotValidationError("excessive_fssp_duplicates") + rows = list( + staged.filter(external_id=duplicate["external_id"]).order_by("row_number") + ) + by_dataset = {} + for row in rows: + dataset = row.normalized_data["payload"]["dataset"] + previous = by_dataset.get(dataset) + if previous is not None and previous.raw_data != row.raw_data: + raise SnapshotValidationError("conflicting_fssp_identity") + by_dataset[dataset] = row + selected = rows[0] + if len(by_dataset) > 1: + selected = by_dataset[ENDED_DATASET] + ended_date = selected.normalized_data["payload"]["provenance"][ + "published_at" + ] + if any( + row.normalized_data["payload"]["provenance"]["published_at"] + > ended_date + for row in rows + ): + raise SnapshotValidationError("conflicting_fssp_identity") + selected.normalized_data["payload"]["observations"] = [ + { + "provenance": row.normalized_data["payload"]["provenance"], + "raw_columns": row.raw_data, + } + for row in rows + ] + selected.save(update_fields=("normalized_data",)) + staged.filter(external_id=duplicate["external_id"]).exclude( + pk=selected.pk + ).delete() + + +def import_fssp_snapshots( + *, + inputs: Iterable[FsspInput], + load_batch: int, + uploaded_by_id: int | None = None, + on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None, + on_progress: Callable[[int, int, int, int], None] | None = None, +) -> tuple[ParserSourceArtifact, SnapshotResult]: + inputs = tuple(inputs) + if len(inputs) != 2 or {item.dataset for item in inputs} != set(FSSP_DATASETS): + raise SnapshotValidationError("missing_fssp_input") + artifact = ParserSourceArtifact.objects.create( + source=FSSP_SOURCE, + original_name="fssp-manifest.json", + content_type="application/json", + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + source_published_at=max(item.published_at for item in inputs), + version="general-20260820T2100+ended-20220620T2006", + ) + counts: Counter = Counter() + raw_artifacts = [] + try: + index = OwnOrganizationIndex() + for item in inputs: + raw = item.artifact or ParserSourceArtifact.objects.create( + source=FSSP_SOURCE, + original_name=item.original_name, + content_type="text/csv", + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + source_published_at=item.published_at, + metadata={ + "dataset": item.dataset, + "aggregate_artifact_id": str(artifact.uid), + }, + ) + raw_artifacts.append(raw) + raw.metadata = {**raw.metadata, "aggregate_artifact_id": str(artifact.uid)} + raw.save(update_fields=("metadata",)) + if item.artifact is None: + save_artifact_file(raw, item.handle) + _stage_input(item, raw, artifact, index, counts, on_progress) + _deduplicate(artifact) + artifact.parsed_count = counts["parsed"] + artifact.quarantined_count = counts["quarantined"] + artifact.metadata = { + "inputs": [ + { + "artifact_id": str(raw.uid), + "dataset": raw.metadata["dataset"], + "checksum_sha256": raw.sha256, + "size_bytes": raw.size_bytes, + "published_at": raw.source_published_at.isoformat(), + "parsed_records_count": raw.parsed_count, + } + for raw in raw_artifacts + ], + "skipped_records_count": counts["skipped"], + "reasons": { + key: value + for key, value in counts.items() + if key not in ("parsed", "skipped", "quarantined") + }, + "deduplicated_records_count": counts["parsed"] + - counts["skipped"] + - counts["quarantined"] + - artifact.staged_records.filter(disposition="staged").count(), + } + artifact.rejection_reasons = artifact.metadata["reasons"] + manifest = BytesIO( + json.dumps(artifact.metadata, ensure_ascii=False, sort_keys=True).encode() + ) + save_artifact_file(artifact, manifest) + artifact.status = ParserSourceArtifact.Status.PARSED + artifact.save() + return artifact, publish_snapshot(artifact, on_publish=on_publish) + except Exception as exc: + for raw in raw_artifacts: + reject_snapshot(raw, exc) + reject_snapshot(artifact, exc) + raise + + +FSSP_HOSTS = frozenset({"opendata.fssp.gov.ru"}) +FSSP_VERSIONS = { + "7709576929-iplegallist": "20260820T2100", + "7709576929-iplegallistcomplete": "20220620T2006", +} + + +def _verify_download_url(dataset: str, key: str, url: str) -> None: + parsed = urlparse(url) + if ( + parsed.scheme != "https" + or parsed.hostname not in FSSP_HOSTS + or parsed.username + or parsed.password + or parsed.port not in (None, 443) + or parsed.query + or parsed.fragment + or parsed.path != f"/{dataset}/{key}.csv" + ): + raise SnapshotValidationError("unsafe_fssp_download_url") + + +def discover_fssp_input(session, dataset: str) -> tuple[str, date, str]: + """Discover a dated official artifact from its bounded machine-readable passport.""" + if dataset not in FSSP_DATASETS: + raise SnapshotValidationError("unknown_fssp_dataset") + with registry_response( + session, f"https://opendata.fssp.gov.ru/{dataset}/meta.csv", hosts=FSSP_HOSTS + ) as response: + body = limited_bytes(response, 1024**2).decode("utf-8-sig") + reader = csv.reader(StringIO(body), strict=True) + if next(reader, []) != ["property", "value"]: + raise SnapshotValidationError("invalid_fssp_passport") + entries = list(reader) + if any(len(row) != 2 for row in entries): + raise SnapshotValidationError("invalid_fssp_passport") + metadata = dict(entries) + if metadata.get("identifier") != dataset or metadata.get("format") != "csv": + raise SnapshotValidationError("invalid_fssp_passport") + candidates = [] + for key, url in entries: + match = re.fullmatch( + r"data-([0-9]{8}T[0-9]{4})-structure-([0-9]{8}T[0-9]{4})", key + ) + if match: + candidates.append((match.group(1), match.group(2), key, url)) + if not candidates: + raise SnapshotValidationError("missing_fssp_download") + timestamp, version, key, url = max(candidates) + _verify_download_url(dataset, key, url) + if version != FSSP_VERSIONS[dataset]: + raise SnapshotValidationError("unknown_fssp_structure_version") + try: + published = date(int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8])) + except ValueError as exc: + raise SnapshotValidationError("invalid_fssp_publication_date") from exc + return url, published, version + + +def refresh_fssp_enforcement_proceedings( + *, + load_batch: int, + uploaded_by_id: int | None = None, + on_publish=None, + on_progress=None, +) -> tuple[ParserSourceArtifact, SnapshotResult]: + downloaded = [] + try: + with ExitStack() as stack: + session = stack.enter_context(registry_session()) + inputs = [] + for dataset in FSSP_DATASETS: + url, published, version = discover_fssp_input(session, dataset) + handle = stack.enter_context(tempfile.TemporaryFile()) + download_registry_archive( + session, + url, + handle, + hosts=FSSP_HOSTS, + maximum=settings.FSSP_MAX_CSV_BYTES, + ) + raw = ParserSourceArtifact.objects.create( + source=FSSP_SOURCE, + original_name=urlparse(url).path.rsplit("/", 1)[-1], + content_type="text/csv", + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + source_published_at=published, + version=version, + metadata={"dataset": dataset}, + ) + downloaded.append(raw) + save_artifact_file(raw, handle) + inputs.append( + FsspInput(dataset, handle, raw.original_name, published, raw) + ) + return import_fssp_snapshots( + inputs=inputs, + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + on_publish=on_publish, + on_progress=on_progress, + ) + except Exception as exc: + for artifact in downloaded: + reject_snapshot(artifact, exc) + raise diff --git a/src/apps/parsers/migrations/0042_fssp_sources.py b/src/apps/parsers/migrations/0042_fssp_sources.py new file mode 100644 index 0000000..24822f3 --- /dev/null +++ b/src/apps/parsers/migrations/0042_fssp_sources.py @@ -0,0 +1,204 @@ +# Generated by Django 3.2.25 on 2026-10-05 13:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0041_fedresurs_disabled_schedule"), + ] + + operations = [ + migrations.AlterField( + model_name="genericparserrecord", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ("hh", "Вакансии HeadHunter"), + ("superjob", "Вакансии SuperJob"), + ], + db_index=True, + help_text="Источник данных", + max_length=50, + verbose_name="источник", + ), + ), + migrations.AlterField( + model_name="parserbatchsequence", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ], + help_text="Источник данных", + max_length=50, + unique=True, + verbose_name="источник", + ), + ), + migrations.AlterField( + model_name="parserloadlog", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ], + db_index=True, + help_text="Источник данных", + max_length=50, + verbose_name="источник", + ), + ), + migrations.AlterField( + model_name="parsersourceartifact", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ], + db_index=True, + max_length=50, + ), + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 41c0d95..65b67c8 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -61,6 +61,10 @@ class ParserLoadLog(TimestampMixin, models.Model): "fedresurs_company_profile", _("Карточки компаний Федресурса"), ) + FSSP_ENFORCEMENT_PROCEEDINGS = ( + "fssp_enforcement_proceedings", + _("Исполнительные производства ФССП"), + ) class Status(models.TextChoices): SUCCESS = "success", _("Успешно") diff --git a/src/organizations/migrations/0016_fssp_sources.py b/src/organizations/migrations/0016_fssp_sources.py new file mode 100644 index 0000000..05e320a --- /dev/null +++ b/src/organizations/migrations/0016_fssp_sources.py @@ -0,0 +1,70 @@ +# Generated by Django 3.2.25 on 2026-10-05 13:29 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("organizations", "0015_auto_20260924_1114"), + ] + + operations = [ + migrations.CreateModel( + name="EnforcementProceedingExtension", + fields=[ + ( + "organizationsourceextension_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="organizations.organizationsourceextension", + ), + ), + ], + options={ + "verbose_name": "исполнительные производства", + "verbose_name_plural": "исполнительные производства", + "db_table": "organizations_enforcement_proceeding_extension", + }, + bases=("organizations.organizationsourceextension",), + ), + migrations.AlterField( + model_name="organizationsourceextension", + name="source_group", + field=models.CharField( + choices=[ + ("financial_indicators", "Финансово-экономические показатели"), + ("government_procurements", "Государственные закупки"), + ("industrial_production", "Производители и продукция России"), + ("planned_inspections", "Плановые проверки"), + ("bankruptcy", "Сведения о процедурах банкротства"), + ("defense_suppliers", "Недобросовестные поставщики ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("security_registries", "Реестры по информационной безопасности"), + ("vacancies", "Вакансии"), + ("electronic_document_exchange", "Электронный документооборот"), + ("media_mentions", "Упоминания в СМИ"), + ("sanctions", "Санкции"), + ("government_support", "Государственная поддержка"), + ("sro_membership", "Членство в СРО"), + ( + "budget_process_registry", + "Реестр участников бюджетного процесса", + ), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ("enforcement_proceedings", "Исполнительные производства"), + ], + db_index=True, + max_length=64, + verbose_name="группа источников", + ), + ), + ] diff --git a/src/organizations/models.py b/src/organizations/models.py index ad26c1b..8761893 100644 --- a/src/organizations/models.py +++ b/src/organizations/models.py @@ -52,6 +52,10 @@ class SourceGroup(models.TextChoices): "fedresurs_company_profile", _("Карточки компаний Федресурса"), ) + ENFORCEMENT_PROCEEDINGS = ( + "enforcement_proceedings", + _("Исполнительные производства"), + ) class SourceExtensionStatus(models.TextChoices): @@ -772,6 +776,17 @@ class FedresursCompanyProfileExtension(OrganizationSourceExtension): verbose_name_plural = _("карточки компаний Федресурса") +class EnforcementProceedingExtension(OrganizationSourceExtension): + """FSSP proceedings for one canonical organization.""" + + source_group_value = SourceGroup.ENFORCEMENT_PROCEEDINGS + + class Meta: + db_table = "organizations_enforcement_proceeding_extension" + verbose_name = _("исполнительные производства") + verbose_name_plural = _("исполнительные производства") + + class OrganizationSourceRecord(models.Model): """Subordinate source record stored under a source extension.""" diff --git a/src/organizations/source_groups.py b/src/organizations/source_groups.py index f1f8131..8343a90 100644 --- a/src/organizations/source_groups.py +++ b/src/organizations/source_groups.py @@ -12,6 +12,7 @@ from organizations.models import ( BudgetProcessRegistryExtension, DefenseSupplierExtension, ElectronicDocumentExchangeExtension, + EnforcementProceedingExtension, FedresursCompanyProfileExtension, FinancialIndicatorsExtension, GovernmentProcurementExtension, @@ -41,6 +42,13 @@ class SourceGroupDescriptor: SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = { + ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS: SourceGroupDescriptor( + source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS, + source_group=SourceGroup.ENFORCEMENT_PROCEEDINGS, + record_type="enforcement_proceeding", + title="Исполнительные производства ФССП", + extension_model=EnforcementProceedingExtension, + ), ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE: SourceGroupDescriptor( source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, source_group=SourceGroup.FEDRESURS_COMPANY_PROFILE, diff --git a/src/settings/base.py b/src/settings/base.py index 51b1bf6..1bf1cd3 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -515,6 +515,9 @@ FNS_FAILED_DIRECTORY = PROJECT_ROOT / "input" / "fns" / "failed" CHECKO_API_KEY = os.getenv("CHECKO_API_KEY", "") +# Verified FSSP daily legal-entity CSVs currently occupy 1.97/2.72 GB. +FSSP_MAX_CSV_BYTES = 4 * 1024**3 + # Default closed; the explicit collection override is configured only on dev. SRO_DEV_COLLECTION_ENABLED = _env_bool("SRO_DEV_COLLECTION_ENABLED", default=False) SRO_UPSTREAM_ACCESS_APPROVED = ( diff --git a/tests/apps/parsers/test_fssp_snapshot.py b/tests/apps/parsers/test_fssp_snapshot.py new file mode 100644 index 0000000..a7cab29 --- /dev/null +++ b/tests/apps/parsers/test_fssp_snapshot.py @@ -0,0 +1,300 @@ +"""Atomic FSSP publication: real staging/database, synthetic input files.""" + +import csv +import importlib +from datetime import date +from io import BytesIO, StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import pytest +import requests +from apps.parsers.fssp_csv import ENDED_DATASET, FSSP_SOURCE, GENERAL_DATASET +from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord +from apps.parsers.registry_snapshots import SnapshotValidationError +from django.test import TestCase, override_settings +from django.utils import timezone +from organizations.models import Organization, OrganizationSourceRecord + +FIXTURES = Path(__file__).parents[2] / "fixtures" / "fssp" + + +def adapter(): + try: + return importlib.import_module("apps.parsers.fssp_snapshot") + except ModuleNotFoundError: + pytest.fail("FSSP snapshot importer is not implemented") + + +def csv_bytes(name, **changes): + rows = list(csv.reader(StringIO((FIXTURES / name).read_text()))) + for key, value in changes.items(): + rows[1][rows[0].index(key)] = value + handle = StringIO() + csv.writer(handle).writerows(rows) + return handle.getvalue().encode() + + +class FsspSnapshotTests(TestCase): + def setUp(self): + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.settings = override_settings( + MEDIA_ROOT=self.tmp.name + "/public", + PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private", + ) + self.settings.enable() + self.addCleanup(self.settings.disable) + self.org = Organization.objects.create( + name="АО Проверка", + inn="7707083893", + ogrn="1027700132195", + okpo="12345678", + directory_imported_at=timezone.now(), + ) + + def run_import(self, general=None, ended=None, batch=1, **kwargs): + module = adapter() + inputs = [ + module.FsspInput( + GENERAL_DATASET, + BytesIO(general or csv_bytes("general.csv")), + "general.csv", + date(2026, 10, 5), + ), + module.FsspInput( + ENDED_DATASET, + BytesIO(ended or csv_bytes("ended.csv")), + "ended.csv", + date(2026, 10, 5), + ), + ] + return module.import_fssp_snapshots(inputs=inputs, load_batch=batch, **kwargs) + + def published(self): + return list( + OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE).values_list( + "uid", "payload" + ) + ) + + def test_two_inputs_manifest_and_stable_uid(self): + artifact, result = self.run_import() + assert result.parsed == 2 + assert result.published == 1 + assert len(artifact.metadata["inputs"]) == 2 + row = OrganizationSourceRecord.objects.get(source=FSSP_SOURCE) + assert row.status == "ended" + assert row.amount is None + assert len(row.payload["observations"]) == 2 + assert row.extension.organization_id == self.org.uid + uid = row.uid + self.run_import(batch=2) + assert OrganizationSourceRecord.objects.get(source=FSSP_SOURCE).uid == uid + assert Organization.objects.count() == 1 + assert row.legacy_model == "" + assert row.legacy_pk == "" + with pytest.raises(ValueError): + _ = artifact.file.url + + def test_second_input_failure_preserves_snapshot(self): + self.run_import() + before = self.published() + with pytest.raises(SnapshotValidationError): + self.run_import(ended=b"invalid,csv\n", batch=2) + assert self.published() == before + assert ( + ParserSourceArtifact.objects.filter( + status=ParserSourceArtifact.Status.PUBLISHED + ).count() + == 1 + ) + + def test_conflicting_duplicate_blocks_publication(self): + self.run_import() + before = self.published() + good = csv_bytes("general.csv") + changed = csv_bytes("general.csv", **{"Debt remaining balance": "100"}) + rows = list(csv.reader(StringIO(changed.decode()))) + handle = StringIO() + csv.writer(handle).writerow(rows[1]) + with pytest.raises(SnapshotValidationError, match="conflicting_fssp_identity"): + self.run_import(general=good + handle.getvalue().encode(), batch=2) + assert self.published() == before + + def test_foreign_inn_skipped_and_invalid_inn_quarantined(self): + general = csv_bytes("general.csv", **{"Debtor TIN": "7704228075"}) + ended = csv_bytes("ended.csv", **{"Debtor TIN": "7707083894"}) + artifact, result = self.run_import(general=general, ended=ended) + assert result.skipped == 1 + assert result.quarantined == 1 + assert result.published == 0 + assert Organization.objects.count() == 1 + assert artifact.metadata["reasons"]["invalid_debtor_inn"] == 1 + assert artifact.staged_records.count() == 1 + + def test_ambiguous_identity_quarantined(self): + Organization.objects.create( + name="Другой филиал", + inn=self.org.inn, + kpp="111111111", + ogrn="1027700132195", + okpo="12345679", + directory_imported_at=timezone.now(), + ) + artifact, result = self.run_import() + assert result.published == 0 + assert result.quarantined == 2 + assert artifact.metadata["reasons"]["ambiguous_organization"] == 2 + + def test_missing_one_input_rejected(self): + module = adapter() + with pytest.raises(SnapshotValidationError, match="missing_fssp_input"): + module.import_fssp_snapshots(inputs=[], load_batch=1) + assert not OrganizationSourceRecord.objects.exists() + + def test_publication_callback_failure_rolls_back_records(self): + self.run_import() + before = self.published() + + def fail(artifact, result): + raise RuntimeError("fixture publication failure") + + with pytest.raises(RuntimeError): + self.run_import(batch=2, on_publish=fail) + assert self.published() == before + assert ( + ParserStagedRecord.objects.filter( + disposition="published", artifact__load_batch=2 + ).count() + == 0 + ) + + def test_valid_empty_inputs_clear_previous_snapshot(self): + self.run_import() + general = csv_bytes("general.csv").split(b"\r\n", 1)[0] + b"\r\n" + ended = csv_bytes("ended.csv").split(b"\r\n", 1)[0] + b"\r\n" + artifact, result = self.run_import(general=general, ended=ended, batch=2) + assert result.published == 0 + assert not self.published() + assert artifact.status == "published" + + def test_progress_includes_rows_outside_directory(self): + rows = list( + csv.reader( + StringIO( + csv_bytes("general.csv", **{"Debtor TIN": "7704228075"}).decode() + ) + ) + ) + handle = StringIO() + csv.writer(handle).writerows([rows[0]] + [rows[1]] * 10000) + progress = [] + artifact, result = self.run_import( + general=handle.getvalue().encode(), + on_progress=lambda *counts: progress.append(counts), + ) + assert progress and progress[-1][0] == 10000 + assert result.skipped == 10000 + + def test_more_than_one_staging_batch_publishes_all(self): + rows = list(csv.reader(StringIO(csv_bytes("general.csv").decode()))) + handle = StringIO() + writer = csv.writer(handle) + writer.writerow(rows[0]) + number_column = rows[0].index("The number of enforcement proceeding") + for number in range(501): + row = list(rows[1]) + row[number_column] = f"{number}/26/77001-ИП" + writer.writerow(row) + artifact, result = self.run_import(general=handle.getvalue().encode()) + assert result.parsed == 502 + assert result.published == 501 + assert artifact.metadata["deduplicated_records_count"] == 1 + assert artifact.staged_records.filter(disposition="published").count() == 501 + + def test_refresh_downloaded_artifacts_link_to_aggregate(self): + module = adapter() + with patch.object(module, "registry_session", return_value=DownloadSession()): + artifact, result = module.refresh_fssp_enforcement_proceedings(load_batch=1) + assert result.published == 1 + for item in artifact.metadata["inputs"]: + raw = ParserSourceArtifact.objects.get(pk=item["artifact_id"]) + assert raw.metadata["aggregate_artifact_id"] == str(artifact.uid) + assert raw.sha256 == item["checksum_sha256"] + + def test_failed_second_download_retains_first_raw_and_old_snapshot(self): + self.run_import() + before = self.published() + module = adapter() + with patch.object( + module, "registry_session", return_value=DownloadSession(fail_ended=True) + ), pytest.raises(requests.HTTPError): + module.refresh_fssp_enforcement_proceedings(load_batch=2) + assert self.published() == before + raw = ParserSourceArtifact.objects.get(load_batch=2) + assert raw.status == "rejected" + assert raw.file.size > 0 + + +class PassportSession: + def __init__(self, body): + self.body = body + + def get(self, url, **kwargs): + response = requests.Response() + response.status_code = 200 + response._content = self.body + response._content_consumed = True + response.headers["Content-Type"] = "text/csv" + return response + + +class DownloadSession: + def __init__(self, fail_ended=False): + self.fail_ended = fail_ended + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def get(self, url, **kwargs): + ended = ENDED_DATASET in url + if url.endswith("/meta.csv"): + name = "ended-meta.csv" if ended else "general-meta.csv" + return PassportSession((FIXTURES / name).read_bytes()).get(url) + body = csv_bytes("ended.csv" if ended else "general.csv") + response = PassportSession(body).get(url) + response.status_code = 503 if ended and self.fail_ended else 206 + response.headers["Content-Range"] = f"bytes 0-{len(body) - 1}/{len(body)}" + response.headers["ETag"] = '"fixture"' + response.url = url + return response + + +def test_passport_discovery_selects_latest_timestamp_not_row_order(): + body = (FIXTURES / "general-meta.csv").read_text() + lines = body.splitlines() + session = PassportSession(("\n".join(reversed(lines[1:])) + "\n").encode()) + with pytest.raises(SnapshotValidationError): + adapter().discover_fssp_input(session, GENERAL_DATASET) + session = PassportSession( + ("property,value\n" + "\n".join(reversed(lines[1:]))).encode() + ) + url, published_at, version = adapter().discover_fssp_input(session, GENERAL_DATASET) + assert url.endswith("data-20261005T0655-structure-20260820T2100.csv") + assert published_at == date(2026, 10, 5) + assert version == "20260820T2100" + + +def test_passport_external_download_host_rejected(): + body = ( + (FIXTURES / "general-meta.csv") + .read_text() + .replace("https://opendata.fssp.gov.ru/", "https://foreign.example/") + ) + with pytest.raises(SnapshotValidationError): + adapter().discover_fssp_input(PassportSession(body.encode()), GENERAL_DATASET)