feat: publish complete FSSP snapshots atomically

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 15:35:10 +02:00
parent b3b4e7691b
commit 1884ecc546
9 changed files with 971 additions and 0 deletions

View File

@@ -11,6 +11,7 @@ PRIVATE_ARTIFACT_PREFIXES = (
"parser-artifacts/sro_membership_check/",
"parser-artifacts/integrated_structure_documents/",
"parser-artifacts/fedresurs_company_profile/",
"parser-artifacts/fssp_enforcement_proceedings/",
)

View File

@@ -0,0 +1,366 @@
"""Two-input FSSP staging and one atomic canonical publication."""
from __future__ import annotations
import csv
import json
import re
import tempfile
from collections import Counter
from collections.abc import Callable, Iterable
from contextlib import ExitStack
from dataclasses import dataclass
from datetime import date
from io import BytesIO, StringIO, TextIOWrapper
from typing import BinaryIO
from urllib.parse import urlparse
from apps.parsers.fssp_csv import (
ENDED_DATASET,
FSSP_DATASETS,
FSSP_SOURCE,
iter_fssp_records,
valid_legal_inn,
)
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from apps.parsers.registry_http import (
download_registry_archive,
limited_bytes,
registry_response,
registry_session,
)
from apps.parsers.registry_snapshots import (
SNAPSHOT_CHUNK_SIZE,
OwnOrganizationIndex,
SnapshotResult,
SnapshotValidationError,
publish_snapshot,
reject_snapshot,
save_artifact_file,
stage_records,
)
from django.conf import settings
from django.db.models import Count
@dataclass(frozen=True)
class FsspInput:
dataset: str
handle: BinaryIO
original_name: str
published_at: date
artifact: ParserSourceArtifact | None = None
def _identity(row, index: OwnOrganizationIndex):
if not valid_legal_inn(row.inn):
return None, "invalid_debtor_inn"
if not row.payload["proceeding_number"] or len(row.external_id) > 255:
return None, "invalid_proceeding_number"
if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]:
return None, "missing_fssp_required_field"
return index.resolve(inn=row.inn, ogrn="")
def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress):
pending = []
item.handle.seek(0)
text = TextIOWrapper(item.handle, encoding="utf-8-sig", newline="")
try:
rows = iter_fssp_records(
text,
dataset=item.dataset,
snapshot_date=item.published_at.isoformat(),
fetched_at=raw_artifact.created_at.isoformat(),
)
for row in rows:
counts["parsed"] += 1
raw_artifact.parsed_count += 1
organization, reason = _identity(row, index)
if reason == "outside_directory":
counts["skipped"] += 1
elif reason:
counts["quarantined"] += 1
counts[reason] += 1
if on_progress is not None and counts["parsed"] % 10000 == 0:
on_progress(
counts["parsed"],
0,
counts["parsed"] - counts["skipped"],
counts["quarantined"],
)
if reason == "outside_directory":
continue
provenance = {
"artifact_id": str(raw_artifact.uid),
"dataset": item.dataset,
"published_at": item.published_at.isoformat(),
"row_number": row.payload["source_row_number"],
"checksum_sha256": raw_artifact.sha256,
}
row.payload["provenance"] = provenance
pending.append(
ParserStagedRecord(
artifact=aggregate,
row_number=counts["parsed"],
organization=organization,
external_id=row.external_id,
record_type=row.record_type,
raw_data=row.payload["raw_columns"],
normalized_data={
"title": row.title,
"record_date": row.record_date,
"amount": format(row.amount, ".2f")
if row.amount is not None
else None,
"status": row.status,
"url": row.url,
"payload": row.payload,
},
disposition="quarantined" if reason else "staged",
reason=reason,
)
)
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
stage_records(pending)
finally:
text.detach()
stage_records(pending)
raw_artifact.status = ParserSourceArtifact.Status.PARSED
raw_artifact.save()
def _deduplicate(artifact):
staged = artifact.staged_records.filter(disposition="staged")
duplicates = (
staged.order_by().values("external_id").annotate(n=Count("uid")).filter(n__gt=1)
)
for duplicate in duplicates.iterator(chunk_size=SNAPSHOT_CHUNK_SIZE):
if duplicate["n"] > 20:
raise SnapshotValidationError("excessive_fssp_duplicates")
rows = list(
staged.filter(external_id=duplicate["external_id"]).order_by("row_number")
)
by_dataset = {}
for row in rows:
dataset = row.normalized_data["payload"]["dataset"]
previous = by_dataset.get(dataset)
if previous is not None and previous.raw_data != row.raw_data:
raise SnapshotValidationError("conflicting_fssp_identity")
by_dataset[dataset] = row
selected = rows[0]
if len(by_dataset) > 1:
selected = by_dataset[ENDED_DATASET]
ended_date = selected.normalized_data["payload"]["provenance"][
"published_at"
]
if any(
row.normalized_data["payload"]["provenance"]["published_at"]
> ended_date
for row in rows
):
raise SnapshotValidationError("conflicting_fssp_identity")
selected.normalized_data["payload"]["observations"] = [
{
"provenance": row.normalized_data["payload"]["provenance"],
"raw_columns": row.raw_data,
}
for row in rows
]
selected.save(update_fields=("normalized_data",))
staged.filter(external_id=duplicate["external_id"]).exclude(
pk=selected.pk
).delete()
def import_fssp_snapshots(
*,
inputs: Iterable[FsspInput],
load_batch: int,
uploaded_by_id: int | None = None,
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
on_progress: Callable[[int, int, int, int], None] | None = None,
) -> tuple[ParserSourceArtifact, SnapshotResult]:
inputs = tuple(inputs)
if len(inputs) != 2 or {item.dataset for item in inputs} != set(FSSP_DATASETS):
raise SnapshotValidationError("missing_fssp_input")
artifact = ParserSourceArtifact.objects.create(
source=FSSP_SOURCE,
original_name="fssp-manifest.json",
content_type="application/json",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
source_published_at=max(item.published_at for item in inputs),
version="general-20260820T2100+ended-20220620T2006",
)
counts: Counter = Counter()
raw_artifacts = []
try:
index = OwnOrganizationIndex()
for item in inputs:
raw = item.artifact or ParserSourceArtifact.objects.create(
source=FSSP_SOURCE,
original_name=item.original_name,
content_type="text/csv",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
source_published_at=item.published_at,
metadata={
"dataset": item.dataset,
"aggregate_artifact_id": str(artifact.uid),
},
)
raw_artifacts.append(raw)
raw.metadata = {**raw.metadata, "aggregate_artifact_id": str(artifact.uid)}
raw.save(update_fields=("metadata",))
if item.artifact is None:
save_artifact_file(raw, item.handle)
_stage_input(item, raw, artifact, index, counts, on_progress)
_deduplicate(artifact)
artifact.parsed_count = counts["parsed"]
artifact.quarantined_count = counts["quarantined"]
artifact.metadata = {
"inputs": [
{
"artifact_id": str(raw.uid),
"dataset": raw.metadata["dataset"],
"checksum_sha256": raw.sha256,
"size_bytes": raw.size_bytes,
"published_at": raw.source_published_at.isoformat(),
"parsed_records_count": raw.parsed_count,
}
for raw in raw_artifacts
],
"skipped_records_count": counts["skipped"],
"reasons": {
key: value
for key, value in counts.items()
if key not in ("parsed", "skipped", "quarantined")
},
"deduplicated_records_count": counts["parsed"]
- counts["skipped"]
- counts["quarantined"]
- artifact.staged_records.filter(disposition="staged").count(),
}
artifact.rejection_reasons = artifact.metadata["reasons"]
manifest = BytesIO(
json.dumps(artifact.metadata, ensure_ascii=False, sort_keys=True).encode()
)
save_artifact_file(artifact, manifest)
artifact.status = ParserSourceArtifact.Status.PARSED
artifact.save()
return artifact, publish_snapshot(artifact, on_publish=on_publish)
except Exception as exc:
for raw in raw_artifacts:
reject_snapshot(raw, exc)
reject_snapshot(artifact, exc)
raise
FSSP_HOSTS = frozenset({"opendata.fssp.gov.ru"})
FSSP_VERSIONS = {
"7709576929-iplegallist": "20260820T2100",
"7709576929-iplegallistcomplete": "20220620T2006",
}
def _verify_download_url(dataset: str, key: str, url: str) -> None:
parsed = urlparse(url)
if (
parsed.scheme != "https"
or parsed.hostname not in FSSP_HOSTS
or parsed.username
or parsed.password
or parsed.port not in (None, 443)
or parsed.query
or parsed.fragment
or parsed.path != f"/{dataset}/{key}.csv"
):
raise SnapshotValidationError("unsafe_fssp_download_url")
def discover_fssp_input(session, dataset: str) -> tuple[str, date, str]:
"""Discover a dated official artifact from its bounded machine-readable passport."""
if dataset not in FSSP_DATASETS:
raise SnapshotValidationError("unknown_fssp_dataset")
with registry_response(
session, f"https://opendata.fssp.gov.ru/{dataset}/meta.csv", hosts=FSSP_HOSTS
) as response:
body = limited_bytes(response, 1024**2).decode("utf-8-sig")
reader = csv.reader(StringIO(body), strict=True)
if next(reader, []) != ["property", "value"]:
raise SnapshotValidationError("invalid_fssp_passport")
entries = list(reader)
if any(len(row) != 2 for row in entries):
raise SnapshotValidationError("invalid_fssp_passport")
metadata = dict(entries)
if metadata.get("identifier") != dataset or metadata.get("format") != "csv":
raise SnapshotValidationError("invalid_fssp_passport")
candidates = []
for key, url in entries:
match = re.fullmatch(
r"data-([0-9]{8}T[0-9]{4})-structure-([0-9]{8}T[0-9]{4})", key
)
if match:
candidates.append((match.group(1), match.group(2), key, url))
if not candidates:
raise SnapshotValidationError("missing_fssp_download")
timestamp, version, key, url = max(candidates)
_verify_download_url(dataset, key, url)
if version != FSSP_VERSIONS[dataset]:
raise SnapshotValidationError("unknown_fssp_structure_version")
try:
published = date(int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]))
except ValueError as exc:
raise SnapshotValidationError("invalid_fssp_publication_date") from exc
return url, published, version
def refresh_fssp_enforcement_proceedings(
*,
load_batch: int,
uploaded_by_id: int | None = None,
on_publish=None,
on_progress=None,
) -> tuple[ParserSourceArtifact, SnapshotResult]:
downloaded = []
try:
with ExitStack() as stack:
session = stack.enter_context(registry_session())
inputs = []
for dataset in FSSP_DATASETS:
url, published, version = discover_fssp_input(session, dataset)
handle = stack.enter_context(tempfile.TemporaryFile())
download_registry_archive(
session,
url,
handle,
hosts=FSSP_HOSTS,
maximum=settings.FSSP_MAX_CSV_BYTES,
)
raw = ParserSourceArtifact.objects.create(
source=FSSP_SOURCE,
original_name=urlparse(url).path.rsplit("/", 1)[-1],
content_type="text/csv",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
source_published_at=published,
version=version,
metadata={"dataset": dataset},
)
downloaded.append(raw)
save_artifact_file(raw, handle)
inputs.append(
FsspInput(dataset, handle, raw.original_name, published, raw)
)
return import_fssp_snapshots(
inputs=inputs,
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
on_publish=on_publish,
on_progress=on_progress,
)
except Exception as exc:
for artifact in downloaded:
reject_snapshot(artifact, exc)
raise

View File

@@ -0,0 +1,204 @@
# Generated by Django 3.2.25 on 2026-10-05 13:29
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("parsers", "0041_fedresurs_disabled_schedule"),
]
operations = [
migrations.AlterField(
model_name="genericparserrecord",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("hh", "Вакансии HeadHunter"),
("superjob", "Вакансии SuperJob"),
],
db_index=True,
help_text="Источник данных",
max_length=50,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parserbatchsequence",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
],
help_text="Источник данных",
max_length=50,
unique=True,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parserloadlog",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
],
db_index=True,
help_text="Источник данных",
max_length=50,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parsersourceartifact",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
],
db_index=True,
max_length=50,
),
),
]

View File

@@ -61,6 +61,10 @@ class ParserLoadLog(TimestampMixin, models.Model):
"fedresurs_company_profile",
_("Карточки компаний Федресурса"),
)
FSSP_ENFORCEMENT_PROCEEDINGS = (
"fssp_enforcement_proceedings",
_("Исполнительные производства ФССП"),
)
class Status(models.TextChoices):
SUCCESS = "success", _("Успешно")

View File

@@ -0,0 +1,70 @@
# Generated by Django 3.2.25 on 2026-10-05 13:29
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("organizations", "0015_auto_20260924_1114"),
]
operations = [
migrations.CreateModel(
name="EnforcementProceedingExtension",
fields=[
(
"organizationsourceextension_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="organizations.organizationsourceextension",
),
),
],
options={
"verbose_name": "исполнительные производства",
"verbose_name_plural": "исполнительные производства",
"db_table": "organizations_enforcement_proceeding_extension",
},
bases=("organizations.organizationsourceextension",),
),
migrations.AlterField(
model_name="organizationsourceextension",
name="source_group",
field=models.CharField(
choices=[
("financial_indicators", "Финансово-экономические показатели"),
("government_procurements", "Государственные закупки"),
("industrial_production", "Производители и продукция России"),
("planned_inspections", "Плановые проверки"),
("bankruptcy", "Сведения о процедурах банкротства"),
("defense_suppliers", "Недобросовестные поставщики ГОЗ"),
("arbitration", "Арбитражные дела"),
("security_registries", "Реестры по информационной безопасности"),
("vacancies", "Вакансии"),
("electronic_document_exchange", "Электронный документооборот"),
("media_mentions", "Упоминания в СМИ"),
("sanctions", "Санкции"),
("government_support", "Государственная поддержка"),
("sro_membership", "Членство в СРО"),
(
"budget_process_registry",
"Реестр участников бюджетного процесса",
),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
("enforcement_proceedings", "Исполнительные производства"),
],
db_index=True,
max_length=64,
verbose_name="группа источников",
),
),
]

View File

@@ -52,6 +52,10 @@ class SourceGroup(models.TextChoices):
"fedresurs_company_profile",
_("Карточки компаний Федресурса"),
)
ENFORCEMENT_PROCEEDINGS = (
"enforcement_proceedings",
_("Исполнительные производства"),
)
class SourceExtensionStatus(models.TextChoices):
@@ -772,6 +776,17 @@ class FedresursCompanyProfileExtension(OrganizationSourceExtension):
verbose_name_plural = _("карточки компаний Федресурса")
class EnforcementProceedingExtension(OrganizationSourceExtension):
"""FSSP proceedings for one canonical organization."""
source_group_value = SourceGroup.ENFORCEMENT_PROCEEDINGS
class Meta:
db_table = "organizations_enforcement_proceeding_extension"
verbose_name = _("исполнительные производства")
verbose_name_plural = _("исполнительные производства")
class OrganizationSourceRecord(models.Model):
"""Subordinate source record stored under a source extension."""

View File

@@ -12,6 +12,7 @@ from organizations.models import (
BudgetProcessRegistryExtension,
DefenseSupplierExtension,
ElectronicDocumentExchangeExtension,
EnforcementProceedingExtension,
FedresursCompanyProfileExtension,
FinancialIndicatorsExtension,
GovernmentProcurementExtension,
@@ -41,6 +42,13 @@ class SourceGroupDescriptor:
SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = {
ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS: SourceGroupDescriptor(
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
source_group=SourceGroup.ENFORCEMENT_PROCEEDINGS,
record_type="enforcement_proceeding",
title="Исполнительные производства ФССП",
extension_model=EnforcementProceedingExtension,
),
ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE: SourceGroupDescriptor(
source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,
source_group=SourceGroup.FEDRESURS_COMPANY_PROFILE,

View File

@@ -515,6 +515,9 @@ FNS_FAILED_DIRECTORY = PROJECT_ROOT / "input" / "fns" / "failed"
CHECKO_API_KEY = os.getenv("CHECKO_API_KEY", "")
# Verified FSSP daily legal-entity CSVs currently occupy 1.97/2.72 GB.
FSSP_MAX_CSV_BYTES = 4 * 1024**3
# Default closed; the explicit collection override is configured only on dev.
SRO_DEV_COLLECTION_ENABLED = _env_bool("SRO_DEV_COLLECTION_ENABLED", default=False)
SRO_UPSTREAM_ACCESS_APPROVED = (