feat: publish complete FSSP snapshots atomically
This commit is contained in:
@@ -11,6 +11,7 @@ PRIVATE_ARTIFACT_PREFIXES = (
|
||||
"parser-artifacts/sro_membership_check/",
|
||||
"parser-artifacts/integrated_structure_documents/",
|
||||
"parser-artifacts/fedresurs_company_profile/",
|
||||
"parser-artifacts/fssp_enforcement_proceedings/",
|
||||
)
|
||||
|
||||
|
||||
|
||||
366
src/apps/parsers/fssp_snapshot.py
Normal file
366
src/apps/parsers/fssp_snapshot.py
Normal file
@@ -0,0 +1,366 @@
|
||||
"""Two-input FSSP staging and one atomic canonical publication."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from io import BytesIO, StringIO, TextIOWrapper
|
||||
from typing import BinaryIO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from apps.parsers.fssp_csv import (
|
||||
ENDED_DATASET,
|
||||
FSSP_DATASETS,
|
||||
FSSP_SOURCE,
|
||||
iter_fssp_records,
|
||||
valid_legal_inn,
|
||||
)
|
||||
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
||||
from apps.parsers.registry_http import (
|
||||
download_registry_archive,
|
||||
limited_bytes,
|
||||
registry_response,
|
||||
registry_session,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import (
|
||||
SNAPSHOT_CHUNK_SIZE,
|
||||
OwnOrganizationIndex,
|
||||
SnapshotResult,
|
||||
SnapshotValidationError,
|
||||
publish_snapshot,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
stage_records,
|
||||
)
|
||||
from django.conf import settings
|
||||
from django.db.models import Count
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FsspInput:
|
||||
dataset: str
|
||||
handle: BinaryIO
|
||||
original_name: str
|
||||
published_at: date
|
||||
artifact: ParserSourceArtifact | None = None
|
||||
|
||||
|
||||
def _identity(row, index: OwnOrganizationIndex):
|
||||
if not valid_legal_inn(row.inn):
|
||||
return None, "invalid_debtor_inn"
|
||||
if not row.payload["proceeding_number"] or len(row.external_id) > 255:
|
||||
return None, "invalid_proceeding_number"
|
||||
if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]:
|
||||
return None, "missing_fssp_required_field"
|
||||
return index.resolve(inn=row.inn, ogrn="")
|
||||
|
||||
|
||||
def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress):
|
||||
pending = []
|
||||
item.handle.seek(0)
|
||||
text = TextIOWrapper(item.handle, encoding="utf-8-sig", newline="")
|
||||
try:
|
||||
rows = iter_fssp_records(
|
||||
text,
|
||||
dataset=item.dataset,
|
||||
snapshot_date=item.published_at.isoformat(),
|
||||
fetched_at=raw_artifact.created_at.isoformat(),
|
||||
)
|
||||
for row in rows:
|
||||
counts["parsed"] += 1
|
||||
raw_artifact.parsed_count += 1
|
||||
organization, reason = _identity(row, index)
|
||||
if reason == "outside_directory":
|
||||
counts["skipped"] += 1
|
||||
elif reason:
|
||||
counts["quarantined"] += 1
|
||||
counts[reason] += 1
|
||||
if on_progress is not None and counts["parsed"] % 10000 == 0:
|
||||
on_progress(
|
||||
counts["parsed"],
|
||||
0,
|
||||
counts["parsed"] - counts["skipped"],
|
||||
counts["quarantined"],
|
||||
)
|
||||
if reason == "outside_directory":
|
||||
continue
|
||||
provenance = {
|
||||
"artifact_id": str(raw_artifact.uid),
|
||||
"dataset": item.dataset,
|
||||
"published_at": item.published_at.isoformat(),
|
||||
"row_number": row.payload["source_row_number"],
|
||||
"checksum_sha256": raw_artifact.sha256,
|
||||
}
|
||||
row.payload["provenance"] = provenance
|
||||
pending.append(
|
||||
ParserStagedRecord(
|
||||
artifact=aggregate,
|
||||
row_number=counts["parsed"],
|
||||
organization=organization,
|
||||
external_id=row.external_id,
|
||||
record_type=row.record_type,
|
||||
raw_data=row.payload["raw_columns"],
|
||||
normalized_data={
|
||||
"title": row.title,
|
||||
"record_date": row.record_date,
|
||||
"amount": format(row.amount, ".2f")
|
||||
if row.amount is not None
|
||||
else None,
|
||||
"status": row.status,
|
||||
"url": row.url,
|
||||
"payload": row.payload,
|
||||
},
|
||||
disposition="quarantined" if reason else "staged",
|
||||
reason=reason,
|
||||
)
|
||||
)
|
||||
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
|
||||
stage_records(pending)
|
||||
finally:
|
||||
text.detach()
|
||||
stage_records(pending)
|
||||
raw_artifact.status = ParserSourceArtifact.Status.PARSED
|
||||
raw_artifact.save()
|
||||
|
||||
|
||||
def _deduplicate(artifact):
|
||||
staged = artifact.staged_records.filter(disposition="staged")
|
||||
duplicates = (
|
||||
staged.order_by().values("external_id").annotate(n=Count("uid")).filter(n__gt=1)
|
||||
)
|
||||
for duplicate in duplicates.iterator(chunk_size=SNAPSHOT_CHUNK_SIZE):
|
||||
if duplicate["n"] > 20:
|
||||
raise SnapshotValidationError("excessive_fssp_duplicates")
|
||||
rows = list(
|
||||
staged.filter(external_id=duplicate["external_id"]).order_by("row_number")
|
||||
)
|
||||
by_dataset = {}
|
||||
for row in rows:
|
||||
dataset = row.normalized_data["payload"]["dataset"]
|
||||
previous = by_dataset.get(dataset)
|
||||
if previous is not None and previous.raw_data != row.raw_data:
|
||||
raise SnapshotValidationError("conflicting_fssp_identity")
|
||||
by_dataset[dataset] = row
|
||||
selected = rows[0]
|
||||
if len(by_dataset) > 1:
|
||||
selected = by_dataset[ENDED_DATASET]
|
||||
ended_date = selected.normalized_data["payload"]["provenance"][
|
||||
"published_at"
|
||||
]
|
||||
if any(
|
||||
row.normalized_data["payload"]["provenance"]["published_at"]
|
||||
> ended_date
|
||||
for row in rows
|
||||
):
|
||||
raise SnapshotValidationError("conflicting_fssp_identity")
|
||||
selected.normalized_data["payload"]["observations"] = [
|
||||
{
|
||||
"provenance": row.normalized_data["payload"]["provenance"],
|
||||
"raw_columns": row.raw_data,
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
selected.save(update_fields=("normalized_data",))
|
||||
staged.filter(external_id=duplicate["external_id"]).exclude(
|
||||
pk=selected.pk
|
||||
).delete()
|
||||
|
||||
|
||||
def import_fssp_snapshots(
|
||||
*,
|
||||
inputs: Iterable[FsspInput],
|
||||
load_batch: int,
|
||||
uploaded_by_id: int | None = None,
|
||||
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
|
||||
on_progress: Callable[[int, int, int, int], None] | None = None,
|
||||
) -> tuple[ParserSourceArtifact, SnapshotResult]:
|
||||
inputs = tuple(inputs)
|
||||
if len(inputs) != 2 or {item.dataset for item in inputs} != set(FSSP_DATASETS):
|
||||
raise SnapshotValidationError("missing_fssp_input")
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source=FSSP_SOURCE,
|
||||
original_name="fssp-manifest.json",
|
||||
content_type="application/json",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
source_published_at=max(item.published_at for item in inputs),
|
||||
version="general-20260820T2100+ended-20220620T2006",
|
||||
)
|
||||
counts: Counter = Counter()
|
||||
raw_artifacts = []
|
||||
try:
|
||||
index = OwnOrganizationIndex()
|
||||
for item in inputs:
|
||||
raw = item.artifact or ParserSourceArtifact.objects.create(
|
||||
source=FSSP_SOURCE,
|
||||
original_name=item.original_name,
|
||||
content_type="text/csv",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
source_published_at=item.published_at,
|
||||
metadata={
|
||||
"dataset": item.dataset,
|
||||
"aggregate_artifact_id": str(artifact.uid),
|
||||
},
|
||||
)
|
||||
raw_artifacts.append(raw)
|
||||
raw.metadata = {**raw.metadata, "aggregate_artifact_id": str(artifact.uid)}
|
||||
raw.save(update_fields=("metadata",))
|
||||
if item.artifact is None:
|
||||
save_artifact_file(raw, item.handle)
|
||||
_stage_input(item, raw, artifact, index, counts, on_progress)
|
||||
_deduplicate(artifact)
|
||||
artifact.parsed_count = counts["parsed"]
|
||||
artifact.quarantined_count = counts["quarantined"]
|
||||
artifact.metadata = {
|
||||
"inputs": [
|
||||
{
|
||||
"artifact_id": str(raw.uid),
|
||||
"dataset": raw.metadata["dataset"],
|
||||
"checksum_sha256": raw.sha256,
|
||||
"size_bytes": raw.size_bytes,
|
||||
"published_at": raw.source_published_at.isoformat(),
|
||||
"parsed_records_count": raw.parsed_count,
|
||||
}
|
||||
for raw in raw_artifacts
|
||||
],
|
||||
"skipped_records_count": counts["skipped"],
|
||||
"reasons": {
|
||||
key: value
|
||||
for key, value in counts.items()
|
||||
if key not in ("parsed", "skipped", "quarantined")
|
||||
},
|
||||
"deduplicated_records_count": counts["parsed"]
|
||||
- counts["skipped"]
|
||||
- counts["quarantined"]
|
||||
- artifact.staged_records.filter(disposition="staged").count(),
|
||||
}
|
||||
artifact.rejection_reasons = artifact.metadata["reasons"]
|
||||
manifest = BytesIO(
|
||||
json.dumps(artifact.metadata, ensure_ascii=False, sort_keys=True).encode()
|
||||
)
|
||||
save_artifact_file(artifact, manifest)
|
||||
artifact.status = ParserSourceArtifact.Status.PARSED
|
||||
artifact.save()
|
||||
return artifact, publish_snapshot(artifact, on_publish=on_publish)
|
||||
except Exception as exc:
|
||||
for raw in raw_artifacts:
|
||||
reject_snapshot(raw, exc)
|
||||
reject_snapshot(artifact, exc)
|
||||
raise
|
||||
|
||||
|
||||
FSSP_HOSTS = frozenset({"opendata.fssp.gov.ru"})
|
||||
FSSP_VERSIONS = {
|
||||
"7709576929-iplegallist": "20260820T2100",
|
||||
"7709576929-iplegallistcomplete": "20220620T2006",
|
||||
}
|
||||
|
||||
|
||||
def _verify_download_url(dataset: str, key: str, url: str) -> None:
|
||||
parsed = urlparse(url)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname not in FSSP_HOSTS
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.port not in (None, 443)
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or parsed.path != f"/{dataset}/{key}.csv"
|
||||
):
|
||||
raise SnapshotValidationError("unsafe_fssp_download_url")
|
||||
|
||||
|
||||
def discover_fssp_input(session, dataset: str) -> tuple[str, date, str]:
|
||||
"""Discover a dated official artifact from its bounded machine-readable passport."""
|
||||
if dataset not in FSSP_DATASETS:
|
||||
raise SnapshotValidationError("unknown_fssp_dataset")
|
||||
with registry_response(
|
||||
session, f"https://opendata.fssp.gov.ru/{dataset}/meta.csv", hosts=FSSP_HOSTS
|
||||
) as response:
|
||||
body = limited_bytes(response, 1024**2).decode("utf-8-sig")
|
||||
reader = csv.reader(StringIO(body), strict=True)
|
||||
if next(reader, []) != ["property", "value"]:
|
||||
raise SnapshotValidationError("invalid_fssp_passport")
|
||||
entries = list(reader)
|
||||
if any(len(row) != 2 for row in entries):
|
||||
raise SnapshotValidationError("invalid_fssp_passport")
|
||||
metadata = dict(entries)
|
||||
if metadata.get("identifier") != dataset or metadata.get("format") != "csv":
|
||||
raise SnapshotValidationError("invalid_fssp_passport")
|
||||
candidates = []
|
||||
for key, url in entries:
|
||||
match = re.fullmatch(
|
||||
r"data-([0-9]{8}T[0-9]{4})-structure-([0-9]{8}T[0-9]{4})", key
|
||||
)
|
||||
if match:
|
||||
candidates.append((match.group(1), match.group(2), key, url))
|
||||
if not candidates:
|
||||
raise SnapshotValidationError("missing_fssp_download")
|
||||
timestamp, version, key, url = max(candidates)
|
||||
_verify_download_url(dataset, key, url)
|
||||
if version != FSSP_VERSIONS[dataset]:
|
||||
raise SnapshotValidationError("unknown_fssp_structure_version")
|
||||
try:
|
||||
published = date(int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]))
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_fssp_publication_date") from exc
|
||||
return url, published, version
|
||||
|
||||
|
||||
def refresh_fssp_enforcement_proceedings(
|
||||
*,
|
||||
load_batch: int,
|
||||
uploaded_by_id: int | None = None,
|
||||
on_publish=None,
|
||||
on_progress=None,
|
||||
) -> tuple[ParserSourceArtifact, SnapshotResult]:
|
||||
downloaded = []
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
session = stack.enter_context(registry_session())
|
||||
inputs = []
|
||||
for dataset in FSSP_DATASETS:
|
||||
url, published, version = discover_fssp_input(session, dataset)
|
||||
handle = stack.enter_context(tempfile.TemporaryFile())
|
||||
download_registry_archive(
|
||||
session,
|
||||
url,
|
||||
handle,
|
||||
hosts=FSSP_HOSTS,
|
||||
maximum=settings.FSSP_MAX_CSV_BYTES,
|
||||
)
|
||||
raw = ParserSourceArtifact.objects.create(
|
||||
source=FSSP_SOURCE,
|
||||
original_name=urlparse(url).path.rsplit("/", 1)[-1],
|
||||
content_type="text/csv",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
source_published_at=published,
|
||||
version=version,
|
||||
metadata={"dataset": dataset},
|
||||
)
|
||||
downloaded.append(raw)
|
||||
save_artifact_file(raw, handle)
|
||||
inputs.append(
|
||||
FsspInput(dataset, handle, raw.original_name, published, raw)
|
||||
)
|
||||
return import_fssp_snapshots(
|
||||
inputs=inputs,
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
on_publish=on_publish,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
except Exception as exc:
|
||||
for artifact in downloaded:
|
||||
reject_snapshot(artifact, exc)
|
||||
raise
|
||||
204
src/apps/parsers/migrations/0042_fssp_sources.py
Normal file
204
src/apps/parsers/migrations/0042_fssp_sources.py
Normal file
@@ -0,0 +1,204 @@
|
||||
# Generated by Django 3.2.25 on 2026-10-05 13:29
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("parsers", "0041_fedresurs_disabled_schedule"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="genericparserrecord",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
("hh", "Вакансии HeadHunter"),
|
||||
("superjob", "Вакансии SuperJob"),
|
||||
],
|
||||
db_index=True,
|
||||
help_text="Источник данных",
|
||||
max_length=50,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="parserbatchsequence",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
],
|
||||
help_text="Источник данных",
|
||||
max_length=50,
|
||||
unique=True,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="parserloadlog",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
],
|
||||
db_index=True,
|
||||
help_text="Источник данных",
|
||||
max_length=50,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="parsersourceartifact",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
],
|
||||
db_index=True,
|
||||
max_length=50,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -61,6 +61,10 @@ class ParserLoadLog(TimestampMixin, models.Model):
|
||||
"fedresurs_company_profile",
|
||||
_("Карточки компаний Федресурса"),
|
||||
)
|
||||
FSSP_ENFORCEMENT_PROCEEDINGS = (
|
||||
"fssp_enforcement_proceedings",
|
||||
_("Исполнительные производства ФССП"),
|
||||
)
|
||||
|
||||
class Status(models.TextChoices):
|
||||
SUCCESS = "success", _("Успешно")
|
||||
|
||||
70
src/organizations/migrations/0016_fssp_sources.py
Normal file
70
src/organizations/migrations/0016_fssp_sources.py
Normal file
@@ -0,0 +1,70 @@
|
||||
# Generated by Django 3.2.25 on 2026-10-05 13:29
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("organizations", "0015_auto_20260924_1114"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="EnforcementProceedingExtension",
|
||||
fields=[
|
||||
(
|
||||
"organizationsourceextension_ptr",
|
||||
models.OneToOneField(
|
||||
auto_created=True,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
parent_link=True,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
to="organizations.organizationsourceextension",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "исполнительные производства",
|
||||
"verbose_name_plural": "исполнительные производства",
|
||||
"db_table": "organizations_enforcement_proceeding_extension",
|
||||
},
|
||||
bases=("organizations.organizationsourceextension",),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="organizationsourceextension",
|
||||
name="source_group",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("financial_indicators", "Финансово-экономические показатели"),
|
||||
("government_procurements", "Государственные закупки"),
|
||||
("industrial_production", "Производители и продукция России"),
|
||||
("planned_inspections", "Плановые проверки"),
|
||||
("bankruptcy", "Сведения о процедурах банкротства"),
|
||||
("defense_suppliers", "Недобросовестные поставщики ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("security_registries", "Реестры по информационной безопасности"),
|
||||
("vacancies", "Вакансии"),
|
||||
("electronic_document_exchange", "Электронный документооборот"),
|
||||
("media_mentions", "Упоминания в СМИ"),
|
||||
("sanctions", "Санкции"),
|
||||
("government_support", "Государственная поддержка"),
|
||||
("sro_membership", "Членство в СРО"),
|
||||
(
|
||||
"budget_process_registry",
|
||||
"Реестр участников бюджетного процесса",
|
||||
),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
("enforcement_proceedings", "Исполнительные производства"),
|
||||
],
|
||||
db_index=True,
|
||||
max_length=64,
|
||||
verbose_name="группа источников",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -52,6 +52,10 @@ class SourceGroup(models.TextChoices):
|
||||
"fedresurs_company_profile",
|
||||
_("Карточки компаний Федресурса"),
|
||||
)
|
||||
ENFORCEMENT_PROCEEDINGS = (
|
||||
"enforcement_proceedings",
|
||||
_("Исполнительные производства"),
|
||||
)
|
||||
|
||||
|
||||
class SourceExtensionStatus(models.TextChoices):
|
||||
@@ -772,6 +776,17 @@ class FedresursCompanyProfileExtension(OrganizationSourceExtension):
|
||||
verbose_name_plural = _("карточки компаний Федресурса")
|
||||
|
||||
|
||||
class EnforcementProceedingExtension(OrganizationSourceExtension):
|
||||
"""FSSP proceedings for one canonical organization."""
|
||||
|
||||
source_group_value = SourceGroup.ENFORCEMENT_PROCEEDINGS
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_enforcement_proceeding_extension"
|
||||
verbose_name = _("исполнительные производства")
|
||||
verbose_name_plural = _("исполнительные производства")
|
||||
|
||||
|
||||
class OrganizationSourceRecord(models.Model):
|
||||
"""Subordinate source record stored under a source extension."""
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from organizations.models import (
|
||||
BudgetProcessRegistryExtension,
|
||||
DefenseSupplierExtension,
|
||||
ElectronicDocumentExchangeExtension,
|
||||
EnforcementProceedingExtension,
|
||||
FedresursCompanyProfileExtension,
|
||||
FinancialIndicatorsExtension,
|
||||
GovernmentProcurementExtension,
|
||||
@@ -41,6 +42,13 @@ class SourceGroupDescriptor:
|
||||
|
||||
|
||||
SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = {
|
||||
ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS: SourceGroupDescriptor(
|
||||
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
|
||||
source_group=SourceGroup.ENFORCEMENT_PROCEEDINGS,
|
||||
record_type="enforcement_proceeding",
|
||||
title="Исполнительные производства ФССП",
|
||||
extension_model=EnforcementProceedingExtension,
|
||||
),
|
||||
ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE: SourceGroupDescriptor(
|
||||
source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,
|
||||
source_group=SourceGroup.FEDRESURS_COMPANY_PROFILE,
|
||||
|
||||
@@ -515,6 +515,9 @@ FNS_FAILED_DIRECTORY = PROJECT_ROOT / "input" / "fns" / "failed"
|
||||
|
||||
CHECKO_API_KEY = os.getenv("CHECKO_API_KEY", "")
|
||||
|
||||
# Verified FSSP daily legal-entity CSVs currently occupy 1.97/2.72 GB.
|
||||
FSSP_MAX_CSV_BYTES = 4 * 1024**3
|
||||
|
||||
# Default closed; the explicit collection override is configured only on dev.
|
||||
SRO_DEV_COLLECTION_ENABLED = _env_bool("SRO_DEV_COLLECTION_ENABLED", default=False)
|
||||
SRO_UPSTREAM_ACCESS_APPROVED = (
|
||||
|
||||
Reference in New Issue
Block a user