feat: expose FSSP source APIs and exports

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:00:54 +02:00
parent 1884ecc546
commit 8d2c16bc24
31 changed files with 2112 additions and 44 deletions

View File

@@ -84,7 +84,7 @@ def _termination(payload: dict, warnings: set[str]) -> None:
if not raw:
warnings.add("unparsed_termination")
return
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw)
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", raw)
if match:
payload["terminated_on"] = _date(match.group(), warnings)
for label, field in (
@@ -94,6 +94,9 @@ def _termination(payload: dict, warnings: set[str]) -> None:
):
match = re.search(rf"\b{label}\.\s*([0-9]+)", raw)
if match:
if field == "termination_point" and "п.п." in raw:
warnings.add("unparsed_termination")
continue
payload[field] = match.group(1)
if not payload["terminated_on"] or not payload["termination_article"]:
warnings.add("unparsed_termination")
@@ -207,7 +210,7 @@ def iter_fssp_records(
number = payload["proceeding_number"] or ""
yield SourceRecordInput(
external_id=f"{inn}:{number}",
title=number[:255],
title=f"Исполнительное производство {number}"[:255],
organization_name=payload["debtor_name"] or "",
record_type="enforcement_proceeding",
inn=inn,

View File

@@ -59,7 +59,13 @@ def _identity(row, index: OwnOrganizationIndex):
return None, "invalid_proceeding_number"
if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]:
return None, "missing_fssp_required_field"
return index.resolve(inn=row.inn, ogrn="")
organization, reason = index.resolve(inn=row.inn, ogrn="")
if not reason and (
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
):
return organization, "invalid_canonical_requisites"
return organization, reason
def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress):

View File

@@ -0,0 +1,34 @@
"""Daily FSSP collection remains disabled until complete dev acceptance."""
from django.db import migrations
def seed(apps, schema_editor):
intervals = apps.get_model("django_celery_beat", "IntervalSchedule")
tasks = apps.get_model("django_celery_beat", "PeriodicTask")
interval, _ = intervals.objects.get_or_create(every=1, period="days")
tasks.objects.update_or_create(
name="parser:fssp-enforcement-proceedings:daily",
defaults={
"task": "parsers.fssp_enforcement_proceedings.refresh",
"interval": interval,
"enabled": False,
"args": "[]",
"kwargs": "{}",
"description": "Enable only after a complete successful snapshot publication on dev.",
},
)
def unseed(apps, schema_editor):
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
name="parser:fssp-enforcement-proceedings:daily"
).delete()
class Migration(migrations.Migration):
dependencies = [
("parsers", "0042_fssp_sources"),
("django_celery_beat", "0018_improve_crontab_helptext"),
]
operations = [migrations.RunPython(seed, unseed)]

View File

@@ -1,7 +1,7 @@
"""Public snapshot provenance shared by cards, dashboard and parser history."""
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from django.db.models import Count
from django.db.models import Count, Q
def attach_snapshot_artifacts(logs) -> None:
@@ -10,10 +10,17 @@ def attach_snapshot_artifacts(logs) -> None:
if not logs:
return
artifacts = {}
for artifact in ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
).order_by("-created_at", "-uid"):
for artifact in (
ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
)
.exclude(
Q(source="fssp_enforcement_proceedings")
& Q(metadata__dataset__isnull=False)
)
.order_by("-created_at", "-uid")
):
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
legacy_artifacts = []
for artifact in artifacts.values():

View File

@@ -46,12 +46,14 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60
SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60
SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4
SNAPSHOT_CARD_SLUGS = {
"fssp_enforcement_proceedings": "fssp-enforcement-proceedings",
"budget_ubpandnubp": "budget-process-registry",
"fns_sme_support_recipients": "sme-support-recipients-registry",
"sro_membership_check": "sro-membership-check",
"fedresurs_company_profile": "fedresurs-company-profile",
}
SNAPSHOT_SOURCE_GROUPS = (
"enforcement_proceedings",
"budget_process_registry",
"government_support",
"sro_membership",
@@ -503,6 +505,23 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
),
refresh_interval=timedelta(days=1),
),
SourceCardDefinition(
slug="fssp-enforcement-proceedings",
title="ФССП: Банк данных исполнительных производств",
description="Открытые CSV ФССП для юридических лиц.",
order=1000,
task_names=("parsers.fssp_enforcement_proceedings.refresh",),
source_items=(
SourceItemDefinition(
code="fssp_inn_search",
title="Исполнительные производства ФССП",
description="Производства организаций канонического реестра.",
parser_source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
refresh_key="fssp_enforcement_proceedings",
),
),
refresh_interval=timedelta(days=1),
),
)
SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS}
@@ -1328,11 +1347,20 @@ class SourceCardService:
from apps.parsers.tasks_registry_snapshots import (
parse_budget_registry,
parse_fedresurs_company_profiles,
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
)
if definition.slug == "budget-process-registry":
if definition.slug == "fssp-enforcement-proceedings":
task_specs = (
(
parse_fssp_enforcement_proceedings,
"parsers.fssp_enforcement_proceedings.refresh",
"fssp_enforcement_proceedings",
),
)
elif definition.slug == "budget-process-registry":
task_specs = (
(
parse_budget_registry,

View File

@@ -56,6 +56,19 @@ class ParserSourceDescriptor:
PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
"fssp_enforcement_proceedings": ParserSourceDescriptor(
key="fssp_enforcement_proceedings",
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
title="ФССП: Банк данных исполнительных производств",
agency="ФССП России",
data_scope="Исполнительные производства организаций канонического реестра",
task_name="parsers.fssp_enforcement_proceedings.refresh",
mode="open_data_csv",
upstream_url="https://opendata.fssp.gov.ru/",
access_method="open_data",
parser_strategy="streaming_two_dataset_snapshot",
admin_only=True,
),
"fedresurs_company_profile": ParserSourceDescriptor(
key="fedresurs_company_profile",
source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,

View File

@@ -75,6 +75,7 @@ from apps.parsers.source_registry import PARSER_SOURCES
from apps.parsers.tasks_registry_snapshots import ( # noqa: F401
parse_budget_registry,
parse_fedresurs_company_profiles,
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
)

View File

@@ -62,7 +62,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"version": artifact.version or None,
"checksum_sha256": artifact.sha256 or None,
}
if artifact.source == "sro_membership_check":
if artifact.source in ("sro_membership_check", "fssp_enforcement_proceedings"):
payload.update(artifact.metadata)
job.meta = {
**job.meta,
@@ -79,7 +79,11 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
def _claim_snapshot_job(job, source, task_name):
with transaction.atomic():
if source in ("sro_membership_check", "fedresurs_company_profile"):
if source in (
"sro_membership_check",
"fedresurs_company_profile",
"fssp_enforcement_proceedings",
):
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
if (
@@ -107,6 +111,18 @@ def _update_snapshot_progress(job, queried, candidates, found, quarantined):
job.refresh_from_db()
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
if job.meta.get("source") == "fssp_enforcement_proceedings":
job.update_progress(5, f"Прочитано {queried} строк CSV")
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
meta={
**job.meta,
"raw_records_count": queried,
"matched_records_count": found,
"quarantined_records_count": quarantined,
},
updated_at=timezone.now(),
)
return
if queried % 10 == 0 or queried == candidates:
job.update_progress(
5 + (90 * queried // max(candidates, 1)),
@@ -290,3 +306,23 @@ def parse_fedresurs_company_profiles(
track_progress=True,
mode=mode,
)
@shared_task(
bind=True,
name="parsers.fssp_enforcement_proceedings.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_fssp_enforcement_proceedings(
self, *, requested_by_id: int | None = None
) -> dict:
from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings
return _run_snapshot(
self,
source="fssp_enforcement_proceedings",
refresh=refresh_fssp_enforcement_proceedings,
requested_by_id=requested_by_id,
track_progress=True,
)

View File

@@ -146,6 +146,7 @@ SOURCE_RESULT_TAGS = {
}
TASKS_BY_NAME = {
"parsers.fssp_enforcement_proceedings.refresh": tasks.parse_fssp_enforcement_proceedings,
"parsers.fedresurs_company_profile.refresh": tasks.parse_fedresurs_company_profiles,
"apps.parsers.tasks.parse_industrial_production": tasks.parse_industrial_production,
"apps.parsers.tasks.parse_industrial_products": tasks.parse_industrial_products,