feat: expose FSSP source APIs and exports

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:00:54 +02:00
parent 1884ecc546
commit 8d2c16bc24
31 changed files with 2112 additions and 44 deletions

View File

@@ -84,7 +84,7 @@ def _termination(payload: dict, warnings: set[str]) -> None:
if not raw:
warnings.add("unparsed_termination")
return
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw)
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", raw)
if match:
payload["terminated_on"] = _date(match.group(), warnings)
for label, field in (
@@ -94,6 +94,9 @@ def _termination(payload: dict, warnings: set[str]) -> None:
):
match = re.search(rf"\b{label}\.\s*([0-9]+)", raw)
if match:
if field == "termination_point" and "п.п." in raw:
warnings.add("unparsed_termination")
continue
payload[field] = match.group(1)
if not payload["terminated_on"] or not payload["termination_article"]:
warnings.add("unparsed_termination")
@@ -207,7 +210,7 @@ def iter_fssp_records(
number = payload["proceeding_number"] or ""
yield SourceRecordInput(
external_id=f"{inn}:{number}",
title=number[:255],
title=f"Исполнительное производство {number}"[:255],
organization_name=payload["debtor_name"] or "",
record_type="enforcement_proceeding",
inn=inn,

View File

@@ -59,7 +59,13 @@ def _identity(row, index: OwnOrganizationIndex):
return None, "invalid_proceeding_number"
if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]:
return None, "missing_fssp_required_field"
return index.resolve(inn=row.inn, ogrn="")
organization, reason = index.resolve(inn=row.inn, ogrn="")
if not reason and (
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
):
return organization, "invalid_canonical_requisites"
return organization, reason
def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress):

View File

@@ -0,0 +1,34 @@
"""Daily FSSP collection remains disabled until complete dev acceptance."""
from django.db import migrations
def seed(apps, schema_editor):
intervals = apps.get_model("django_celery_beat", "IntervalSchedule")
tasks = apps.get_model("django_celery_beat", "PeriodicTask")
interval, _ = intervals.objects.get_or_create(every=1, period="days")
tasks.objects.update_or_create(
name="parser:fssp-enforcement-proceedings:daily",
defaults={
"task": "parsers.fssp_enforcement_proceedings.refresh",
"interval": interval,
"enabled": False,
"args": "[]",
"kwargs": "{}",
"description": "Enable only after a complete successful snapshot publication on dev.",
},
)
def unseed(apps, schema_editor):
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
name="parser:fssp-enforcement-proceedings:daily"
).delete()
class Migration(migrations.Migration):
dependencies = [
("parsers", "0042_fssp_sources"),
("django_celery_beat", "0018_improve_crontab_helptext"),
]
operations = [migrations.RunPython(seed, unseed)]

View File

@@ -1,7 +1,7 @@
"""Public snapshot provenance shared by cards, dashboard and parser history."""
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from django.db.models import Count
from django.db.models import Count, Q
def attach_snapshot_artifacts(logs) -> None:
@@ -10,10 +10,17 @@ def attach_snapshot_artifacts(logs) -> None:
if not logs:
return
artifacts = {}
for artifact in ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
).order_by("-created_at", "-uid"):
for artifact in (
ParserSourceArtifact.objects.filter(
source__in={log.source for log in logs},
load_batch__in={log.batch_id for log in logs},
)
.exclude(
Q(source="fssp_enforcement_proceedings")
& Q(metadata__dataset__isnull=False)
)
.order_by("-created_at", "-uid")
):
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
legacy_artifacts = []
for artifact in artifacts.values():

View File

@@ -46,12 +46,14 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60
SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60
SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4
SNAPSHOT_CARD_SLUGS = {
"fssp_enforcement_proceedings": "fssp-enforcement-proceedings",
"budget_ubpandnubp": "budget-process-registry",
"fns_sme_support_recipients": "sme-support-recipients-registry",
"sro_membership_check": "sro-membership-check",
"fedresurs_company_profile": "fedresurs-company-profile",
}
SNAPSHOT_SOURCE_GROUPS = (
"enforcement_proceedings",
"budget_process_registry",
"government_support",
"sro_membership",
@@ -503,6 +505,23 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
),
refresh_interval=timedelta(days=1),
),
SourceCardDefinition(
slug="fssp-enforcement-proceedings",
title="ФССП: Банк данных исполнительных производств",
description="Открытые CSV ФССП для юридических лиц.",
order=1000,
task_names=("parsers.fssp_enforcement_proceedings.refresh",),
source_items=(
SourceItemDefinition(
code="fssp_inn_search",
title="Исполнительные производства ФССП",
description="Производства организаций канонического реестра.",
parser_source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
refresh_key="fssp_enforcement_proceedings",
),
),
refresh_interval=timedelta(days=1),
),
)
SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS}
@@ -1328,11 +1347,20 @@ class SourceCardService:
from apps.parsers.tasks_registry_snapshots import (
parse_budget_registry,
parse_fedresurs_company_profiles,
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
)
if definition.slug == "budget-process-registry":
if definition.slug == "fssp-enforcement-proceedings":
task_specs = (
(
parse_fssp_enforcement_proceedings,
"parsers.fssp_enforcement_proceedings.refresh",
"fssp_enforcement_proceedings",
),
)
elif definition.slug == "budget-process-registry":
task_specs = (
(
parse_budget_registry,

View File

@@ -56,6 +56,19 @@ class ParserSourceDescriptor:
PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
"fssp_enforcement_proceedings": ParserSourceDescriptor(
key="fssp_enforcement_proceedings",
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
title="ФССП: Банк данных исполнительных производств",
agency="ФССП России",
data_scope="Исполнительные производства организаций канонического реестра",
task_name="parsers.fssp_enforcement_proceedings.refresh",
mode="open_data_csv",
upstream_url="https://opendata.fssp.gov.ru/",
access_method="open_data",
parser_strategy="streaming_two_dataset_snapshot",
admin_only=True,
),
"fedresurs_company_profile": ParserSourceDescriptor(
key="fedresurs_company_profile",
source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,

View File

@@ -75,6 +75,7 @@ from apps.parsers.source_registry import PARSER_SOURCES
from apps.parsers.tasks_registry_snapshots import ( # noqa: F401
parse_budget_registry,
parse_fedresurs_company_profiles,
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
)

View File

@@ -62,7 +62,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"version": artifact.version or None,
"checksum_sha256": artifact.sha256 or None,
}
if artifact.source == "sro_membership_check":
if artifact.source in ("sro_membership_check", "fssp_enforcement_proceedings"):
payload.update(artifact.metadata)
job.meta = {
**job.meta,
@@ -79,7 +79,11 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
def _claim_snapshot_job(job, source, task_name):
with transaction.atomic():
if source in ("sro_membership_check", "fedresurs_company_profile"):
if source in (
"sro_membership_check",
"fedresurs_company_profile",
"fssp_enforcement_proceedings",
):
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
if (
@@ -107,6 +111,18 @@ def _update_snapshot_progress(job, queried, candidates, found, quarantined):
job.refresh_from_db()
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
if job.meta.get("source") == "fssp_enforcement_proceedings":
job.update_progress(5, f"Прочитано {queried} строк CSV")
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
meta={
**job.meta,
"raw_records_count": queried,
"matched_records_count": found,
"quarantined_records_count": quarantined,
},
updated_at=timezone.now(),
)
return
if queried % 10 == 0 or queried == candidates:
job.update_progress(
5 + (90 * queried // max(candidates, 1)),
@@ -290,3 +306,23 @@ def parse_fedresurs_company_profiles(
track_progress=True,
mode=mode,
)
@shared_task(
bind=True,
name="parsers.fssp_enforcement_proceedings.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_fssp_enforcement_proceedings(
self, *, requested_by_id: int | None = None
) -> dict:
from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings
return _run_snapshot(
self,
source="fssp_enforcement_proceedings",
refresh=refresh_fssp_enforcement_proceedings,
requested_by_id=requested_by_id,
track_progress=True,
)

View File

@@ -146,6 +146,7 @@ SOURCE_RESULT_TAGS = {
}
TASKS_BY_NAME = {
"parsers.fssp_enforcement_proceedings.refresh": tasks.parse_fssp_enforcement_proceedings,
"parsers.fedresurs_company_profile.refresh": tasks.parse_fedresurs_company_profiles,
"apps.parsers.tasks.parse_industrial_production": tasks.parse_industrial_production,
"apps.parsers.tasks.parse_industrial_products": tasks.parse_industrial_products,

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,60 @@
"""Public fields of the verified FSSP CSV contract, shared by list and detail."""
from rest_framework import serializers
from organizations.registry_payload_serializers import RegistryUTCDateTimeField
class FsspServiceLinkSerializer(serializers.Serializer):
label = serializers.CharField()
url = serializers.CharField()
class FsspEnforcementProceedingPayloadSerializer(serializers.Serializer):
origin_kind = serializers.ChoiceField(
choices=("web_search", "open_data_csv"), allow_null=True
)
queried_inn = serializers.CharField(allow_null=True)
debtor_name = serializers.CharField(allow_null=True)
debtor_legal_address = serializers.CharField(allow_null=True)
debtor_actual_address = serializers.CharField(allow_null=True)
consolidated_proceeding_number = serializers.CharField(allow_null=True)
document_raw = serializers.CharField(allow_null=True)
document_type = serializers.CharField(allow_null=True)
document_number = serializers.CharField(allow_null=True)
document_issuer = serializers.CharField(allow_null=True)
document_subject_raw = serializers.CharField(allow_null=True)
creditor_inn = serializers.CharField(allow_null=True)
termination_raw = serializers.CharField(allow_null=True)
termination_reason = serializers.CharField(allow_null=True)
termination_article = serializers.CharField(allow_null=True)
termination_part = serializers.CharField(allow_null=True)
termination_point = serializers.CharField(allow_null=True)
service_text = serializers.CharField(allow_null=True)
subject_raw = serializers.CharField(allow_null=True)
subject = serializers.CharField(allow_null=True)
debt_amount = serializers.CharField(allow_null=True)
claim_amount = serializers.CharField(allow_null=True)
debt_currency = serializers.CharField(allow_null=True)
department_raw = serializers.CharField(allow_null=True)
department_name = serializers.CharField(allow_null=True)
department_address = serializers.CharField(allow_null=True)
bailiff_raw = serializers.CharField(allow_null=True)
bailiff_name = serializers.CharField(allow_null=True)
debtor_raw = serializers.CharField()
debtor_inn = serializers.CharField()
proceeding_raw = serializers.CharField()
proceeding_number = serializers.CharField()
source_url = serializers.CharField()
initiated_on = serializers.DateField(allow_null=True)
document_date = serializers.DateField(allow_null=True)
terminated_on = serializers.DateField(allow_null=True)
fetched_at = RegistryUTCDateTimeField()
service_links = FsspServiceLinkSerializer(many=True)
bailiff_phones = serializers.ListField(child=serializers.CharField())
parse_warnings = serializers.ListField(child=serializers.CharField())
raw_columns = serializers.DictField(child=serializers.CharField())
dataset = serializers.CharField()
source_row_number = serializers.IntegerField()
provenance = serializers.JSONField(required=False)
observations = serializers.JSONField(required=False)

View File

@@ -322,6 +322,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
)
def get_payload(self, obj) -> dict | list | str | int | float | bool | None: # noqa: C901
payload = obj.payload
if obj.source == "fssp_enforcement_proceedings":
from organizations.fssp_payload_serializers import (
FsspEnforcementProceedingPayloadSerializer,
)
return FsspEnforcementProceedingPayloadSerializer(payload).data
if obj.source == "fedresurs_company_profile":
from organizations.fedresurs_payload import detail_payload

View File

@@ -80,6 +80,7 @@ EXPORT_PROVIDER_NAME_PATTERN = re.compile(
)
SNAPSHOT_EXPORT_SOURCES = {
"enforcement_proceedings": "fssp_enforcement_proceedings",
"budget_process_registry": "budget_ubpandnubp",
"government_support": "fns_sme_support_recipients",
"sro_membership": "sro_membership_check",
@@ -89,6 +90,7 @@ SNAPSHOT_EXPORT_SOURCES = {
GENERATION_ID_PATTERN = re.compile(r"[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}")
SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = {
SourceGroup.ENFORCEMENT_PROCEEDINGS.value: "fssp-enforcement-proceedings",
SourceGroup.SRO_MEMBERSHIP.value: "sro-membership-check",
SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS.value: "integrated-structure-documents",
SourceGroup.FEDRESURS_COMPANY_PROFILE.value: "fedresurs-company-profile",

View File

@@ -9,6 +9,7 @@ from decimal import Decimal, InvalidOperation
from uuid import UUID
from apps.core.pagination import StandardPagination
from django.core.paginator import Page
from django.db.models import (
BooleanField,
Case,
@@ -23,8 +24,10 @@ from django.db.models import (
from django.db.models.fields.json import KeyTextTransform
from django.db.models.functions import Cast, Coalesce, NullIf
from drf_yasg import openapi
from rest_framework.exceptions import NotFound
REGISTRY_GROUPS = (
"enforcement_proceedings",
"budget_process_registry",
"government_support",
"sro_membership",
@@ -32,6 +35,7 @@ REGISTRY_GROUPS = (
"fedresurs_company_profile",
)
REGISTRY_SOURCE_GROUPS = {
"fssp_enforcement_proceedings": "enforcement_proceedings",
"budget_ubpandnubp": "budget_process_registry",
"fns_sme_support_recipients": "government_support",
"sro_membership_check": "sro_membership",
@@ -40,6 +44,9 @@ REGISTRY_SOURCE_GROUPS = {
}
REGISTRY_ORDERING_FIELDS = (
"payload__document_raw",
"payload__terminated_on",
"payload__service_text",
"organization__full_name",
"organization__inn",
"organization__ogrn",
@@ -136,10 +143,30 @@ FILTER_PARAMS = (
class SourceRecordPagination(StandardPagination):
page_size = 50
def paginate_queryset(self, queryset, request, view=None):
try:
return super().paginate_queryset(queryset, request, view)
except NotFound:
if not (
request.query_params.get("source_group") == "enforcement_proceedings"
or request.query_params.get("source") == "fssp_enforcement_proceedings"
):
raise
number = int(request.query_params.get(self.page_query_param, "1"))
paginator = self.django_paginator_class(
queryset, self.get_page_size(request)
)
if number <= paginator.num_pages:
raise
self.page = Page([], number, paginator)
self.request = request
return []
def get_paginated_response(self, data):
response = super().get_paginated_response(data)
if not self.page.paginator.count:
response.data["meta"]["pagination"]["total_pages"] = 0
response.data["meta"]["pagination"]["has_previous"] = False
return response
@@ -286,11 +313,18 @@ def _validate_dates(values, error):
def _validate_amounts(values, error):
signed = (
values.get("source_group") == "enforcement_proceedings"
or values.get("source") == "fssp_enforcement_proceedings"
)
pattern = (
r"-?[0-9]{1,18}(?:\.[0-9]{1,2})?" if signed else r"[0-9]{1,18}(?:\.[0-9]{1,2})?"
)
for name in ("amount_from", "amount_to"):
if name in values:
try:
raw = values[name]
if not re.fullmatch(r"[0-9]{1,18}(?:\.[0-9]{1,2})?", raw):
if not re.fullmatch(pattern, raw):
raise InvalidOperation
values[name] = Decimal(raw)
except InvalidOperation:
@@ -370,6 +404,22 @@ def source_specific_search(term):
"""Search selected business fields, never arbitrary raw payload or URLs."""
query = Q()
for group, fields in (
(
"enforcement_proceedings",
tuple(
f"payload__{field}"
for field in (
"debtor_raw",
"proceeding_raw",
"document_raw",
"termination_raw",
"service_text",
"subject_raw",
"department_raw",
"bailiff_raw",
)
),
),
(
"budget_process_registry",
(

View File

@@ -44,6 +44,8 @@ CANONICAL_ONLY_TEST_SOURCES = {
ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK,
}
FILE_UPLOAD_ONLY_TEST_SOURCES = {
# Real FSSP snapshots must not be fabricated by the demo command.
ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
ParserLoadLog.Source.ROPK_SANCTIONS,
ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS,
ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,