feat: expose FSSP source APIs and exports
This commit is contained in:
@@ -84,7 +84,7 @@ def _termination(payload: dict, warnings: set[str]) -> None:
|
||||
if not raw:
|
||||
warnings.add("unparsed_termination")
|
||||
return
|
||||
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw)
|
||||
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", raw)
|
||||
if match:
|
||||
payload["terminated_on"] = _date(match.group(), warnings)
|
||||
for label, field in (
|
||||
@@ -94,6 +94,9 @@ def _termination(payload: dict, warnings: set[str]) -> None:
|
||||
):
|
||||
match = re.search(rf"\b{label}\.\s*([0-9]+)", raw)
|
||||
if match:
|
||||
if field == "termination_point" and "п.п." in raw:
|
||||
warnings.add("unparsed_termination")
|
||||
continue
|
||||
payload[field] = match.group(1)
|
||||
if not payload["terminated_on"] or not payload["termination_article"]:
|
||||
warnings.add("unparsed_termination")
|
||||
@@ -207,7 +210,7 @@ def iter_fssp_records(
|
||||
number = payload["proceeding_number"] or ""
|
||||
yield SourceRecordInput(
|
||||
external_id=f"{inn}:{number}",
|
||||
title=number[:255],
|
||||
title=f"Исполнительное производство {number}"[:255],
|
||||
organization_name=payload["debtor_name"] or "",
|
||||
record_type="enforcement_proceeding",
|
||||
inn=inn,
|
||||
|
||||
@@ -59,7 +59,13 @@ def _identity(row, index: OwnOrganizationIndex):
|
||||
return None, "invalid_proceeding_number"
|
||||
if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]:
|
||||
return None, "missing_fssp_required_field"
|
||||
return index.resolve(inn=row.inn, ogrn="")
|
||||
organization, reason = index.resolve(inn=row.inn, ogrn="")
|
||||
if not reason and (
|
||||
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
|
||||
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
|
||||
):
|
||||
return organization, "invalid_canonical_requisites"
|
||||
return organization, reason
|
||||
|
||||
|
||||
def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress):
|
||||
|
||||
34
src/apps/parsers/migrations/0043_fssp_disabled_schedule.py
Normal file
34
src/apps/parsers/migrations/0043_fssp_disabled_schedule.py
Normal file
@@ -0,0 +1,34 @@
|
||||
"""Daily FSSP collection remains disabled until complete dev acceptance."""
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
def seed(apps, schema_editor):
|
||||
intervals = apps.get_model("django_celery_beat", "IntervalSchedule")
|
||||
tasks = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
interval, _ = intervals.objects.get_or_create(every=1, period="days")
|
||||
tasks.objects.update_or_create(
|
||||
name="parser:fssp-enforcement-proceedings:daily",
|
||||
defaults={
|
||||
"task": "parsers.fssp_enforcement_proceedings.refresh",
|
||||
"interval": interval,
|
||||
"enabled": False,
|
||||
"args": "[]",
|
||||
"kwargs": "{}",
|
||||
"description": "Enable only after a complete successful snapshot publication on dev.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def unseed(apps, schema_editor):
|
||||
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
|
||||
name="parser:fssp-enforcement-proceedings:daily"
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("parsers", "0042_fssp_sources"),
|
||||
("django_celery_beat", "0018_improve_crontab_helptext"),
|
||||
]
|
||||
operations = [migrations.RunPython(seed, unseed)]
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Public snapshot provenance shared by cards, dashboard and parser history."""
|
||||
|
||||
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
||||
from django.db.models import Count
|
||||
from django.db.models import Count, Q
|
||||
|
||||
|
||||
def attach_snapshot_artifacts(logs) -> None:
|
||||
@@ -10,10 +10,17 @@ def attach_snapshot_artifacts(logs) -> None:
|
||||
if not logs:
|
||||
return
|
||||
artifacts = {}
|
||||
for artifact in ParserSourceArtifact.objects.filter(
|
||||
source__in={log.source for log in logs},
|
||||
load_batch__in={log.batch_id for log in logs},
|
||||
).order_by("-created_at", "-uid"):
|
||||
for artifact in (
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source__in={log.source for log in logs},
|
||||
load_batch__in={log.batch_id for log in logs},
|
||||
)
|
||||
.exclude(
|
||||
Q(source="fssp_enforcement_proceedings")
|
||||
& Q(metadata__dataset__isnull=False)
|
||||
)
|
||||
.order_by("-created_at", "-uid")
|
||||
):
|
||||
artifacts.setdefault((artifact.source, artifact.load_batch), artifact)
|
||||
legacy_artifacts = []
|
||||
for artifact in artifacts.values():
|
||||
|
||||
@@ -46,12 +46,14 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60
|
||||
SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60
|
||||
SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4
|
||||
SNAPSHOT_CARD_SLUGS = {
|
||||
"fssp_enforcement_proceedings": "fssp-enforcement-proceedings",
|
||||
"budget_ubpandnubp": "budget-process-registry",
|
||||
"fns_sme_support_recipients": "sme-support-recipients-registry",
|
||||
"sro_membership_check": "sro-membership-check",
|
||||
"fedresurs_company_profile": "fedresurs-company-profile",
|
||||
}
|
||||
SNAPSHOT_SOURCE_GROUPS = (
|
||||
"enforcement_proceedings",
|
||||
"budget_process_registry",
|
||||
"government_support",
|
||||
"sro_membership",
|
||||
@@ -503,6 +505,23 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
),
|
||||
refresh_interval=timedelta(days=1),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="fssp-enforcement-proceedings",
|
||||
title="ФССП: Банк данных исполнительных производств",
|
||||
description="Открытые CSV ФССП для юридических лиц.",
|
||||
order=1000,
|
||||
task_names=("parsers.fssp_enforcement_proceedings.refresh",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="fssp_inn_search",
|
||||
title="Исполнительные производства ФССП",
|
||||
description="Производства организаций канонического реестра.",
|
||||
parser_source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
|
||||
refresh_key="fssp_enforcement_proceedings",
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=1),
|
||||
),
|
||||
)
|
||||
|
||||
SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS}
|
||||
@@ -1328,11 +1347,20 @@ class SourceCardService:
|
||||
from apps.parsers.tasks_registry_snapshots import (
|
||||
parse_budget_registry,
|
||||
parse_fedresurs_company_profiles,
|
||||
parse_fssp_enforcement_proceedings,
|
||||
parse_sme_support_recipients,
|
||||
parse_sro_membership,
|
||||
)
|
||||
|
||||
if definition.slug == "budget-process-registry":
|
||||
if definition.slug == "fssp-enforcement-proceedings":
|
||||
task_specs = (
|
||||
(
|
||||
parse_fssp_enforcement_proceedings,
|
||||
"parsers.fssp_enforcement_proceedings.refresh",
|
||||
"fssp_enforcement_proceedings",
|
||||
),
|
||||
)
|
||||
elif definition.slug == "budget-process-registry":
|
||||
task_specs = (
|
||||
(
|
||||
parse_budget_registry,
|
||||
|
||||
@@ -56,6 +56,19 @@ class ParserSourceDescriptor:
|
||||
|
||||
|
||||
PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
"fssp_enforcement_proceedings": ParserSourceDescriptor(
|
||||
key="fssp_enforcement_proceedings",
|
||||
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
|
||||
title="ФССП: Банк данных исполнительных производств",
|
||||
agency="ФССП России",
|
||||
data_scope="Исполнительные производства организаций канонического реестра",
|
||||
task_name="parsers.fssp_enforcement_proceedings.refresh",
|
||||
mode="open_data_csv",
|
||||
upstream_url="https://opendata.fssp.gov.ru/",
|
||||
access_method="open_data",
|
||||
parser_strategy="streaming_two_dataset_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
"fedresurs_company_profile": ParserSourceDescriptor(
|
||||
key="fedresurs_company_profile",
|
||||
source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,
|
||||
|
||||
@@ -75,6 +75,7 @@ from apps.parsers.source_registry import PARSER_SOURCES
|
||||
from apps.parsers.tasks_registry_snapshots import ( # noqa: F401
|
||||
parse_budget_registry,
|
||||
parse_fedresurs_company_profiles,
|
||||
parse_fssp_enforcement_proceedings,
|
||||
parse_sme_support_recipients,
|
||||
parse_sro_membership,
|
||||
)
|
||||
|
||||
@@ -62,7 +62,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||||
"version": artifact.version or None,
|
||||
"checksum_sha256": artifact.sha256 or None,
|
||||
}
|
||||
if artifact.source == "sro_membership_check":
|
||||
if artifact.source in ("sro_membership_check", "fssp_enforcement_proceedings"):
|
||||
payload.update(artifact.metadata)
|
||||
job.meta = {
|
||||
**job.meta,
|
||||
@@ -79,7 +79,11 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||||
|
||||
def _claim_snapshot_job(job, source, task_name):
|
||||
with transaction.atomic():
|
||||
if source in ("sro_membership_check", "fedresurs_company_profile"):
|
||||
if source in (
|
||||
"sro_membership_check",
|
||||
"fedresurs_company_profile",
|
||||
"fssp_enforcement_proceedings",
|
||||
):
|
||||
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
|
||||
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
|
||||
if (
|
||||
@@ -107,6 +111,18 @@ def _update_snapshot_progress(job, queried, candidates, found, quarantined):
|
||||
job.refresh_from_db()
|
||||
if job.status != JobStatus.STARTED:
|
||||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||||
if job.meta.get("source") == "fssp_enforcement_proceedings":
|
||||
job.update_progress(5, f"Прочитано {queried} строк CSV")
|
||||
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
|
||||
meta={
|
||||
**job.meta,
|
||||
"raw_records_count": queried,
|
||||
"matched_records_count": found,
|
||||
"quarantined_records_count": quarantined,
|
||||
},
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
return
|
||||
if queried % 10 == 0 or queried == candidates:
|
||||
job.update_progress(
|
||||
5 + (90 * queried // max(candidates, 1)),
|
||||
@@ -290,3 +306,23 @@ def parse_fedresurs_company_profiles(
|
||||
track_progress=True,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
name="parsers.fssp_enforcement_proceedings.refresh",
|
||||
soft_time_limit=8 * 3600,
|
||||
time_limit=8 * 3600 + 300,
|
||||
)
|
||||
def parse_fssp_enforcement_proceedings(
|
||||
self, *, requested_by_id: int | None = None
|
||||
) -> dict:
|
||||
from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings
|
||||
|
||||
return _run_snapshot(
|
||||
self,
|
||||
source="fssp_enforcement_proceedings",
|
||||
refresh=refresh_fssp_enforcement_proceedings,
|
||||
requested_by_id=requested_by_id,
|
||||
track_progress=True,
|
||||
)
|
||||
|
||||
@@ -146,6 +146,7 @@ SOURCE_RESULT_TAGS = {
|
||||
}
|
||||
|
||||
TASKS_BY_NAME = {
|
||||
"parsers.fssp_enforcement_proceedings.refresh": tasks.parse_fssp_enforcement_proceedings,
|
||||
"parsers.fedresurs_company_profile.refresh": tasks.parse_fedresurs_company_profiles,
|
||||
"apps.parsers.tasks.parse_industrial_production": tasks.parse_industrial_production,
|
||||
"apps.parsers.tasks.parse_industrial_products": tasks.parse_industrial_products,
|
||||
|
||||
Reference in New Issue
Block a user