Files
mostovik-backend/src/apps/parsers/fssp_csv.py
2026-10-05 15:22:14 +02:00

222 lines
8.7 KiB
Python

"""Streaming normalization of the two verified FSSP legal-entity CSV schemas."""
from __future__ import annotations
import csv
import re
from collections.abc import Iterator
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from typing import TextIO
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.utils import timezone
from organizations.source_ingestion import SourceRecordInput
FSSP_SOURCE = "fssp_enforcement_proceedings"
GENERAL_DATASET = "7709576929-iplegallist"
ENDED_DATASET = "7709576929-iplegallistcomplete"
FSSP_DATASETS = (GENERAL_DATASET, ENDED_DATASET)
FSSP_HEADERS = {
"The name of the debtor": "debtor_name",
"Address of the debtor organization": "debtor_legal_address",
"Actual address of the debtor organization": "debtor_actual_address",
"The number of enforcement proceeding": "proceeding_number",
"Date of the institution of proceeding": "initiated_on",
"The number of the consolidated enforcement proceedings": "consolidated_proceeding_number",
"The executive document type": "document_type",
"Date of the executive document": "document_date",
"The number of executive document": "document_number",
"The object of executive documents": "document_subject_raw",
"The object of the execution": "subject",
"Departments of bailiffs": "department_name",
"Address of departments of bailiff": "department_address",
"Debtor TIN": "debtor_inn",
"TIN of the organization collector": "creditor_inn",
}
def valid_legal_inn(value: str) -> bool:
"""Validate a ten-digit legal-entity INN without dropping leading zeroes."""
if not re.fullmatch(r"[0-9]{10}", value):
return False
weights = (2, 4, 10, 3, 5, 9, 4, 6, 8)
return sum(
int(digit) * weight for digit, weight in zip(value, weights, strict=False)
) % 11 % 10 == int(value[-1])
def _date(value: str | None, warnings: set[str]) -> str | None:
if not value:
return None
for pattern in ("%d.%m.%Y", "%Y-%m-%d"):
try:
return datetime.strptime(value, pattern).date().isoformat()
except ValueError:
continue
warnings.add("unparsed_date")
return None
def _money(value: str | None, warnings: set[str], *, signed: bool) -> Decimal | None:
if not value:
return None
normalized = re.sub(r"\s", "", value).replace(",", ".")
pattern = r"-?[0-9]+(?:\.[0-9]{1,2})?" if signed else r"[0-9]+(?:\.[0-9]{1,2})?"
try:
if not re.fullmatch(pattern, normalized):
raise InvalidOperation
result = Decimal(normalized)
if abs(result) >= Decimal("1000000000000000000"):
raise InvalidOperation
return result
except InvalidOperation:
warnings.add("unparsed_amount")
return None
def _join(*values: str | None) -> str | None:
return "\n".join(value for value in values if value) or None
def _termination(payload: dict, warnings: set[str]) -> None:
raw = payload["termination_raw"]
if not raw:
warnings.add("unparsed_termination")
return
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw)
if match:
payload["terminated_on"] = _date(match.group(), warnings)
for label, field in (
("ст", "termination_article"),
("ч", "termination_part"),
("п", "termination_point"),
):
match = re.search(rf"\b{label}\.\s*([0-9]+)", raw)
if match:
payload[field] = match.group(1)
if not payload["terminated_on"] or not payload["termination_article"]:
warnings.add("unparsed_termination")
def _balances(raw: dict, warnings: set[str]) -> tuple[Decimal | None, Decimal | None]:
debt = _money(raw.get("Debt remaining balance"), warnings, signed=True)
claim = _money(raw.get("Amount due"), warnings, signed=False)
if debt is not None and debt < 0:
warnings.add("negative_debt_balance")
if debt is not None and claim is not None and debt > claim:
warnings.add("debt_exceeds_claim_amount")
return debt, claim
def _dates(payload: dict, comparison_date: date, warnings: set[str]) -> None:
for field in ("initiated_on", "document_date"):
payload[field] = _date(payload[field], warnings)
if (
payload["document_date"]
and date.fromisoformat(payload["document_date"]) > comparison_date
):
warnings.add("future_document_date")
def iter_fssp_records(
handle: TextIO,
*,
dataset: str,
snapshot_date: str | None = None,
fetched_at: str | None = None,
) -> Iterator[SourceRecordInput]:
"""Read one logical row at a time; retain original columns for provenance."""
if dataset not in FSSP_DATASETS:
raise SnapshotValidationError("unknown_fssp_dataset")
fetched_at = fetched_at or timezone.now().isoformat()
comparison_date = date.fromisoformat(snapshot_date or fetched_at[:10])
expected = set(FSSP_HEADERS)
if dataset == GENERAL_DATASET:
expected.update(("Amount due", "Debt remaining balance"))
else:
expected.add("Date, complete IP reason")
reader = csv.reader(handle, strict=True)
try:
headers = next(reader, [])
if headers:
headers[0] = headers[0].lstrip("\ufeff")
if len(headers) != len(expected) or set(headers) != expected:
raise SnapshotValidationError("unknown_fssp_csv_headers")
for row_number, values in enumerate(reader, 1):
if len(values) != len(headers):
raise SnapshotValidationError("invalid_fssp_csv_row_width")
raw = dict(zip(headers, values, strict=True))
payload = {
field: raw[key].strip() or None for key, field in FSSP_HEADERS.items()
}
warnings: set[str] = set()
_dates(payload, comparison_date, warnings)
debt, claim = _balances(raw, warnings)
payload.update(
{
"origin_kind": "open_data_csv",
"queried_inn": None,
"debt_amount": format(debt, ".2f") if debt is not None else None,
"claim_amount": format(claim, ".2f") if claim is not None else None,
"debt_currency": "RUB"
if debt is not None or claim is not None
else None,
"document_issuer": None,
"termination_raw": raw.get("Date, complete IP reason", "").strip()
or None,
"terminated_on": None,
"termination_reason": None,
"termination_article": None,
"termination_part": None,
"termination_point": None,
"service_text": None,
"service_links": [],
"bailiff_raw": None,
"bailiff_name": None,
"bailiff_phones": [],
"source_url": f"https://opendata.fssp.gov.ru/{dataset}",
"fetched_at": fetched_at,
"raw_columns": raw,
"dataset": dataset,
"source_row_number": row_number,
}
)
payload["debtor_raw"] = _join(
payload["debtor_name"],
payload["debtor_inn"],
payload["debtor_legal_address"],
)
payload["proceeding_raw"] = _join(
payload["proceeding_number"], payload["initiated_on"]
)
payload["document_raw"] = _join(
payload["document_type"],
payload["document_date"],
payload["document_number"],
payload["document_subject_raw"],
)
payload["subject_raw"] = _join(payload["subject"], payload["debt_amount"])
payload["department_raw"] = _join(
payload["department_name"], payload["department_address"]
)
if dataset == ENDED_DATASET:
_termination(payload, warnings)
payload["parse_warnings"] = sorted(warnings)
inn = payload["debtor_inn"] or ""
number = payload["proceeding_number"] or ""
yield SourceRecordInput(
external_id=f"{inn}:{number}",
title=number[:255],
organization_name=payload["debtor_name"] or "",
record_type="enforcement_proceeding",
inn=inn,
record_date=payload["initiated_on"] or "",
amount=debt,
status="ended" if dataset == ENDED_DATASET else "unknown",
url=payload["source_url"],
payload=payload,
)
except (csv.Error, UnicodeError) as exc:
raise SnapshotValidationError("invalid_fssp_csv") from exc