"""Streaming normalization of the two verified FSSP legal-entity CSV schemas.""" from __future__ import annotations import csv import re from collections.abc import Iterator from datetime import date, datetime from decimal import Decimal, InvalidOperation from typing import TextIO from apps.parsers.registry_snapshots import SnapshotValidationError from django.utils import timezone from organizations.source_ingestion import SourceRecordInput FSSP_SOURCE = "fssp_enforcement_proceedings" GENERAL_DATASET = "7709576929-iplegallist" ENDED_DATASET = "7709576929-iplegallistcomplete" FSSP_DATASETS = (GENERAL_DATASET, ENDED_DATASET) FSSP_HEADERS = { "The name of the debtor": "debtor_name", "Address of the debtor organization": "debtor_legal_address", "Actual address of the debtor organization": "debtor_actual_address", "The number of enforcement proceeding": "proceeding_number", "Date of the institution of proceeding": "initiated_on", "The number of the consolidated enforcement proceedings": "consolidated_proceeding_number", "The executive document type": "document_type", "Date of the executive document": "document_date", "The number of executive document": "document_number", "The object of executive documents": "document_subject_raw", "The object of the execution": "subject", "Departments of bailiffs": "department_name", "Address of departments of bailiff": "department_address", "Debtor TIN": "debtor_inn", "TIN of the organization collector": "creditor_inn", } def valid_legal_inn(value: str) -> bool: """Validate a ten-digit legal-entity INN without dropping leading zeroes.""" if not re.fullmatch(r"[0-9]{10}", value): return False weights = (2, 4, 10, 3, 5, 9, 4, 6, 8) return sum( int(digit) * weight for digit, weight in zip(value, weights, strict=False) ) % 11 % 10 == int(value[-1]) def _date(value: str | None, warnings: set[str]) -> str | None: if not value: return None for pattern in ("%d.%m.%Y", "%Y-%m-%d"): try: return datetime.strptime(value, pattern).date().isoformat() except ValueError: continue warnings.add("unparsed_date") return None def _money(value: str | None, warnings: set[str], *, signed: bool) -> Decimal | None: if not value: return None normalized = re.sub(r"\s", "", value).replace(",", ".") pattern = r"-?[0-9]+(?:\.[0-9]{1,2})?" if signed else r"[0-9]+(?:\.[0-9]{1,2})?" try: if not re.fullmatch(pattern, normalized): raise InvalidOperation result = Decimal(normalized) if abs(result) >= Decimal("1000000000000000000"): raise InvalidOperation return result except InvalidOperation: warnings.add("unparsed_amount") return None def _join(*values: str | None) -> str | None: return "\n".join(value for value in values if value) or None def _termination(payload: dict, warnings: set[str]) -> None: raw = payload["termination_raw"] if not raw: warnings.add("unparsed_termination") return match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", raw) if match: payload["terminated_on"] = _date(match.group(), warnings) for label, field in ( ("ст", "termination_article"), ("ч", "termination_part"), ("п", "termination_point"), ): match = re.search(rf"\b{label}\.\s*([0-9]+)", raw) if match: if field == "termination_point" and "п.п." in raw: warnings.add("unparsed_termination") continue payload[field] = match.group(1) if not payload["terminated_on"] or not payload["termination_article"]: warnings.add("unparsed_termination") def _balances(raw: dict, warnings: set[str]) -> tuple[Decimal | None, Decimal | None]: debt = _money(raw.get("Debt remaining balance"), warnings, signed=True) claim = _money(raw.get("Amount due"), warnings, signed=False) if debt is not None and debt < 0: warnings.add("negative_debt_balance") if debt is not None and claim is not None and debt > claim: warnings.add("debt_exceeds_claim_amount") return debt, claim def _dates(payload: dict, comparison_date: date, warnings: set[str]) -> None: for field in ("initiated_on", "document_date"): payload[field] = _date(payload[field], warnings) if ( payload["document_date"] and date.fromisoformat(payload["document_date"]) > comparison_date ): warnings.add("future_document_date") def iter_fssp_records( handle: TextIO, *, dataset: str, snapshot_date: str | None = None, fetched_at: str | None = None, ) -> Iterator[SourceRecordInput]: """Read one logical row at a time; retain original columns for provenance.""" if dataset not in FSSP_DATASETS: raise SnapshotValidationError("unknown_fssp_dataset") fetched_at = fetched_at or timezone.now().isoformat() comparison_date = date.fromisoformat(snapshot_date or fetched_at[:10]) expected = set(FSSP_HEADERS) if dataset == GENERAL_DATASET: expected.update(("Amount due", "Debt remaining balance")) else: expected.add("Date, complete IP reason") reader = csv.reader(handle, strict=True) try: headers = next(reader, []) if headers: headers[0] = headers[0].lstrip("\ufeff") if len(headers) != len(expected) or set(headers) != expected: raise SnapshotValidationError("unknown_fssp_csv_headers") for row_number, values in enumerate(reader, 1): if len(values) != len(headers): raise SnapshotValidationError("invalid_fssp_csv_row_width") raw = dict(zip(headers, values, strict=True)) payload = { field: raw[key].strip() or None for key, field in FSSP_HEADERS.items() } warnings: set[str] = set() _dates(payload, comparison_date, warnings) debt, claim = _balances(raw, warnings) payload.update( { "origin_kind": "open_data_csv", "queried_inn": None, "debt_amount": format(debt, ".2f") if debt is not None else None, "claim_amount": format(claim, ".2f") if claim is not None else None, "debt_currency": "RUB" if debt is not None or claim is not None else None, "document_issuer": None, "termination_raw": raw.get("Date, complete IP reason", "").strip() or None, "terminated_on": None, "termination_reason": None, "termination_article": None, "termination_part": None, "termination_point": None, "service_text": None, "service_links": [], "bailiff_raw": None, "bailiff_name": None, "bailiff_phones": [], "source_url": f"https://opendata.fssp.gov.ru/{dataset}", "fetched_at": fetched_at, "raw_columns": raw, "dataset": dataset, "source_row_number": row_number, } ) payload["debtor_raw"] = _join( payload["debtor_name"], payload["debtor_inn"], payload["debtor_legal_address"], ) payload["proceeding_raw"] = _join( payload["proceeding_number"], payload["initiated_on"] ) payload["document_raw"] = _join( payload["document_type"], payload["document_date"], payload["document_number"], payload["document_subject_raw"], ) payload["subject_raw"] = _join(payload["subject"], payload["debt_amount"]) payload["department_raw"] = _join( payload["department_name"], payload["department_address"] ) if dataset == ENDED_DATASET: _termination(payload, warnings) payload["parse_warnings"] = sorted(warnings) inn = payload["debtor_inn"] or "" number = payload["proceeding_number"] or "" yield SourceRecordInput( external_id=f"{inn}:{number}", title=f"Исполнительное производство {number}"[:255], organization_name=payload["debtor_name"] or "", record_type="enforcement_proceeding", inn=inn, record_date=payload["initiated_on"] or "", amount=debt, status="ended" if dataset == ENDED_DATASET else "unknown", url=payload["source_url"], payload=payload, ) except (csv.Error, UnicodeError) as exc: raise SnapshotValidationError("invalid_fssp_csv") from exc