From 92290511c2cb6e016b7668d70113d4decc01ae49 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 15:20:47 +0200 Subject: [PATCH] feat: normalize streaming FSSP CSV records --- src/apps/parsers/fssp_csv.py | 212 ++++++++++++++++++++++++++++ tests/apps/parsers/test_fssp_csv.py | 134 ++++++++++++++++++ 2 files changed, 346 insertions(+) create mode 100644 src/apps/parsers/fssp_csv.py create mode 100644 tests/apps/parsers/test_fssp_csv.py diff --git a/src/apps/parsers/fssp_csv.py b/src/apps/parsers/fssp_csv.py new file mode 100644 index 0000000..0cd363a --- /dev/null +++ b/src/apps/parsers/fssp_csv.py @@ -0,0 +1,212 @@ +"""Streaming normalization of the two verified FSSP legal-entity CSV schemas.""" + +from __future__ import annotations + +import csv +import re +from collections.abc import Iterator +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from typing import TextIO + +from apps.parsers.registry_snapshots import SnapshotValidationError +from django.utils import timezone +from organizations.source_ingestion import SourceRecordInput + +FSSP_SOURCE = "fssp_enforcement_proceedings" +GENERAL_DATASET = "7709576929-iplegallist" +ENDED_DATASET = "7709576929-iplegallistcomplete" +FSSP_DATASETS = (GENERAL_DATASET, ENDED_DATASET) +FSSP_HEADERS = { + "The name of the debtor": "debtor_name", + "Address of the debtor organization": "debtor_legal_address", + "Actual address of the debtor organization": "debtor_actual_address", + "The number of enforcement proceeding": "proceeding_number", + "Date of the institution of proceeding": "initiated_on", + "The number of the consolidated enforcement proceedings": "consolidated_proceeding_number", + "The executive document type": "document_type", + "Date of the executive document": "document_date", + "The number of executive document": "document_number", + "The object of executive documents": "document_subject_raw", + "The object of the execution": "subject", + "Departments of bailiffs": "department_name", + "Address of departments of bailiff": "department_address", + "Debtor TIN": "debtor_inn", + "TIN of the organization collector": "creditor_inn", +} + + +def valid_legal_inn(value: str) -> bool: + """Validate a ten-digit legal-entity INN without dropping leading zeroes.""" + if not re.fullmatch(r"[0-9]{10}", value): + return False + weights = (2, 4, 10, 3, 5, 9, 4, 6, 8) + return sum( + int(digit) * weight for digit, weight in zip(value, weights) + ) % 11 % 10 == int(value[-1]) + + +def _date(value: str | None, warnings: set[str]) -> str | None: + if not value: + return None + for pattern in ("%d.%m.%Y", "%Y-%m-%d"): + try: + return datetime.strptime(value, pattern).date().isoformat() + except ValueError: + continue + warnings.add("unparsed_date") + return None + + +def _money(value: str | None, warnings: set[str], *, signed: bool) -> Decimal | None: + if not value: + return None + normalized = re.sub(r"\s", "", value).replace(",", ".") + pattern = r"-?[0-9]+(?:\.[0-9]{1,2})?" if signed else r"[0-9]+(?:\.[0-9]{1,2})?" + try: + if not re.fullmatch(pattern, normalized): + raise InvalidOperation + result = Decimal(normalized) + if abs(result) >= Decimal("1000000000000000000"): + raise InvalidOperation + return result + except InvalidOperation: + warnings.add("unparsed_amount") + return None + + +def _join(*values: str | None) -> str | None: + return "\n".join(value for value in values if value) or None + + +def _termination(payload: dict, warnings: set[str]) -> None: + raw = payload["termination_raw"] + if not raw: + warnings.add("unparsed_termination") + return + match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw) + if match: + payload["terminated_on"] = _date(match.group(), warnings) + for label, field in ( + ("ст", "termination_article"), + ("ч", "termination_part"), + ("п", "termination_point"), + ): + match = re.search(rf"\b{label}\.\s*([0-9]+)", raw) + if match: + payload[field] = match.group(1) + if not payload["terminated_on"] or not payload["termination_article"]: + warnings.add("unparsed_termination") + + +def iter_fssp_records( + handle: TextIO, + *, + dataset: str, + snapshot_date: str | None = None, + fetched_at: str | None = None, +) -> Iterator[SourceRecordInput]: + """Read one logical row at a time; retain original columns for provenance.""" + if dataset not in FSSP_DATASETS: + raise SnapshotValidationError("unknown_fssp_dataset") + fetched_at = fetched_at or timezone.now().isoformat() + comparison_date = date.fromisoformat(snapshot_date or fetched_at[:10]) + expected = set(FSSP_HEADERS) + if dataset == GENERAL_DATASET: + expected.update(("Amount due", "Debt remaining balance")) + else: + expected.add("Date, complete IP reason") + reader = csv.reader(handle, strict=True) + try: + headers = next(reader, []) + if headers: + headers[0] = headers[0].lstrip("\ufeff") + if len(headers) != len(expected) or set(headers) != expected: + raise SnapshotValidationError("unknown_fssp_csv_headers") + for row_number, values in enumerate(reader, 1): + if len(values) != len(headers): + raise SnapshotValidationError("invalid_fssp_csv_row_width") + raw = dict(zip(headers, values)) + payload = { + field: raw[key].strip() or None for key, field in FSSP_HEADERS.items() + } + warnings: set[str] = set() + for field in ("initiated_on", "document_date"): + payload[field] = _date(payload[field], warnings) + if ( + payload["document_date"] + and date.fromisoformat(payload["document_date"]) > comparison_date + ): + warnings.add("future_document_date") + debt = _money(raw.get("Debt remaining balance"), warnings, signed=True) + claim = _money(raw.get("Amount due"), warnings, signed=False) + if debt is not None and debt < 0: + warnings.add("negative_debt_balance") + if debt is not None and claim is not None and debt > claim: + warnings.add("debt_exceeds_claim_amount") + payload.update( + { + "origin_kind": "open_data_csv", + "queried_inn": None, + "debt_amount": format(debt, ".2f") if debt is not None else None, + "claim_amount": format(claim, ".2f") if claim is not None else None, + "debt_currency": "RUB" + if debt is not None or claim is not None + else None, + "document_issuer": None, + "termination_raw": raw.get("Date, complete IP reason", "").strip() + or None, + "terminated_on": None, + "termination_reason": None, + "termination_article": None, + "termination_part": None, + "termination_point": None, + "service_text": None, + "service_links": [], + "bailiff_raw": None, + "bailiff_name": None, + "bailiff_phones": [], + "source_url": f"https://opendata.fssp.gov.ru/{dataset}", + "fetched_at": fetched_at, + "raw_columns": raw, + "dataset": dataset, + "source_row_number": row_number, + } + ) + payload["debtor_raw"] = _join( + payload["debtor_name"], + payload["debtor_inn"], + payload["debtor_legal_address"], + ) + payload["proceeding_raw"] = _join( + payload["proceeding_number"], payload["initiated_on"] + ) + payload["document_raw"] = _join( + payload["document_type"], + payload["document_date"], + payload["document_number"], + payload["document_subject_raw"], + ) + payload["subject_raw"] = _join(payload["subject"], payload["debt_amount"]) + payload["department_raw"] = _join( + payload["department_name"], payload["department_address"] + ) + if dataset == ENDED_DATASET: + _termination(payload, warnings) + payload["parse_warnings"] = sorted(warnings) + inn = payload["debtor_inn"] or "" + number = payload["proceeding_number"] or "" + yield SourceRecordInput( + external_id=f"{inn}:{number}", + title=number[:255], + organization_name=payload["debtor_name"] or "", + record_type="enforcement_proceeding", + inn=inn, + record_date=payload["initiated_on"] or "", + amount=debt, + status="ended" if dataset == ENDED_DATASET else "unknown", + url=payload["source_url"], + payload=payload, + ) + except (csv.Error, UnicodeError) as exc: + raise SnapshotValidationError("invalid_fssp_csv") from exc diff --git a/tests/apps/parsers/test_fssp_csv.py b/tests/apps/parsers/test_fssp_csv.py new file mode 100644 index 0000000..ef0e8d0 --- /dev/null +++ b/tests/apps/parsers/test_fssp_csv.py @@ -0,0 +1,134 @@ +"""FSSP CSV normalization without network or business data.""" + +import csv +import importlib +from decimal import Decimal +from io import StringIO +from pathlib import Path + +import pytest +from apps.parsers.registry_snapshots import SnapshotValidationError + +FIXTURES = Path(__file__).parents[2] / "fixtures" / "fssp" +GENERAL = "7709576929-iplegallist" +ENDED = "7709576929-iplegallistcomplete" + + +def records(text, dataset=GENERAL): + try: + module = importlib.import_module("apps.parsers.fssp_csv") + except ModuleNotFoundError: + pytest.fail("iter_fssp_records is not implemented") + return module.iter_fssp_records( + StringIO(text), + dataset=dataset, + snapshot_date="2026-10-05", + fetched_at="2026-10-05T08:00:00+00:00", + ) + + +def fixture_text(name="general.csv", **changes): + rows = list(csv.reader(StringIO((FIXTURES / name).read_text()))) + for key, value in changes.items(): + rows[1][rows[0].index(key)] = value + handle = StringIO() + csv.writer(handle, lineterminator="\n").writerows(rows) + return handle.getvalue() + + +def test_signed_debt_and_multiline_preserved(): + row = next(records(fixture_text())) + assert row.amount == Decimal("-12.50") + assert row.payload["debt_amount"] == "-12.50" + assert row.payload["claim_amount"] == "250.00" + assert row.payload["debtor_actual_address"] == "Москва, офис 2\nэтаж 3" + assert row.payload["origin_kind"] == "open_data_csv" + assert row.payload["queried_inn"] is None + assert row.payload["document_issuer"] is None + assert row.payload["document_subject_raw"] == "Взыскание сбора" + assert len(row.payload["raw_columns"]) == 17 + assert row.record_date == "2026-10-01" + + +@pytest.mark.parametrize( + "value,want", [("0", "0.00"), ("", None), ("abc", None), ("1 234,50", "1234.50")] +) +def test_money_preserves_zero_and_missing(value, want): + row = next(records(fixture_text(**{"Debt remaining balance": value}))) + assert row.payload["debt_amount"] == want + assert row.amount == (Decimal(want) if want is not None else None) + assert ("unparsed_amount" in row.payload["parse_warnings"]) == (value == "abc") + + +def test_debt_exceeds_claim_warning(): + row = next(records(fixture_text(**{"Debt remaining balance": "251"}))) + assert row.amount == Decimal("251.00") + assert "debt_exceeds_claim_amount" in row.payload["parse_warnings"] + + +def test_future_document_date_is_preserved(): + row = next( + records(fixture_text(**{"Date of the executive document": "01.01.2099"})) + ) + assert row.payload["document_date"] == "2099-01-01" + assert "future_document_date" in row.payload["parse_warnings"] + + +@pytest.mark.parametrize("raw,want", [("", None), ("bad", None), ("31.02.2026", None)]) +def test_missing_and_invalid_dates(raw, want): + row = next(records(fixture_text(**{"Date of the institution of proceeding": raw}))) + assert row.payload["initiated_on"] is want + assert row.record_date == "" + assert ("unparsed_date" in row.payload["parse_warnings"]) == bool(raw) + + +def test_reordered_headers_preserve_mapping(): + rows = list(csv.reader(StringIO(fixture_text()))) + handle = StringIO() + csv.writer(handle).writerows([list(reversed(row)) for row in rows]) + row = next(records(handle.getvalue())) + assert row.inn == "7707083893" + assert row.payload["proceeding_number"] == "123/26/77001-ИП" + assert row.amount == Decimal("-12.50") + + +@pytest.mark.parametrize( + "kind", ["unknown", "duplicate", "missing", "truncated", "unclosed"] +) +def test_unknown_or_corrupt_structure_rejected(kind): + rows = list(csv.reader(StringIO(fixture_text()))) + if kind == "unknown": + rows[0][0] = "new unknown column" + elif kind == "duplicate": + rows[0][0] = rows[0][1] + elif kind == "missing": + rows = [row[:-1] for row in rows] + elif kind == "truncated": + rows[1] = rows[1][:-1] + handle = StringIO() + csv.writer(handle).writerows(rows) + text = handle.getvalue() if kind != "unclosed" else handle.getvalue() + '"unclosed' + with pytest.raises(SnapshotValidationError): + list(records(text)) + + +def test_ended_dataset_has_distinct_mapping_and_no_fake_debt(): + row = next(records(fixture_text("ended.csv"), dataset=ENDED)) + assert row.amount is None + assert row.payload["claim_amount"] is None + assert row.status == "ended" + assert row.payload["terminated_on"] == "2026-10-02" + assert row.payload["termination_article"] == "46" + assert row.payload["termination_part"] == "1" + assert row.payload["termination_point"] == "3" + assert len(row.payload["raw_columns"]) == 16 + + +def test_iterator_does_not_read_next_row_until_requested(): + rows = list(csv.reader(StringIO(fixture_text()))) + handle = StringIO() + csv.writer(handle).writerows(rows) + iterator = records(handle.getvalue() + '"unclosed') + assert next(iterator).inn == "7707083893" + with pytest.raises(SnapshotValidationError): + next(iterator)