feat: normalize streaming FSSP CSV records
This commit is contained in:
212
src/apps/parsers/fssp_csv.py
Normal file
212
src/apps/parsers/fssp_csv.py
Normal file
@@ -0,0 +1,212 @@
|
|||||||
|
"""Streaming normalization of the two verified FSSP legal-entity CSV schemas."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import re
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from datetime import date, datetime
|
||||||
|
from decimal import Decimal, InvalidOperation
|
||||||
|
from typing import TextIO
|
||||||
|
|
||||||
|
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||||
|
from django.utils import timezone
|
||||||
|
from organizations.source_ingestion import SourceRecordInput
|
||||||
|
|
||||||
|
FSSP_SOURCE = "fssp_enforcement_proceedings"
|
||||||
|
GENERAL_DATASET = "7709576929-iplegallist"
|
||||||
|
ENDED_DATASET = "7709576929-iplegallistcomplete"
|
||||||
|
FSSP_DATASETS = (GENERAL_DATASET, ENDED_DATASET)
|
||||||
|
FSSP_HEADERS = {
|
||||||
|
"The name of the debtor": "debtor_name",
|
||||||
|
"Address of the debtor organization": "debtor_legal_address",
|
||||||
|
"Actual address of the debtor organization": "debtor_actual_address",
|
||||||
|
"The number of enforcement proceeding": "proceeding_number",
|
||||||
|
"Date of the institution of proceeding": "initiated_on",
|
||||||
|
"The number of the consolidated enforcement proceedings": "consolidated_proceeding_number",
|
||||||
|
"The executive document type": "document_type",
|
||||||
|
"Date of the executive document": "document_date",
|
||||||
|
"The number of executive document": "document_number",
|
||||||
|
"The object of executive documents": "document_subject_raw",
|
||||||
|
"The object of the execution": "subject",
|
||||||
|
"Departments of bailiffs": "department_name",
|
||||||
|
"Address of departments of bailiff": "department_address",
|
||||||
|
"Debtor TIN": "debtor_inn",
|
||||||
|
"TIN of the organization collector": "creditor_inn",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def valid_legal_inn(value: str) -> bool:
|
||||||
|
"""Validate a ten-digit legal-entity INN without dropping leading zeroes."""
|
||||||
|
if not re.fullmatch(r"[0-9]{10}", value):
|
||||||
|
return False
|
||||||
|
weights = (2, 4, 10, 3, 5, 9, 4, 6, 8)
|
||||||
|
return sum(
|
||||||
|
int(digit) * weight for digit, weight in zip(value, weights)
|
||||||
|
) % 11 % 10 == int(value[-1])
|
||||||
|
|
||||||
|
|
||||||
|
def _date(value: str | None, warnings: set[str]) -> str | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
for pattern in ("%d.%m.%Y", "%Y-%m-%d"):
|
||||||
|
try:
|
||||||
|
return datetime.strptime(value, pattern).date().isoformat()
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
warnings.add("unparsed_date")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _money(value: str | None, warnings: set[str], *, signed: bool) -> Decimal | None:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
normalized = re.sub(r"\s", "", value).replace(",", ".")
|
||||||
|
pattern = r"-?[0-9]+(?:\.[0-9]{1,2})?" if signed else r"[0-9]+(?:\.[0-9]{1,2})?"
|
||||||
|
try:
|
||||||
|
if not re.fullmatch(pattern, normalized):
|
||||||
|
raise InvalidOperation
|
||||||
|
result = Decimal(normalized)
|
||||||
|
if abs(result) >= Decimal("1000000000000000000"):
|
||||||
|
raise InvalidOperation
|
||||||
|
return result
|
||||||
|
except InvalidOperation:
|
||||||
|
warnings.add("unparsed_amount")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _join(*values: str | None) -> str | None:
|
||||||
|
return "\n".join(value for value in values if value) or None
|
||||||
|
|
||||||
|
|
||||||
|
def _termination(payload: dict, warnings: set[str]) -> None:
|
||||||
|
raw = payload["termination_raw"]
|
||||||
|
if not raw:
|
||||||
|
warnings.add("unparsed_termination")
|
||||||
|
return
|
||||||
|
match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw)
|
||||||
|
if match:
|
||||||
|
payload["terminated_on"] = _date(match.group(), warnings)
|
||||||
|
for label, field in (
|
||||||
|
("ст", "termination_article"),
|
||||||
|
("ч", "termination_part"),
|
||||||
|
("п", "termination_point"),
|
||||||
|
):
|
||||||
|
match = re.search(rf"\b{label}\.\s*([0-9]+)", raw)
|
||||||
|
if match:
|
||||||
|
payload[field] = match.group(1)
|
||||||
|
if not payload["terminated_on"] or not payload["termination_article"]:
|
||||||
|
warnings.add("unparsed_termination")
|
||||||
|
|
||||||
|
|
||||||
|
def iter_fssp_records(
|
||||||
|
handle: TextIO,
|
||||||
|
*,
|
||||||
|
dataset: str,
|
||||||
|
snapshot_date: str | None = None,
|
||||||
|
fetched_at: str | None = None,
|
||||||
|
) -> Iterator[SourceRecordInput]:
|
||||||
|
"""Read one logical row at a time; retain original columns for provenance."""
|
||||||
|
if dataset not in FSSP_DATASETS:
|
||||||
|
raise SnapshotValidationError("unknown_fssp_dataset")
|
||||||
|
fetched_at = fetched_at or timezone.now().isoformat()
|
||||||
|
comparison_date = date.fromisoformat(snapshot_date or fetched_at[:10])
|
||||||
|
expected = set(FSSP_HEADERS)
|
||||||
|
if dataset == GENERAL_DATASET:
|
||||||
|
expected.update(("Amount due", "Debt remaining balance"))
|
||||||
|
else:
|
||||||
|
expected.add("Date, complete IP reason")
|
||||||
|
reader = csv.reader(handle, strict=True)
|
||||||
|
try:
|
||||||
|
headers = next(reader, [])
|
||||||
|
if headers:
|
||||||
|
headers[0] = headers[0].lstrip("\ufeff")
|
||||||
|
if len(headers) != len(expected) or set(headers) != expected:
|
||||||
|
raise SnapshotValidationError("unknown_fssp_csv_headers")
|
||||||
|
for row_number, values in enumerate(reader, 1):
|
||||||
|
if len(values) != len(headers):
|
||||||
|
raise SnapshotValidationError("invalid_fssp_csv_row_width")
|
||||||
|
raw = dict(zip(headers, values))
|
||||||
|
payload = {
|
||||||
|
field: raw[key].strip() or None for key, field in FSSP_HEADERS.items()
|
||||||
|
}
|
||||||
|
warnings: set[str] = set()
|
||||||
|
for field in ("initiated_on", "document_date"):
|
||||||
|
payload[field] = _date(payload[field], warnings)
|
||||||
|
if (
|
||||||
|
payload["document_date"]
|
||||||
|
and date.fromisoformat(payload["document_date"]) > comparison_date
|
||||||
|
):
|
||||||
|
warnings.add("future_document_date")
|
||||||
|
debt = _money(raw.get("Debt remaining balance"), warnings, signed=True)
|
||||||
|
claim = _money(raw.get("Amount due"), warnings, signed=False)
|
||||||
|
if debt is not None and debt < 0:
|
||||||
|
warnings.add("negative_debt_balance")
|
||||||
|
if debt is not None and claim is not None and debt > claim:
|
||||||
|
warnings.add("debt_exceeds_claim_amount")
|
||||||
|
payload.update(
|
||||||
|
{
|
||||||
|
"origin_kind": "open_data_csv",
|
||||||
|
"queried_inn": None,
|
||||||
|
"debt_amount": format(debt, ".2f") if debt is not None else None,
|
||||||
|
"claim_amount": format(claim, ".2f") if claim is not None else None,
|
||||||
|
"debt_currency": "RUB"
|
||||||
|
if debt is not None or claim is not None
|
||||||
|
else None,
|
||||||
|
"document_issuer": None,
|
||||||
|
"termination_raw": raw.get("Date, complete IP reason", "").strip()
|
||||||
|
or None,
|
||||||
|
"terminated_on": None,
|
||||||
|
"termination_reason": None,
|
||||||
|
"termination_article": None,
|
||||||
|
"termination_part": None,
|
||||||
|
"termination_point": None,
|
||||||
|
"service_text": None,
|
||||||
|
"service_links": [],
|
||||||
|
"bailiff_raw": None,
|
||||||
|
"bailiff_name": None,
|
||||||
|
"bailiff_phones": [],
|
||||||
|
"source_url": f"https://opendata.fssp.gov.ru/{dataset}",
|
||||||
|
"fetched_at": fetched_at,
|
||||||
|
"raw_columns": raw,
|
||||||
|
"dataset": dataset,
|
||||||
|
"source_row_number": row_number,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
payload["debtor_raw"] = _join(
|
||||||
|
payload["debtor_name"],
|
||||||
|
payload["debtor_inn"],
|
||||||
|
payload["debtor_legal_address"],
|
||||||
|
)
|
||||||
|
payload["proceeding_raw"] = _join(
|
||||||
|
payload["proceeding_number"], payload["initiated_on"]
|
||||||
|
)
|
||||||
|
payload["document_raw"] = _join(
|
||||||
|
payload["document_type"],
|
||||||
|
payload["document_date"],
|
||||||
|
payload["document_number"],
|
||||||
|
payload["document_subject_raw"],
|
||||||
|
)
|
||||||
|
payload["subject_raw"] = _join(payload["subject"], payload["debt_amount"])
|
||||||
|
payload["department_raw"] = _join(
|
||||||
|
payload["department_name"], payload["department_address"]
|
||||||
|
)
|
||||||
|
if dataset == ENDED_DATASET:
|
||||||
|
_termination(payload, warnings)
|
||||||
|
payload["parse_warnings"] = sorted(warnings)
|
||||||
|
inn = payload["debtor_inn"] or ""
|
||||||
|
number = payload["proceeding_number"] or ""
|
||||||
|
yield SourceRecordInput(
|
||||||
|
external_id=f"{inn}:{number}",
|
||||||
|
title=number[:255],
|
||||||
|
organization_name=payload["debtor_name"] or "",
|
||||||
|
record_type="enforcement_proceeding",
|
||||||
|
inn=inn,
|
||||||
|
record_date=payload["initiated_on"] or "",
|
||||||
|
amount=debt,
|
||||||
|
status="ended" if dataset == ENDED_DATASET else "unknown",
|
||||||
|
url=payload["source_url"],
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
except (csv.Error, UnicodeError) as exc:
|
||||||
|
raise SnapshotValidationError("invalid_fssp_csv") from exc
|
||||||
134
tests/apps/parsers/test_fssp_csv.py
Normal file
134
tests/apps/parsers/test_fssp_csv.py
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
"""FSSP CSV normalization without network or business data."""
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import importlib
|
||||||
|
from decimal import Decimal
|
||||||
|
from io import StringIO
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||||
|
|
||||||
|
FIXTURES = Path(__file__).parents[2] / "fixtures" / "fssp"
|
||||||
|
GENERAL = "7709576929-iplegallist"
|
||||||
|
ENDED = "7709576929-iplegallistcomplete"
|
||||||
|
|
||||||
|
|
||||||
|
def records(text, dataset=GENERAL):
|
||||||
|
try:
|
||||||
|
module = importlib.import_module("apps.parsers.fssp_csv")
|
||||||
|
except ModuleNotFoundError:
|
||||||
|
pytest.fail("iter_fssp_records is not implemented")
|
||||||
|
return module.iter_fssp_records(
|
||||||
|
StringIO(text),
|
||||||
|
dataset=dataset,
|
||||||
|
snapshot_date="2026-10-05",
|
||||||
|
fetched_at="2026-10-05T08:00:00+00:00",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fixture_text(name="general.csv", **changes):
|
||||||
|
rows = list(csv.reader(StringIO((FIXTURES / name).read_text())))
|
||||||
|
for key, value in changes.items():
|
||||||
|
rows[1][rows[0].index(key)] = value
|
||||||
|
handle = StringIO()
|
||||||
|
csv.writer(handle, lineterminator="\n").writerows(rows)
|
||||||
|
return handle.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def test_signed_debt_and_multiline_preserved():
|
||||||
|
row = next(records(fixture_text()))
|
||||||
|
assert row.amount == Decimal("-12.50")
|
||||||
|
assert row.payload["debt_amount"] == "-12.50"
|
||||||
|
assert row.payload["claim_amount"] == "250.00"
|
||||||
|
assert row.payload["debtor_actual_address"] == "Москва, офис 2\nэтаж 3"
|
||||||
|
assert row.payload["origin_kind"] == "open_data_csv"
|
||||||
|
assert row.payload["queried_inn"] is None
|
||||||
|
assert row.payload["document_issuer"] is None
|
||||||
|
assert row.payload["document_subject_raw"] == "Взыскание сбора"
|
||||||
|
assert len(row.payload["raw_columns"]) == 17
|
||||||
|
assert row.record_date == "2026-10-01"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"value,want", [("0", "0.00"), ("", None), ("abc", None), ("1 234,50", "1234.50")]
|
||||||
|
)
|
||||||
|
def test_money_preserves_zero_and_missing(value, want):
|
||||||
|
row = next(records(fixture_text(**{"Debt remaining balance": value})))
|
||||||
|
assert row.payload["debt_amount"] == want
|
||||||
|
assert row.amount == (Decimal(want) if want is not None else None)
|
||||||
|
assert ("unparsed_amount" in row.payload["parse_warnings"]) == (value == "abc")
|
||||||
|
|
||||||
|
|
||||||
|
def test_debt_exceeds_claim_warning():
|
||||||
|
row = next(records(fixture_text(**{"Debt remaining balance": "251"})))
|
||||||
|
assert row.amount == Decimal("251.00")
|
||||||
|
assert "debt_exceeds_claim_amount" in row.payload["parse_warnings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_future_document_date_is_preserved():
|
||||||
|
row = next(
|
||||||
|
records(fixture_text(**{"Date of the executive document": "01.01.2099"}))
|
||||||
|
)
|
||||||
|
assert row.payload["document_date"] == "2099-01-01"
|
||||||
|
assert "future_document_date" in row.payload["parse_warnings"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("raw,want", [("", None), ("bad", None), ("31.02.2026", None)])
|
||||||
|
def test_missing_and_invalid_dates(raw, want):
|
||||||
|
row = next(records(fixture_text(**{"Date of the institution of proceeding": raw})))
|
||||||
|
assert row.payload["initiated_on"] is want
|
||||||
|
assert row.record_date == ""
|
||||||
|
assert ("unparsed_date" in row.payload["parse_warnings"]) == bool(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def test_reordered_headers_preserve_mapping():
|
||||||
|
rows = list(csv.reader(StringIO(fixture_text())))
|
||||||
|
handle = StringIO()
|
||||||
|
csv.writer(handle).writerows([list(reversed(row)) for row in rows])
|
||||||
|
row = next(records(handle.getvalue()))
|
||||||
|
assert row.inn == "7707083893"
|
||||||
|
assert row.payload["proceeding_number"] == "123/26/77001-ИП"
|
||||||
|
assert row.amount == Decimal("-12.50")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"kind", ["unknown", "duplicate", "missing", "truncated", "unclosed"]
|
||||||
|
)
|
||||||
|
def test_unknown_or_corrupt_structure_rejected(kind):
|
||||||
|
rows = list(csv.reader(StringIO(fixture_text())))
|
||||||
|
if kind == "unknown":
|
||||||
|
rows[0][0] = "new unknown column"
|
||||||
|
elif kind == "duplicate":
|
||||||
|
rows[0][0] = rows[0][1]
|
||||||
|
elif kind == "missing":
|
||||||
|
rows = [row[:-1] for row in rows]
|
||||||
|
elif kind == "truncated":
|
||||||
|
rows[1] = rows[1][:-1]
|
||||||
|
handle = StringIO()
|
||||||
|
csv.writer(handle).writerows(rows)
|
||||||
|
text = handle.getvalue() if kind != "unclosed" else handle.getvalue() + '"unclosed'
|
||||||
|
with pytest.raises(SnapshotValidationError):
|
||||||
|
list(records(text))
|
||||||
|
|
||||||
|
|
||||||
|
def test_ended_dataset_has_distinct_mapping_and_no_fake_debt():
|
||||||
|
row = next(records(fixture_text("ended.csv"), dataset=ENDED))
|
||||||
|
assert row.amount is None
|
||||||
|
assert row.payload["claim_amount"] is None
|
||||||
|
assert row.status == "ended"
|
||||||
|
assert row.payload["terminated_on"] == "2026-10-02"
|
||||||
|
assert row.payload["termination_article"] == "46"
|
||||||
|
assert row.payload["termination_part"] == "1"
|
||||||
|
assert row.payload["termination_point"] == "3"
|
||||||
|
assert len(row.payload["raw_columns"]) == 16
|
||||||
|
|
||||||
|
|
||||||
|
def test_iterator_does_not_read_next_row_until_requested():
|
||||||
|
rows = list(csv.reader(StringIO(fixture_text())))
|
||||||
|
handle = StringIO()
|
||||||
|
csv.writer(handle).writerows(rows)
|
||||||
|
iterator = records(handle.getvalue() + '"unclosed')
|
||||||
|
assert next(iterator).inn == "7707083893"
|
||||||
|
with pytest.raises(SnapshotValidationError):
|
||||||
|
next(iterator)
|
||||||
Reference in New Issue
Block a user