feat: normalize verified Checko sections

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:23:07 +02:00
parent 7375da2548
commit a0f623effe
7 changed files with 554 additions and 5 deletions

View File

@@ -0,0 +1,260 @@
"""Three verified Checko sections, with complete pagination and stable identities."""
from __future__ import annotations
import hashlib
import json
from collections.abc import Iterator
from datetime import date
from apps.parsers.clients.checko import CheckoClient, CompanyRequest
from apps.parsers.clients.checko.exceptions import CheckoError
from apps.parsers.clients.checko.schemas.requests import FedresursMessagesRequest
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.utils import timezone
from organizations.models import Organization
from organizations.source_ingestion import SourceRecordInput
CHECKO_SECTIONS = {
"checko_leasing": "leasing_contract",
"checko_messages": "company_message",
"checko_licenses": "company_license",
}
LEASE_MESSAGE_TYPES = frozenset(
{
"FinancialLeaseContract",
"ChangeFinancialLeaseContract",
"StopFinancialLeaseContract",
"FinancialLeaseContract2",
"ChangeFinancialLeaseContract2",
"StopFinancialLeaseContract2",
}
)
CHECKO_MAX_PAGES = 100
CHECKO_PAGE_SIZE = 100
CHECKO_MAX_RECORDS = CHECKO_MAX_PAGES * CHECKO_PAGE_SIZE
def _text(value, *, required=False):
if value is None:
if required:
raise SnapshotValidationError("missing_checko_identity")
return None
if not isinstance(value, str) or (required and not value.strip()):
raise SnapshotValidationError("invalid_checko_field_type")
return value.strip() or None
def _date(value):
text = _text(value)
if not text:
return None
try:
return date.fromisoformat(text[:10]).isoformat()
except ValueError:
return None
def _digest(value):
return hashlib.sha256(
json.dumps(value, ensure_ascii=False, sort_keys=True).encode()
).hexdigest()
def _licenses(client, organization, source, fetched_at):
response = client.get_company(
CompanyRequest(inn=organization.inn, ogrn=organization.ogrn)
)
if (
response.data is None
or response.data.inn != organization.inn
or response.data.ogrn != organization.ogrn
):
raise SnapshotValidationError("checko_organization_identity_mismatch")
if not response.license_section_present:
raise SnapshotValidationError("checko_license_section_unavailable")
seen = {}
if len(response.data.licenses) > CHECKO_MAX_RECORDS:
raise SnapshotValidationError("checko_license_limit_exceeded")
for license, raw in zip(
response.data.licenses, response.license_upstream, strict=True
):
raw_activities = raw.get("activities", raw.get("виддеят", []))
if not isinstance(raw_activities, list):
raise SnapshotValidationError("invalid_checko_license_activities")
number = _text(license.number, required=True)
authority = _text(license.authority, required=True)
issued = _date(license.date)
if not issued:
raise SnapshotValidationError("unverified_checko_license_identity")
activities = [_text(value, required=True) for value in license.activities]
identity = f"{organization.inn}:{_digest([number, authority, issued])}"
if identity in seen:
if seen[identity] != raw:
raise SnapshotValidationError("conflicting_checko_license_identity")
continue
seen[identity] = raw
payload = {
"license_number": number,
"issued_on": issued,
"valid_from": _date(license.start_date),
"valid_until": _date(license.end_date),
"authority": authority,
"activities": activities,
"source_url": f"https://checko.ru/company/{organization.ogrn}",
"fetched_at": fetched_at,
"upstream": raw,
}
yield SourceRecordInput(
external_id=identity,
title=f"Лицензия {number}"[:255],
organization_name=organization.name,
record_type=CHECKO_SECTIONS[source],
inn=organization.inn,
ogrn=organization.ogrn,
record_date=issued,
amount=None,
status="published",
url=payload["source_url"],
payload=payload,
)
def _message_record(raw, organization, source, fetched_at):
guid = _text(raw.get("GUID"), required=True)
message_type = _text(raw.get("Тип"), required=True)
label = _text(raw.get("ТипНаим"))
publisher = _text(raw.get("Публикатор"), required=True)
participants = raw.get("Участники")
if not isinstance(participants, list):
raise SnapshotValidationError("invalid_checko_participants")
participants = [_text(value, required=True) for value in participants]
if publisher != organization.ogrn and not (
{organization.inn, organization.ogrn} & set(participants)
):
raise SnapshotValidationError("checko_message_subject_mismatch")
payload = {
"message_guid": guid,
"publication_raw": _text(raw.get("Дата")),
"publication_date": _date(raw.get("Дата")),
"message_type": message_type,
"message_type_name": label,
"publisher_identifier": publisher,
"participant_identifiers": participants,
"source_url": _text(raw.get("URL"), required=True),
"fetched_at": fetched_at,
"upstream": raw,
}
if source == "checko_leasing":
payload["lease_event_type"] = message_type
identity = f"{organization.inn}:{guid}"
if len(identity) > 255:
raise SnapshotValidationError("invalid_checko_message_identity")
return SourceRecordInput(
external_id=identity,
title=(label or message_type)[:255],
organization_name=organization.name,
record_type=CHECKO_SECTIONS[source],
inn=organization.inn,
ogrn=organization.ogrn,
record_date=payload["publication_date"] or "",
amount=None,
status="published",
url=payload["source_url"],
payload=payload,
)
def _validate_message_page(response, organization, page, previous_totals):
if (response.company_inn, response.company_ogrn) != (
organization.inn,
organization.ogrn,
):
raise SnapshotValidationError("checko_organization_identity_mismatch")
pagination = response.pagination
current = (pagination.total_records, pagination.total_pages)
if pagination.current_page != page or (
previous_totals is not None and current != previous_totals
):
raise SnapshotValidationError("inconsistent_checko_pagination")
if (
pagination.total_records > CHECKO_MAX_RECORDS
or pagination.total_pages > CHECKO_MAX_PAGES
or len(response.records) > CHECKO_PAGE_SIZE
):
raise SnapshotValidationError("checko_pagination_limit_exceeded")
if not pagination.total_records:
if page != 1 or pagination.total_pages not in (0, 1) or response.records:
raise SnapshotValidationError("invalid_checko_empty_envelope")
return current
if pagination.total_pages < page or not response.records:
raise SnapshotValidationError("incomplete_checko_pagination")
return current
def _new_message(seen, identity, raw):
fingerprint = _digest(raw)
if identity in seen:
if seen[identity] != fingerprint:
raise SnapshotValidationError("conflicting_checko_message_identity")
return False
seen[identity] = fingerprint
return True
def _messages(client, organization, source, fetched_at):
totals = None
pages_seen = set()
seen = {}
received = 0
for page in range(1, CHECKO_MAX_PAGES + 1):
response = client.get_fedresurs_messages(
FedresursMessagesRequest(
inn=organization.inn, ogrn=organization.ogrn, page=page
)
)
totals = _validate_message_page(response, organization, page, totals)
pagination = response.pagination
if not pagination.total_records:
return
signature = _digest(response.records)
if signature in pages_seen:
raise SnapshotValidationError("repeated_checko_page")
pages_seen.add(signature)
for raw in response.records:
received += 1
row = _message_record(raw, organization, source, fetched_at)
if not _new_message(seen, row.external_id, raw):
continue
if (
source != "checko_leasing"
or row.payload["message_type"] in LEASE_MESSAGE_TYPES
):
yield row
if received > pagination.total_records:
raise SnapshotValidationError("inconsistent_checko_record_count")
if page == pagination.total_pages:
if received != pagination.total_records:
raise SnapshotValidationError("incomplete_checko_record_count")
return
raise SnapshotValidationError("checko_pagination_limit_exceeded")
def collect_checko_section(
client: CheckoClient, *, organization: Organization, source: str
) -> Iterator[SourceRecordInput]:
"""Exhaustion proves completeness; callers must stage until then, never publish early."""
if source not in CHECKO_SECTIONS:
raise SnapshotValidationError("unknown_checko_section")
fetched_at = timezone.now().isoformat().replace("+00:00", "Z")
try:
if source == "checko_licenses":
yield from _licenses(client, organization, source, fetched_at)
else:
yield from _messages(client, organization, source, fetched_at)
except CheckoError as exc:
raise SnapshotValidationError(
f"checko_collection_failed:{type(exc).__name__}"
) from exc
except (ValueError, TypeError, KeyError) as exc:
raise SnapshotValidationError("invalid_checko_section_schema") from exc

View File

@@ -61,6 +61,8 @@ from apps.parsers.clients.checko.schemas import (
EntrepreneurRequest,
EntrepreneurResponse,
EntrepreneurShort,
FedresursMessagesRequest,
FedresursMessagesResponse,
FinancesRequest,
FinancesResponse,
InspectionsRequest,
@@ -99,6 +101,7 @@ __all__ = [
# Request schemas
"BankRequest",
"CompanyRequest",
"FedresursMessagesRequest",
"ContractsRequest",
"EnforcementsRequest",
"EntrepreneurRequest",
@@ -110,6 +113,7 @@ __all__ = [
# Response schemas
"BankResponse",
"CompanyResponse",
"FedresursMessagesResponse",
"ContractsResponse",
"EnforcementsResponse",
"EntrepreneurResponse",

View File

@@ -39,6 +39,7 @@ from apps.parsers.clients.checko.schemas.requests import (
ContractsRequest,
EnforcementsRequest,
EntrepreneurRequest,
FedresursMessagesRequest,
FinancesRequest,
InspectionsRequest,
LegalCasesRequest,
@@ -68,6 +69,7 @@ from apps.parsers.clients.checko.schemas.responses import (
EntrepreneurData,
EntrepreneurResponse,
EntrepreneurStatus,
FedresursMessagesResponse,
FinanceReport,
FinanceReportLine,
FinancesData,
@@ -314,7 +316,9 @@ class CheckoClient:
# HTTP методы
# =========================================================================
def _request(self, endpoint: str, params: dict[str, str]) -> dict[str, Any]:
def _request(
self, endpoint: str, params: dict[str, str], *, map_data: bool = True
) -> dict[str, Any]:
"""
Выполнить запрос к API.
@@ -362,7 +366,7 @@ class CheckoClient:
self._raise_api_error(data)
# Map Russian field names to English
if "data" in data:
if map_data and "data" in data:
data["data"] = _map_ru_keys(data["data"])
return data
@@ -477,9 +481,11 @@ class CheckoClient:
number=data.get("number"),
date=data.get("date"),
start_date=data.get("start_date"),
end_date=data.get("end_date"),
authority=data.get("authority"),
activities=tuple(data.get("activities", [])),
end_date=_first_present(data, "end_date", "датаоконч"),
authority=_first_present(data, "authority", "лицорг"),
activities=tuple(
_first_present(data, "activities", "виддеят", default=[]) or []
),
)
def _parse_trademark(self, data: dict) -> Trademark:
@@ -873,12 +879,61 @@ class CheckoClient:
raise CheckoValidationError("Необходимо указать ogrn, inn или okpo")
data = self._request("/company", request.to_params())
company = data.get("data")
if company is not None and not isinstance(company, dict):
raise CheckoValidationError("Invalid company section")
if company is not None and "licenses" in company:
licenses = company["licenses"]
if not isinstance(licenses, list) or any(
not isinstance(license, dict) for license in licenses
):
raise CheckoValidationError("Invalid license section")
return CompanyResponse(
data=self._parse_company_data(data.get("data")),
meta=self._parse_meta(data),
license_section_present=isinstance(
(data.get("data") or {}).get("licenses"), list
),
license_upstream=tuple((data.get("data") or {}).get("licenses") or []),
source_data=data.get("source_data"),
)
def get_fedresurs_messages(
self, request: FedresursMessagesRequest
) -> FedresursMessagesResponse:
"""Read one documented /fedresurs page, preserving its raw message fields."""
if (
not (request.inn and request.ogrn)
or request.page < 1
or not 1 <= request.limit <= 100
):
raise CheckoValidationError("Invalid organization message request")
result = self._request("/fedresurs", request.to_params(), map_data=False)
data = result.get("data")
company = result.get("company")
if not isinstance(data, dict) or not isinstance(company, dict):
raise CheckoValidationError("Missing organization message envelope")
counts = [data.get(key) for key in ("ЗапВсего", "СтрВсего", "СтрТекущ")]
records = data.get("Записи")
if (
any(
not isinstance(value, int) or isinstance(value, bool) or value < 0
for value in counts
)
or not isinstance(records, list)
or any(not isinstance(row, dict) for row in records)
):
raise CheckoValidationError("Invalid organization message envelope")
return FedresursMessagesResponse(
company_inn=company.get("ИНН", ""),
company_ogrn=company.get("ОГРН", ""),
pagination=PaginationInfo(
total_records=counts[0], total_pages=counts[1], current_page=counts[2]
),
records=tuple(records),
meta=self._parse_meta(result),
)
# =========================================================================
# /entrepreneur endpoint
# =========================================================================

View File

@@ -25,6 +25,7 @@ from apps.parsers.clients.checko.schemas.requests import (
ContractsRequest,
EnforcementsRequest,
EntrepreneurRequest,
FedresursMessagesRequest,
FinancesRequest,
InspectionsRequest,
LegalCasesRequest,
@@ -54,6 +55,7 @@ from apps.parsers.clients.checko.schemas.responses import (
EntrepreneurData,
EntrepreneurResponse,
EntrepreneurStatus,
FedresursMessagesResponse,
FinanceReport,
FinanceReportLine,
FinancesData,
@@ -101,6 +103,7 @@ __all__ = [
# Request models
"BankRequest",
"CompanyRequest",
"FedresursMessagesRequest",
"ContractsRequest",
"EnforcementsRequest",
"EntrepreneurRequest",
@@ -119,6 +122,7 @@ __all__ = [
"CaseParty",
"CompanyData",
"CompanyResponse",
"FedresursMessagesResponse",
"CompanyStatistics",
"CompanyStatus",
"Contract",

View File

@@ -421,3 +421,25 @@ class BankRequest:
def to_params(self) -> dict[str, str]:
"""Преобразовать в параметры запроса."""
return {"bic": self.bic}
@dataclass
class FedresursMessagesRequest:
"""Complete organization message search in the documented /fedresurs method."""
inn: str
ogrn: str
page: int = 1
limit: int = 100
role: str = "all"
sort: str = "date"
def to_params(self) -> dict[str, str]:
return {
"inn": self.inn,
"ogrn": self.ogrn,
"page": str(self.page),
"limit": str(self.limit),
"role": self.role,
"sort": self.sort,
}

View File

@@ -418,6 +418,12 @@ class CompanyResponse:
meta: ApiMeta
"""Метаинформация ответа."""
license_section_present: bool = False
"""Distinguish documented empty licenses from omitted/unavailable data."""
license_upstream: tuple[dict, ...] = ()
"""Only license rows, without unrelated company/person data."""
source_data: dict | None = None
"""Исходные данные ЕГРЮЛ (если source=true)."""
@@ -1147,3 +1153,14 @@ class BankResponse:
meta: ApiMeta
"""Метаинформация ответа."""
@dataclass(frozen=True)
class FedresursMessagesResponse:
"""Verified pagination envelope; each raw message is validated by the adapter."""
company_inn: str
company_ogrn: str
pagination: PaginationInfo
records: tuple[dict, ...]
meta: ApiMeta