feat: normalize verified Checko sections
This commit is contained in:
260
src/apps/parsers/checko_sections.py
Normal file
260
src/apps/parsers/checko_sections.py
Normal file
@@ -0,0 +1,260 @@
|
||||
"""Three verified Checko sections, with complete pagination and stable identities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
|
||||
from apps.parsers.clients.checko import CheckoClient, CompanyRequest
|
||||
from apps.parsers.clients.checko.exceptions import CheckoError
|
||||
from apps.parsers.clients.checko.schemas.requests import FedresursMessagesRequest
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization
|
||||
from organizations.source_ingestion import SourceRecordInput
|
||||
|
||||
CHECKO_SECTIONS = {
|
||||
"checko_leasing": "leasing_contract",
|
||||
"checko_messages": "company_message",
|
||||
"checko_licenses": "company_license",
|
||||
}
|
||||
LEASE_MESSAGE_TYPES = frozenset(
|
||||
{
|
||||
"FinancialLeaseContract",
|
||||
"ChangeFinancialLeaseContract",
|
||||
"StopFinancialLeaseContract",
|
||||
"FinancialLeaseContract2",
|
||||
"ChangeFinancialLeaseContract2",
|
||||
"StopFinancialLeaseContract2",
|
||||
}
|
||||
)
|
||||
CHECKO_MAX_PAGES = 100
|
||||
CHECKO_PAGE_SIZE = 100
|
||||
CHECKO_MAX_RECORDS = CHECKO_MAX_PAGES * CHECKO_PAGE_SIZE
|
||||
|
||||
|
||||
def _text(value, *, required=False):
|
||||
if value is None:
|
||||
if required:
|
||||
raise SnapshotValidationError("missing_checko_identity")
|
||||
return None
|
||||
if not isinstance(value, str) or (required and not value.strip()):
|
||||
raise SnapshotValidationError("invalid_checko_field_type")
|
||||
return value.strip() or None
|
||||
|
||||
|
||||
def _date(value):
|
||||
text = _text(value)
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(text[:10]).isoformat()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _digest(value):
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, ensure_ascii=False, sort_keys=True).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def _licenses(client, organization, source, fetched_at):
|
||||
response = client.get_company(
|
||||
CompanyRequest(inn=organization.inn, ogrn=organization.ogrn)
|
||||
)
|
||||
if (
|
||||
response.data is None
|
||||
or response.data.inn != organization.inn
|
||||
or response.data.ogrn != organization.ogrn
|
||||
):
|
||||
raise SnapshotValidationError("checko_organization_identity_mismatch")
|
||||
if not response.license_section_present:
|
||||
raise SnapshotValidationError("checko_license_section_unavailable")
|
||||
seen = {}
|
||||
if len(response.data.licenses) > CHECKO_MAX_RECORDS:
|
||||
raise SnapshotValidationError("checko_license_limit_exceeded")
|
||||
for license, raw in zip(
|
||||
response.data.licenses, response.license_upstream, strict=True
|
||||
):
|
||||
raw_activities = raw.get("activities", raw.get("виддеят", []))
|
||||
if not isinstance(raw_activities, list):
|
||||
raise SnapshotValidationError("invalid_checko_license_activities")
|
||||
number = _text(license.number, required=True)
|
||||
authority = _text(license.authority, required=True)
|
||||
issued = _date(license.date)
|
||||
if not issued:
|
||||
raise SnapshotValidationError("unverified_checko_license_identity")
|
||||
activities = [_text(value, required=True) for value in license.activities]
|
||||
identity = f"{organization.inn}:{_digest([number, authority, issued])}"
|
||||
if identity in seen:
|
||||
if seen[identity] != raw:
|
||||
raise SnapshotValidationError("conflicting_checko_license_identity")
|
||||
continue
|
||||
seen[identity] = raw
|
||||
payload = {
|
||||
"license_number": number,
|
||||
"issued_on": issued,
|
||||
"valid_from": _date(license.start_date),
|
||||
"valid_until": _date(license.end_date),
|
||||
"authority": authority,
|
||||
"activities": activities,
|
||||
"source_url": f"https://checko.ru/company/{organization.ogrn}",
|
||||
"fetched_at": fetched_at,
|
||||
"upstream": raw,
|
||||
}
|
||||
yield SourceRecordInput(
|
||||
external_id=identity,
|
||||
title=f"Лицензия {number}"[:255],
|
||||
organization_name=organization.name,
|
||||
record_type=CHECKO_SECTIONS[source],
|
||||
inn=organization.inn,
|
||||
ogrn=organization.ogrn,
|
||||
record_date=issued,
|
||||
amount=None,
|
||||
status="published",
|
||||
url=payload["source_url"],
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
|
||||
def _message_record(raw, organization, source, fetched_at):
|
||||
guid = _text(raw.get("GUID"), required=True)
|
||||
message_type = _text(raw.get("Тип"), required=True)
|
||||
label = _text(raw.get("ТипНаим"))
|
||||
publisher = _text(raw.get("Публикатор"), required=True)
|
||||
participants = raw.get("Участники")
|
||||
if not isinstance(participants, list):
|
||||
raise SnapshotValidationError("invalid_checko_participants")
|
||||
participants = [_text(value, required=True) for value in participants]
|
||||
if publisher != organization.ogrn and not (
|
||||
{organization.inn, organization.ogrn} & set(participants)
|
||||
):
|
||||
raise SnapshotValidationError("checko_message_subject_mismatch")
|
||||
payload = {
|
||||
"message_guid": guid,
|
||||
"publication_raw": _text(raw.get("Дата")),
|
||||
"publication_date": _date(raw.get("Дата")),
|
||||
"message_type": message_type,
|
||||
"message_type_name": label,
|
||||
"publisher_identifier": publisher,
|
||||
"participant_identifiers": participants,
|
||||
"source_url": _text(raw.get("URL"), required=True),
|
||||
"fetched_at": fetched_at,
|
||||
"upstream": raw,
|
||||
}
|
||||
if source == "checko_leasing":
|
||||
payload["lease_event_type"] = message_type
|
||||
identity = f"{organization.inn}:{guid}"
|
||||
if len(identity) > 255:
|
||||
raise SnapshotValidationError("invalid_checko_message_identity")
|
||||
return SourceRecordInput(
|
||||
external_id=identity,
|
||||
title=(label or message_type)[:255],
|
||||
organization_name=organization.name,
|
||||
record_type=CHECKO_SECTIONS[source],
|
||||
inn=organization.inn,
|
||||
ogrn=organization.ogrn,
|
||||
record_date=payload["publication_date"] or "",
|
||||
amount=None,
|
||||
status="published",
|
||||
url=payload["source_url"],
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
|
||||
def _validate_message_page(response, organization, page, previous_totals):
|
||||
if (response.company_inn, response.company_ogrn) != (
|
||||
organization.inn,
|
||||
organization.ogrn,
|
||||
):
|
||||
raise SnapshotValidationError("checko_organization_identity_mismatch")
|
||||
pagination = response.pagination
|
||||
current = (pagination.total_records, pagination.total_pages)
|
||||
if pagination.current_page != page or (
|
||||
previous_totals is not None and current != previous_totals
|
||||
):
|
||||
raise SnapshotValidationError("inconsistent_checko_pagination")
|
||||
if (
|
||||
pagination.total_records > CHECKO_MAX_RECORDS
|
||||
or pagination.total_pages > CHECKO_MAX_PAGES
|
||||
or len(response.records) > CHECKO_PAGE_SIZE
|
||||
):
|
||||
raise SnapshotValidationError("checko_pagination_limit_exceeded")
|
||||
if not pagination.total_records:
|
||||
if page != 1 or pagination.total_pages not in (0, 1) or response.records:
|
||||
raise SnapshotValidationError("invalid_checko_empty_envelope")
|
||||
return current
|
||||
if pagination.total_pages < page or not response.records:
|
||||
raise SnapshotValidationError("incomplete_checko_pagination")
|
||||
return current
|
||||
|
||||
|
||||
def _new_message(seen, identity, raw):
|
||||
fingerprint = _digest(raw)
|
||||
if identity in seen:
|
||||
if seen[identity] != fingerprint:
|
||||
raise SnapshotValidationError("conflicting_checko_message_identity")
|
||||
return False
|
||||
seen[identity] = fingerprint
|
||||
return True
|
||||
|
||||
|
||||
def _messages(client, organization, source, fetched_at):
|
||||
totals = None
|
||||
pages_seen = set()
|
||||
seen = {}
|
||||
received = 0
|
||||
for page in range(1, CHECKO_MAX_PAGES + 1):
|
||||
response = client.get_fedresurs_messages(
|
||||
FedresursMessagesRequest(
|
||||
inn=organization.inn, ogrn=organization.ogrn, page=page
|
||||
)
|
||||
)
|
||||
totals = _validate_message_page(response, organization, page, totals)
|
||||
pagination = response.pagination
|
||||
if not pagination.total_records:
|
||||
return
|
||||
signature = _digest(response.records)
|
||||
if signature in pages_seen:
|
||||
raise SnapshotValidationError("repeated_checko_page")
|
||||
pages_seen.add(signature)
|
||||
for raw in response.records:
|
||||
received += 1
|
||||
row = _message_record(raw, organization, source, fetched_at)
|
||||
if not _new_message(seen, row.external_id, raw):
|
||||
continue
|
||||
if (
|
||||
source != "checko_leasing"
|
||||
or row.payload["message_type"] in LEASE_MESSAGE_TYPES
|
||||
):
|
||||
yield row
|
||||
if received > pagination.total_records:
|
||||
raise SnapshotValidationError("inconsistent_checko_record_count")
|
||||
if page == pagination.total_pages:
|
||||
if received != pagination.total_records:
|
||||
raise SnapshotValidationError("incomplete_checko_record_count")
|
||||
return
|
||||
raise SnapshotValidationError("checko_pagination_limit_exceeded")
|
||||
|
||||
|
||||
def collect_checko_section(
|
||||
client: CheckoClient, *, organization: Organization, source: str
|
||||
) -> Iterator[SourceRecordInput]:
|
||||
"""Exhaustion proves completeness; callers must stage until then, never publish early."""
|
||||
if source not in CHECKO_SECTIONS:
|
||||
raise SnapshotValidationError("unknown_checko_section")
|
||||
fetched_at = timezone.now().isoformat().replace("+00:00", "Z")
|
||||
try:
|
||||
if source == "checko_licenses":
|
||||
yield from _licenses(client, organization, source, fetched_at)
|
||||
else:
|
||||
yield from _messages(client, organization, source, fetched_at)
|
||||
except CheckoError as exc:
|
||||
raise SnapshotValidationError(
|
||||
f"checko_collection_failed:{type(exc).__name__}"
|
||||
) from exc
|
||||
except (ValueError, TypeError, KeyError) as exc:
|
||||
raise SnapshotValidationError("invalid_checko_section_schema") from exc
|
||||
@@ -61,6 +61,8 @@ from apps.parsers.clients.checko.schemas import (
|
||||
EntrepreneurRequest,
|
||||
EntrepreneurResponse,
|
||||
EntrepreneurShort,
|
||||
FedresursMessagesRequest,
|
||||
FedresursMessagesResponse,
|
||||
FinancesRequest,
|
||||
FinancesResponse,
|
||||
InspectionsRequest,
|
||||
@@ -99,6 +101,7 @@ __all__ = [
|
||||
# Request schemas
|
||||
"BankRequest",
|
||||
"CompanyRequest",
|
||||
"FedresursMessagesRequest",
|
||||
"ContractsRequest",
|
||||
"EnforcementsRequest",
|
||||
"EntrepreneurRequest",
|
||||
@@ -110,6 +113,7 @@ __all__ = [
|
||||
# Response schemas
|
||||
"BankResponse",
|
||||
"CompanyResponse",
|
||||
"FedresursMessagesResponse",
|
||||
"ContractsResponse",
|
||||
"EnforcementsResponse",
|
||||
"EntrepreneurResponse",
|
||||
|
||||
@@ -39,6 +39,7 @@ from apps.parsers.clients.checko.schemas.requests import (
|
||||
ContractsRequest,
|
||||
EnforcementsRequest,
|
||||
EntrepreneurRequest,
|
||||
FedresursMessagesRequest,
|
||||
FinancesRequest,
|
||||
InspectionsRequest,
|
||||
LegalCasesRequest,
|
||||
@@ -68,6 +69,7 @@ from apps.parsers.clients.checko.schemas.responses import (
|
||||
EntrepreneurData,
|
||||
EntrepreneurResponse,
|
||||
EntrepreneurStatus,
|
||||
FedresursMessagesResponse,
|
||||
FinanceReport,
|
||||
FinanceReportLine,
|
||||
FinancesData,
|
||||
@@ -314,7 +316,9 @@ class CheckoClient:
|
||||
# HTTP методы
|
||||
# =========================================================================
|
||||
|
||||
def _request(self, endpoint: str, params: dict[str, str]) -> dict[str, Any]:
|
||||
def _request(
|
||||
self, endpoint: str, params: dict[str, str], *, map_data: bool = True
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Выполнить запрос к API.
|
||||
|
||||
@@ -362,7 +366,7 @@ class CheckoClient:
|
||||
self._raise_api_error(data)
|
||||
|
||||
# Map Russian field names to English
|
||||
if "data" in data:
|
||||
if map_data and "data" in data:
|
||||
data["data"] = _map_ru_keys(data["data"])
|
||||
|
||||
return data
|
||||
@@ -477,9 +481,11 @@ class CheckoClient:
|
||||
number=data.get("number"),
|
||||
date=data.get("date"),
|
||||
start_date=data.get("start_date"),
|
||||
end_date=data.get("end_date"),
|
||||
authority=data.get("authority"),
|
||||
activities=tuple(data.get("activities", [])),
|
||||
end_date=_first_present(data, "end_date", "датаоконч"),
|
||||
authority=_first_present(data, "authority", "лицорг"),
|
||||
activities=tuple(
|
||||
_first_present(data, "activities", "виддеят", default=[]) or []
|
||||
),
|
||||
)
|
||||
|
||||
def _parse_trademark(self, data: dict) -> Trademark:
|
||||
@@ -873,12 +879,61 @@ class CheckoClient:
|
||||
raise CheckoValidationError("Необходимо указать ogrn, inn или okpo")
|
||||
|
||||
data = self._request("/company", request.to_params())
|
||||
company = data.get("data")
|
||||
if company is not None and not isinstance(company, dict):
|
||||
raise CheckoValidationError("Invalid company section")
|
||||
if company is not None and "licenses" in company:
|
||||
licenses = company["licenses"]
|
||||
if not isinstance(licenses, list) or any(
|
||||
not isinstance(license, dict) for license in licenses
|
||||
):
|
||||
raise CheckoValidationError("Invalid license section")
|
||||
return CompanyResponse(
|
||||
data=self._parse_company_data(data.get("data")),
|
||||
meta=self._parse_meta(data),
|
||||
license_section_present=isinstance(
|
||||
(data.get("data") or {}).get("licenses"), list
|
||||
),
|
||||
license_upstream=tuple((data.get("data") or {}).get("licenses") or []),
|
||||
source_data=data.get("source_data"),
|
||||
)
|
||||
|
||||
def get_fedresurs_messages(
|
||||
self, request: FedresursMessagesRequest
|
||||
) -> FedresursMessagesResponse:
|
||||
"""Read one documented /fedresurs page, preserving its raw message fields."""
|
||||
if (
|
||||
not (request.inn and request.ogrn)
|
||||
or request.page < 1
|
||||
or not 1 <= request.limit <= 100
|
||||
):
|
||||
raise CheckoValidationError("Invalid organization message request")
|
||||
result = self._request("/fedresurs", request.to_params(), map_data=False)
|
||||
data = result.get("data")
|
||||
company = result.get("company")
|
||||
if not isinstance(data, dict) or not isinstance(company, dict):
|
||||
raise CheckoValidationError("Missing organization message envelope")
|
||||
counts = [data.get(key) for key in ("ЗапВсего", "СтрВсего", "СтрТекущ")]
|
||||
records = data.get("Записи")
|
||||
if (
|
||||
any(
|
||||
not isinstance(value, int) or isinstance(value, bool) or value < 0
|
||||
for value in counts
|
||||
)
|
||||
or not isinstance(records, list)
|
||||
or any(not isinstance(row, dict) for row in records)
|
||||
):
|
||||
raise CheckoValidationError("Invalid organization message envelope")
|
||||
return FedresursMessagesResponse(
|
||||
company_inn=company.get("ИНН", ""),
|
||||
company_ogrn=company.get("ОГРН", ""),
|
||||
pagination=PaginationInfo(
|
||||
total_records=counts[0], total_pages=counts[1], current_page=counts[2]
|
||||
),
|
||||
records=tuple(records),
|
||||
meta=self._parse_meta(result),
|
||||
)
|
||||
|
||||
# =========================================================================
|
||||
# /entrepreneur endpoint
|
||||
# =========================================================================
|
||||
|
||||
@@ -25,6 +25,7 @@ from apps.parsers.clients.checko.schemas.requests import (
|
||||
ContractsRequest,
|
||||
EnforcementsRequest,
|
||||
EntrepreneurRequest,
|
||||
FedresursMessagesRequest,
|
||||
FinancesRequest,
|
||||
InspectionsRequest,
|
||||
LegalCasesRequest,
|
||||
@@ -54,6 +55,7 @@ from apps.parsers.clients.checko.schemas.responses import (
|
||||
EntrepreneurData,
|
||||
EntrepreneurResponse,
|
||||
EntrepreneurStatus,
|
||||
FedresursMessagesResponse,
|
||||
FinanceReport,
|
||||
FinanceReportLine,
|
||||
FinancesData,
|
||||
@@ -101,6 +103,7 @@ __all__ = [
|
||||
# Request models
|
||||
"BankRequest",
|
||||
"CompanyRequest",
|
||||
"FedresursMessagesRequest",
|
||||
"ContractsRequest",
|
||||
"EnforcementsRequest",
|
||||
"EntrepreneurRequest",
|
||||
@@ -119,6 +122,7 @@ __all__ = [
|
||||
"CaseParty",
|
||||
"CompanyData",
|
||||
"CompanyResponse",
|
||||
"FedresursMessagesResponse",
|
||||
"CompanyStatistics",
|
||||
"CompanyStatus",
|
||||
"Contract",
|
||||
|
||||
@@ -421,3 +421,25 @@ class BankRequest:
|
||||
def to_params(self) -> dict[str, str]:
|
||||
"""Преобразовать в параметры запроса."""
|
||||
return {"bic": self.bic}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FedresursMessagesRequest:
|
||||
"""Complete organization message search in the documented /fedresurs method."""
|
||||
|
||||
inn: str
|
||||
ogrn: str
|
||||
page: int = 1
|
||||
limit: int = 100
|
||||
role: str = "all"
|
||||
sort: str = "date"
|
||||
|
||||
def to_params(self) -> dict[str, str]:
|
||||
return {
|
||||
"inn": self.inn,
|
||||
"ogrn": self.ogrn,
|
||||
"page": str(self.page),
|
||||
"limit": str(self.limit),
|
||||
"role": self.role,
|
||||
"sort": self.sort,
|
||||
}
|
||||
|
||||
@@ -418,6 +418,12 @@ class CompanyResponse:
|
||||
meta: ApiMeta
|
||||
"""Метаинформация ответа."""
|
||||
|
||||
license_section_present: bool = False
|
||||
"""Distinguish documented empty licenses from omitted/unavailable data."""
|
||||
|
||||
license_upstream: tuple[dict, ...] = ()
|
||||
"""Only license rows, without unrelated company/person data."""
|
||||
|
||||
source_data: dict | None = None
|
||||
"""Исходные данные ЕГРЮЛ (если source=true)."""
|
||||
|
||||
@@ -1147,3 +1153,14 @@ class BankResponse:
|
||||
|
||||
meta: ApiMeta
|
||||
"""Метаинформация ответа."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FedresursMessagesResponse:
|
||||
"""Verified pagination envelope; each raw message is validated by the adapter."""
|
||||
|
||||
company_inn: str
|
||||
company_ogrn: str
|
||||
pagination: PaginationInfo
|
||||
records: tuple[dict, ...]
|
||||
meta: ApiMeta
|
||||
|
||||
Reference in New Issue
Block a user