diff --git a/src/apps/parsers/checko_sections.py b/src/apps/parsers/checko_sections.py new file mode 100644 index 0000000..f28b13e --- /dev/null +++ b/src/apps/parsers/checko_sections.py @@ -0,0 +1,260 @@ +"""Three verified Checko sections, with complete pagination and stable identities.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterator +from datetime import date + +from apps.parsers.clients.checko import CheckoClient, CompanyRequest +from apps.parsers.clients.checko.exceptions import CheckoError +from apps.parsers.clients.checko.schemas.requests import FedresursMessagesRequest +from apps.parsers.registry_snapshots import SnapshotValidationError +from django.utils import timezone +from organizations.models import Organization +from organizations.source_ingestion import SourceRecordInput + +CHECKO_SECTIONS = { + "checko_leasing": "leasing_contract", + "checko_messages": "company_message", + "checko_licenses": "company_license", +} +LEASE_MESSAGE_TYPES = frozenset( + { + "FinancialLeaseContract", + "ChangeFinancialLeaseContract", + "StopFinancialLeaseContract", + "FinancialLeaseContract2", + "ChangeFinancialLeaseContract2", + "StopFinancialLeaseContract2", + } +) +CHECKO_MAX_PAGES = 100 +CHECKO_PAGE_SIZE = 100 +CHECKO_MAX_RECORDS = CHECKO_MAX_PAGES * CHECKO_PAGE_SIZE + + +def _text(value, *, required=False): + if value is None: + if required: + raise SnapshotValidationError("missing_checko_identity") + return None + if not isinstance(value, str) or (required and not value.strip()): + raise SnapshotValidationError("invalid_checko_field_type") + return value.strip() or None + + +def _date(value): + text = _text(value) + if not text: + return None + try: + return date.fromisoformat(text[:10]).isoformat() + except ValueError: + return None + + +def _digest(value): + return hashlib.sha256( + json.dumps(value, ensure_ascii=False, sort_keys=True).encode() + ).hexdigest() + + +def _licenses(client, organization, source, fetched_at): + response = client.get_company( + CompanyRequest(inn=organization.inn, ogrn=organization.ogrn) + ) + if ( + response.data is None + or response.data.inn != organization.inn + or response.data.ogrn != organization.ogrn + ): + raise SnapshotValidationError("checko_organization_identity_mismatch") + if not response.license_section_present: + raise SnapshotValidationError("checko_license_section_unavailable") + seen = {} + if len(response.data.licenses) > CHECKO_MAX_RECORDS: + raise SnapshotValidationError("checko_license_limit_exceeded") + for license, raw in zip( + response.data.licenses, response.license_upstream, strict=True + ): + raw_activities = raw.get("activities", raw.get("виддеят", [])) + if not isinstance(raw_activities, list): + raise SnapshotValidationError("invalid_checko_license_activities") + number = _text(license.number, required=True) + authority = _text(license.authority, required=True) + issued = _date(license.date) + if not issued: + raise SnapshotValidationError("unverified_checko_license_identity") + activities = [_text(value, required=True) for value in license.activities] + identity = f"{organization.inn}:{_digest([number, authority, issued])}" + if identity in seen: + if seen[identity] != raw: + raise SnapshotValidationError("conflicting_checko_license_identity") + continue + seen[identity] = raw + payload = { + "license_number": number, + "issued_on": issued, + "valid_from": _date(license.start_date), + "valid_until": _date(license.end_date), + "authority": authority, + "activities": activities, + "source_url": f"https://checko.ru/company/{organization.ogrn}", + "fetched_at": fetched_at, + "upstream": raw, + } + yield SourceRecordInput( + external_id=identity, + title=f"Лицензия {number}"[:255], + organization_name=organization.name, + record_type=CHECKO_SECTIONS[source], + inn=organization.inn, + ogrn=organization.ogrn, + record_date=issued, + amount=None, + status="published", + url=payload["source_url"], + payload=payload, + ) + + +def _message_record(raw, organization, source, fetched_at): + guid = _text(raw.get("GUID"), required=True) + message_type = _text(raw.get("Тип"), required=True) + label = _text(raw.get("ТипНаим")) + publisher = _text(raw.get("Публикатор"), required=True) + participants = raw.get("Участники") + if not isinstance(participants, list): + raise SnapshotValidationError("invalid_checko_participants") + participants = [_text(value, required=True) for value in participants] + if publisher != organization.ogrn and not ( + {organization.inn, organization.ogrn} & set(participants) + ): + raise SnapshotValidationError("checko_message_subject_mismatch") + payload = { + "message_guid": guid, + "publication_raw": _text(raw.get("Дата")), + "publication_date": _date(raw.get("Дата")), + "message_type": message_type, + "message_type_name": label, + "publisher_identifier": publisher, + "participant_identifiers": participants, + "source_url": _text(raw.get("URL"), required=True), + "fetched_at": fetched_at, + "upstream": raw, + } + if source == "checko_leasing": + payload["lease_event_type"] = message_type + identity = f"{organization.inn}:{guid}" + if len(identity) > 255: + raise SnapshotValidationError("invalid_checko_message_identity") + return SourceRecordInput( + external_id=identity, + title=(label or message_type)[:255], + organization_name=organization.name, + record_type=CHECKO_SECTIONS[source], + inn=organization.inn, + ogrn=organization.ogrn, + record_date=payload["publication_date"] or "", + amount=None, + status="published", + url=payload["source_url"], + payload=payload, + ) + + +def _validate_message_page(response, organization, page, previous_totals): + if (response.company_inn, response.company_ogrn) != ( + organization.inn, + organization.ogrn, + ): + raise SnapshotValidationError("checko_organization_identity_mismatch") + pagination = response.pagination + current = (pagination.total_records, pagination.total_pages) + if pagination.current_page != page or ( + previous_totals is not None and current != previous_totals + ): + raise SnapshotValidationError("inconsistent_checko_pagination") + if ( + pagination.total_records > CHECKO_MAX_RECORDS + or pagination.total_pages > CHECKO_MAX_PAGES + or len(response.records) > CHECKO_PAGE_SIZE + ): + raise SnapshotValidationError("checko_pagination_limit_exceeded") + if not pagination.total_records: + if page != 1 or pagination.total_pages not in (0, 1) or response.records: + raise SnapshotValidationError("invalid_checko_empty_envelope") + return current + if pagination.total_pages < page or not response.records: + raise SnapshotValidationError("incomplete_checko_pagination") + return current + + +def _new_message(seen, identity, raw): + fingerprint = _digest(raw) + if identity in seen: + if seen[identity] != fingerprint: + raise SnapshotValidationError("conflicting_checko_message_identity") + return False + seen[identity] = fingerprint + return True + + +def _messages(client, organization, source, fetched_at): + totals = None + pages_seen = set() + seen = {} + received = 0 + for page in range(1, CHECKO_MAX_PAGES + 1): + response = client.get_fedresurs_messages( + FedresursMessagesRequest( + inn=organization.inn, ogrn=organization.ogrn, page=page + ) + ) + totals = _validate_message_page(response, organization, page, totals) + pagination = response.pagination + if not pagination.total_records: + return + signature = _digest(response.records) + if signature in pages_seen: + raise SnapshotValidationError("repeated_checko_page") + pages_seen.add(signature) + for raw in response.records: + received += 1 + row = _message_record(raw, organization, source, fetched_at) + if not _new_message(seen, row.external_id, raw): + continue + if ( + source != "checko_leasing" + or row.payload["message_type"] in LEASE_MESSAGE_TYPES + ): + yield row + if received > pagination.total_records: + raise SnapshotValidationError("inconsistent_checko_record_count") + if page == pagination.total_pages: + if received != pagination.total_records: + raise SnapshotValidationError("incomplete_checko_record_count") + return + raise SnapshotValidationError("checko_pagination_limit_exceeded") + + +def collect_checko_section( + client: CheckoClient, *, organization: Organization, source: str +) -> Iterator[SourceRecordInput]: + """Exhaustion proves completeness; callers must stage until then, never publish early.""" + if source not in CHECKO_SECTIONS: + raise SnapshotValidationError("unknown_checko_section") + fetched_at = timezone.now().isoformat().replace("+00:00", "Z") + try: + if source == "checko_licenses": + yield from _licenses(client, organization, source, fetched_at) + else: + yield from _messages(client, organization, source, fetched_at) + except CheckoError as exc: + raise SnapshotValidationError( + f"checko_collection_failed:{type(exc).__name__}" + ) from exc + except (ValueError, TypeError, KeyError) as exc: + raise SnapshotValidationError("invalid_checko_section_schema") from exc diff --git a/src/apps/parsers/clients/checko/__init__.py b/src/apps/parsers/clients/checko/__init__.py index 287aa8d..5b685f9 100644 --- a/src/apps/parsers/clients/checko/__init__.py +++ b/src/apps/parsers/clients/checko/__init__.py @@ -61,6 +61,8 @@ from apps.parsers.clients.checko.schemas import ( EntrepreneurRequest, EntrepreneurResponse, EntrepreneurShort, + FedresursMessagesRequest, + FedresursMessagesResponse, FinancesRequest, FinancesResponse, InspectionsRequest, @@ -99,6 +101,7 @@ __all__ = [ # Request schemas "BankRequest", "CompanyRequest", + "FedresursMessagesRequest", "ContractsRequest", "EnforcementsRequest", "EntrepreneurRequest", @@ -110,6 +113,7 @@ __all__ = [ # Response schemas "BankResponse", "CompanyResponse", + "FedresursMessagesResponse", "ContractsResponse", "EnforcementsResponse", "EntrepreneurResponse", diff --git a/src/apps/parsers/clients/checko/client.py b/src/apps/parsers/clients/checko/client.py index 5a8c45e..a438921 100644 --- a/src/apps/parsers/clients/checko/client.py +++ b/src/apps/parsers/clients/checko/client.py @@ -39,6 +39,7 @@ from apps.parsers.clients.checko.schemas.requests import ( ContractsRequest, EnforcementsRequest, EntrepreneurRequest, + FedresursMessagesRequest, FinancesRequest, InspectionsRequest, LegalCasesRequest, @@ -68,6 +69,7 @@ from apps.parsers.clients.checko.schemas.responses import ( EntrepreneurData, EntrepreneurResponse, EntrepreneurStatus, + FedresursMessagesResponse, FinanceReport, FinanceReportLine, FinancesData, @@ -314,7 +316,9 @@ class CheckoClient: # HTTP методы # ========================================================================= - def _request(self, endpoint: str, params: dict[str, str]) -> dict[str, Any]: + def _request( + self, endpoint: str, params: dict[str, str], *, map_data: bool = True + ) -> dict[str, Any]: """ Выполнить запрос к API. @@ -362,7 +366,7 @@ class CheckoClient: self._raise_api_error(data) # Map Russian field names to English - if "data" in data: + if map_data and "data" in data: data["data"] = _map_ru_keys(data["data"]) return data @@ -477,9 +481,11 @@ class CheckoClient: number=data.get("number"), date=data.get("date"), start_date=data.get("start_date"), - end_date=data.get("end_date"), - authority=data.get("authority"), - activities=tuple(data.get("activities", [])), + end_date=_first_present(data, "end_date", "датаоконч"), + authority=_first_present(data, "authority", "лицорг"), + activities=tuple( + _first_present(data, "activities", "виддеят", default=[]) or [] + ), ) def _parse_trademark(self, data: dict) -> Trademark: @@ -873,12 +879,61 @@ class CheckoClient: raise CheckoValidationError("Необходимо указать ogrn, inn или okpo") data = self._request("/company", request.to_params()) + company = data.get("data") + if company is not None and not isinstance(company, dict): + raise CheckoValidationError("Invalid company section") + if company is not None and "licenses" in company: + licenses = company["licenses"] + if not isinstance(licenses, list) or any( + not isinstance(license, dict) for license in licenses + ): + raise CheckoValidationError("Invalid license section") return CompanyResponse( data=self._parse_company_data(data.get("data")), meta=self._parse_meta(data), + license_section_present=isinstance( + (data.get("data") or {}).get("licenses"), list + ), + license_upstream=tuple((data.get("data") or {}).get("licenses") or []), source_data=data.get("source_data"), ) + def get_fedresurs_messages( + self, request: FedresursMessagesRequest + ) -> FedresursMessagesResponse: + """Read one documented /fedresurs page, preserving its raw message fields.""" + if ( + not (request.inn and request.ogrn) + or request.page < 1 + or not 1 <= request.limit <= 100 + ): + raise CheckoValidationError("Invalid organization message request") + result = self._request("/fedresurs", request.to_params(), map_data=False) + data = result.get("data") + company = result.get("company") + if not isinstance(data, dict) or not isinstance(company, dict): + raise CheckoValidationError("Missing organization message envelope") + counts = [data.get(key) for key in ("ЗапВсего", "СтрВсего", "СтрТекущ")] + records = data.get("Записи") + if ( + any( + not isinstance(value, int) or isinstance(value, bool) or value < 0 + for value in counts + ) + or not isinstance(records, list) + or any(not isinstance(row, dict) for row in records) + ): + raise CheckoValidationError("Invalid organization message envelope") + return FedresursMessagesResponse( + company_inn=company.get("ИНН", ""), + company_ogrn=company.get("ОГРН", ""), + pagination=PaginationInfo( + total_records=counts[0], total_pages=counts[1], current_page=counts[2] + ), + records=tuple(records), + meta=self._parse_meta(result), + ) + # ========================================================================= # /entrepreneur endpoint # ========================================================================= diff --git a/src/apps/parsers/clients/checko/schemas/__init__.py b/src/apps/parsers/clients/checko/schemas/__init__.py index 4674b26..6e1ce7a 100644 --- a/src/apps/parsers/clients/checko/schemas/__init__.py +++ b/src/apps/parsers/clients/checko/schemas/__init__.py @@ -25,6 +25,7 @@ from apps.parsers.clients.checko.schemas.requests import ( ContractsRequest, EnforcementsRequest, EntrepreneurRequest, + FedresursMessagesRequest, FinancesRequest, InspectionsRequest, LegalCasesRequest, @@ -54,6 +55,7 @@ from apps.parsers.clients.checko.schemas.responses import ( EntrepreneurData, EntrepreneurResponse, EntrepreneurStatus, + FedresursMessagesResponse, FinanceReport, FinanceReportLine, FinancesData, @@ -101,6 +103,7 @@ __all__ = [ # Request models "BankRequest", "CompanyRequest", + "FedresursMessagesRequest", "ContractsRequest", "EnforcementsRequest", "EntrepreneurRequest", @@ -119,6 +122,7 @@ __all__ = [ "CaseParty", "CompanyData", "CompanyResponse", + "FedresursMessagesResponse", "CompanyStatistics", "CompanyStatus", "Contract", diff --git a/src/apps/parsers/clients/checko/schemas/requests.py b/src/apps/parsers/clients/checko/schemas/requests.py index 5f3b74a..c0e7295 100644 --- a/src/apps/parsers/clients/checko/schemas/requests.py +++ b/src/apps/parsers/clients/checko/schemas/requests.py @@ -421,3 +421,25 @@ class BankRequest: def to_params(self) -> dict[str, str]: """Преобразовать в параметры запроса.""" return {"bic": self.bic} + + +@dataclass +class FedresursMessagesRequest: + """Complete organization message search in the documented /fedresurs method.""" + + inn: str + ogrn: str + page: int = 1 + limit: int = 100 + role: str = "all" + sort: str = "date" + + def to_params(self) -> dict[str, str]: + return { + "inn": self.inn, + "ogrn": self.ogrn, + "page": str(self.page), + "limit": str(self.limit), + "role": self.role, + "sort": self.sort, + } diff --git a/src/apps/parsers/clients/checko/schemas/responses.py b/src/apps/parsers/clients/checko/schemas/responses.py index a5c96cc..e6d08e6 100644 --- a/src/apps/parsers/clients/checko/schemas/responses.py +++ b/src/apps/parsers/clients/checko/schemas/responses.py @@ -418,6 +418,12 @@ class CompanyResponse: meta: ApiMeta """Метаинформация ответа.""" + license_section_present: bool = False + """Distinguish documented empty licenses from omitted/unavailable data.""" + + license_upstream: tuple[dict, ...] = () + """Only license rows, without unrelated company/person data.""" + source_data: dict | None = None """Исходные данные ЕГРЮЛ (если source=true).""" @@ -1147,3 +1153,14 @@ class BankResponse: meta: ApiMeta """Метаинформация ответа.""" + + +@dataclass(frozen=True) +class FedresursMessagesResponse: + """Verified pagination envelope; each raw message is validated by the adapter.""" + + company_inn: str + company_ogrn: str + pagination: PaginationInfo + records: tuple[dict, ...] + meta: ApiMeta diff --git a/tests/apps/parsers/test_checko_sections.py b/tests/apps/parsers/test_checko_sections.py new file mode 100644 index 0000000..6a1e16c --- /dev/null +++ b/tests/apps/parsers/test_checko_sections.py @@ -0,0 +1,187 @@ +"""Verified Checko DTOs, complete pagination and fail-closed section collection.""" + +import importlib +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from apps.parsers.clients.checko import CheckoClient +from apps.parsers.registry_snapshots import SnapshotValidationError + +FIXTURES = Path(__file__).parents[2] / "fixtures/checko_sections" +ORG = SimpleNamespace(inn="7707083893", ogrn="1027700132195", name="АО Проверка") + + +def fixture(name): + return json.loads((FIXTURES / name).read_text()) + + +def client_with(*responses): + client = CheckoClient(api_key="synthetic-unit-test") + client._http_client.get_json = Mock(side_effect=responses) + return client + + +def collect(client, source="checko_messages"): + try: + module = importlib.import_module("apps.parsers.checko_sections") + except ModuleNotFoundError: + pytest.fail("collect_checko_section is not implemented") + return list(module.collect_checko_section(client, organization=ORG, source=source)) + + +def test_license_identity_ignores_mutable_expiry_and_keeps_current_aliases(): + first = collect(client_with(fixture("license.json")), "checko_licenses")[0] + later = collect(client_with(fixture("renewed.json")), "checko_licenses")[0] + assert first.external_id == later.external_id + assert first.record_type == "company_license" + assert first.payload["authority"] == "Синтетический лицензирующий орган" + assert first.payload["activities"] == ["Синтетический вид деятельности"] + assert first.payload["valid_until"] is None + assert later.payload["valid_until"] == "2030-01-01" + assert first.record_date == first.payload["issued_on"] == "2020-01-02" + assert first.amount is None + + +def test_all_pages_collected_and_messages_not_bankruptcy(): + client = client_with( + fixture("messages-page-1.json"), fixture("messages-page-2.json") + ) + rows = collect(client) + assert len(rows) == 2 + assert {row.record_type for row in rows} == {"company_message"} + assert rows[0].payload["message_type"] == "FinancialLeaseContract2" + assert rows[1].payload["message_type"] == "Liquidation" + assert rows[0].payload["participant_identifiers"] == [ORG.inn] + for index, call in enumerate(client._http_client.get_json.call_args_list, 1): + assert call.args[0] == "/fedresurs" + assert call.kwargs["params"]["page"] == str(index) + assert call.kwargs["params"]["limit"] == "100" + assert call.kwargs["params"]["role"] == "all" + assert all(row.amount is None and "amount" not in row.payload for row in rows) + assert "full_text" not in rows[0].payload + + +def test_leasing_filters_verified_types_after_full_result(): + client = client_with( + fixture("messages-page-1.json"), fixture("messages-page-2.json") + ) + rows = collect(client, "checko_leasing") + assert len(rows) == 1 + assert rows[0].record_type == "leasing_contract" + assert rows[0].payload["lease_event_type"] == "FinancialLeaseContract2" + assert client._http_client.get_json.call_count == 2 + + +@pytest.mark.parametrize( + "source,name", + [ + ("checko_messages", "messages-empty.json"), + ("checko_leasing", "messages-empty.json"), + ("checko_licenses", "license-empty.json"), + ], +) +def test_genuine_empty(source, name): + assert collect(client_with(fixture(name)), source) == [] + + +@pytest.mark.parametrize("name", ["denied.json", "rate-limit.json"]) +@pytest.mark.parametrize( + "source", ["checko_messages", "checko_leasing", "checko_licenses"] +) +def test_denied_is_not_verified_empty(name, source): + with pytest.raises(SnapshotValidationError): + collect(client_with(fixture(name)), source) + + +def test_repeated_cursor_is_not_complete(): + with pytest.raises(SnapshotValidationError): + collect( + client_with( + fixture("messages-page-1.json"), fixture("messages-page-1.json") + ) + ) + + +def test_conflicting_provider_identity_is_rejected(): + first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json") + second["data"]["Записи"][0]["GUID"] = first["data"]["Записи"][0]["GUID"] + with pytest.raises(SnapshotValidationError): + collect(client_with(first, second)) + + +@pytest.mark.parametrize( + "corruption", ["guid", "identity", "count", "pages", "envelope", "participants"] +) +def test_incomplete_or_unknown_message_schema_rejected(corruption): + response = fixture("messages-page-1.json") + if corruption == "guid": + response["data"]["Записи"][0].pop("GUID") + elif corruption == "identity": + response["company"]["ИНН"] = "0000000000" + elif corruption == "count": + response["data"]["ЗапВсего"] = 3 + elif corruption == "pages": + response["data"]["СтрВсего"] = 101 + elif corruption == "envelope": + response["data"].pop("Записи") + else: + response["data"]["Записи"][0]["Участники"] = "bad" + with pytest.raises(SnapshotValidationError): + collect(client_with(response, fixture("messages-page-2.json"))) + + +@pytest.mark.parametrize( + "corruption", ["section", "number", "authority", "issued", "identity"] +) +def test_missing_license_section_or_composite_identity_rejected(corruption): + response = fixture("license.json") + if corruption == "section": + response["data"].pop("Лиценз") + elif corruption == "identity": + response["data"]["ОГРН"] = "0000000000000" + else: + response["data"]["Лиценз"][0].pop( + {"number": "Номер", "authority": "ЛицОрг", "issued": "Дата"}[corruption] + ) + with pytest.raises(SnapshotValidationError): + collect(client_with(response), "checko_licenses") + + +def test_unknown_section_does_not_call_upstream(): + client = client_with() + with pytest.raises(SnapshotValidationError): + collect(client, "bankruptcy") + client._http_client.get_json.assert_not_called() + + +def test_repeated_page_content_with_advanced_number_rejected(): + first = fixture("messages-page-1.json") + second = fixture("messages-page-1.json") + second["data"]["СтрТекущ"] = 2 + with pytest.raises(SnapshotValidationError): + collect(client_with(first, second)) + + +def test_license_unknown_raw_fields_survive(): + response = fixture("license.json") + response["data"]["Лиценз"][0]["FutureField"] = {"nested": [None, 0, False]} + row = collect(client_with(response), "checko_licenses")[0] + assert row.payload["upstream"]["futurefield"] == {"nested": [None, 0, False]} + + +def test_bad_license_activities_rejected(): + response = fixture("license.json") + response["data"]["Лиценз"][0]["ВидДеят"] = "AB" + with pytest.raises(SnapshotValidationError): + collect(client_with(response), "checko_licenses") + + +@pytest.mark.parametrize("value", [None, "bad", ["bad"]]) +def test_malformed_license_section_rejected(value): + response = fixture("license.json") + response["data"]["Лиценз"] = value + with pytest.raises(SnapshotValidationError): + collect(client_with(response), "checko_licenses")