From 007cecc8d5734f83dc647f26901cac5aa20ce71b Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Thu, 16 Jul 2026 12:59:23 +0200 Subject: [PATCH 1/6] feat: expand registry ingestion and demo exchange --- .env.prod.example | 3 + docker/Dockerfile | 1 + src/apps/exchange/state_corp_services.py | 188 +++++- src/apps/parsers/clients/base.py | 15 +- src/apps/parsers/clients/checko/client.py | 79 ++- .../clients/common/eis_russian_trusted_ca.pem | 79 +++ src/apps/parsers/clients/common/structured.py | 73 ++- src/apps/parsers/clients/fns/__init__.py | 16 +- src/apps/parsers/clients/fns/api.py | 303 ++++++++++ src/apps/parsers/clients/gisp/__init__.py | 13 + src/apps/parsers/clients/gisp/client.py | 226 ++++++++ src/apps/parsers/clients/proverki/client.py | 46 +- src/apps/parsers/clients/trudvsem/client.py | 2 +- src/apps/parsers/clients/vacancies.py | 17 +- .../0025_consolidate_inspections_schedule.py | 63 ++ .../0026_update_fns_financial_schedule.py | 51 ++ src/apps/parsers/source_cards.py | 6 +- src/apps/parsers/source_registry.py | 24 +- src/apps/parsers/tasks.py | 333 ++++++++++- src/apps/parsers/views.py | 1 + src/core/celery.py | 4 - src/organizations/cache.py | 1 + src/organizations/filters.py | 33 +- .../commands/create_test_companies.py | 29 + .../commands/delete_test_companies.py | 24 + src/organizations/test_companies.py | 539 ++++++++++++++++++ src/organizations/views.py | 74 +-- src/settings/base.py | 8 + tests/apps/core/test_celery_module.py | 2 +- tests/apps/organizations/test_api_v2.py | 149 +++-- .../test_api_v2_source_extensions.py | 35 +- .../test_test_companies_commands.py | 168 ++++++ tests/apps/parsers/test_checko.py | 35 ++ tests/apps/parsers/test_fns_api_client.py | 156 +++++ tests/apps/parsers/test_gisp_client.py | 174 ++++++ tests/apps/parsers/test_parser_schedules.py | 104 ++++ tests/apps/parsers/test_proverki_client.py | 115 ++++ .../apps/parsers/test_source_cards_service.py | 23 + tests/apps/parsers/test_source_cards_views.py | 26 +- tests/apps/parsers/test_source_registry.py | 42 ++ .../parsers/test_structured_data_client.py | 249 +++++++- tests/apps/parsers/test_tasks.py | 516 +++++++++++++++-- tests/apps/parsers/test_vacancy_clients.py | 30 + 43 files changed, 3723 insertions(+), 352 deletions(-) create mode 100644 src/apps/parsers/clients/common/eis_russian_trusted_ca.pem create mode 100644 src/apps/parsers/clients/fns/api.py create mode 100644 src/apps/parsers/clients/gisp/__init__.py create mode 100644 src/apps/parsers/clients/gisp/client.py create mode 100644 src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py create mode 100644 src/apps/parsers/migrations/0026_update_fns_financial_schedule.py create mode 100644 src/organizations/management/commands/create_test_companies.py create mode 100644 src/organizations/management/commands/delete_test_companies.py create mode 100644 src/organizations/test_companies.py create mode 100644 tests/apps/organizations/test_test_companies_commands.py create mode 100644 tests/apps/parsers/test_fns_api_client.py create mode 100644 tests/apps/parsers/test_gisp_client.py create mode 100644 tests/apps/parsers/test_parser_schedules.py create mode 100644 tests/apps/parsers/test_source_registry.py diff --git a/.env.prod.example b/.env.prod.example index 20622cb..0d1dedf 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -27,6 +27,9 @@ CELERY_WORKER_CONCURRENCY=2 CHECKO_API_KEY= ZAKUPKI_TOKEN= SUPERJOB_APP_ID= +HH_USER_AGENT= +# Must exceed the 200 MiB parser file limit to include multipart framing. +DATA_UPLOAD_MAX_MEMORY_SIZE=220200960 # Optional: comma-separated HTTP(S) proxies for parser tasks # Example: PARSER_PROXIES=http://user:pass@proxy1:8080,http://user:pass@proxy2:8080 PARSER_PROXIES= diff --git a/docker/Dockerfile b/docker/Dockerfile index 1837414..48db094 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -98,6 +98,7 @@ ENV PATH="/app/.venv/bin:${PATH}" \ CHECKO_API_KEY= \ ZAKUPKI_TOKEN= \ SUPERJOB_APP_ID= \ + HH_USER_AGENT= \ COLLECTSTATIC_ON_MIGRATE=0 \ BACKUP_ENCRYPTION_KEY= \ BACKUP_KEY_ID=default \ diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index 4d28f79..b02cdd4 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -34,7 +34,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.db.models import Q from django.utils import timezone -from organizations.models import Organization +from organizations.models import Organization, OrganizationSourceRecord class StateCorpExchangeError(ValueError): @@ -395,6 +395,42 @@ class StateCorpExchangeService: "ogrn": cls._digits(record.ogrn), } ) + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.INDUSTRIAL], + ): + payload = cls._record_payload(record) + certificate_number = cls._payload_lookup( + payload, ["certificate_number", "registry_number"] + ) + if not certificate_number: + continue + issue_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["issue_date_normalized", "issue_date"]) + or record.record_date, + ) + expiry_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["expiry_date_normalized", "expiry_date"]), + ) + items.append( + { + "organization_inn": cls._digits(record.inn), + "certificate_number": certificate_number, + "issue_date": issue_date.isoformat() if issue_date else None, + "expiry_date": expiry_date.isoformat() if expiry_date else None, + "certificate_file_url": cls._payload_lookup( + payload, ["certificate_file_url"] + ) + or record.url, + "organisation_name": cls._payload_lookup( + payload, ["organisation_name"] + ) + or record.registry_organization.name, + "ogrn": cls._digits(record.ogrn), + } + ) return items @classmethod @@ -405,7 +441,7 @@ class StateCorpExchangeService: queryset = ManufacturerRecord.objects.filter(inn__in=allowed_inns).order_by( "id" ) - return [ + items = [ { "organization_inn": cls._digits(record.inn), "full_legal_name": record.full_legal_name, @@ -415,6 +451,26 @@ class StateCorpExchangeService: } for record in queryset ] + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.MANUFACTURES], + ): + payload = cls._record_payload(record) + items.append( + { + "organization_inn": cls._digits(record.inn), + "full_legal_name": cls._payload_lookup( + payload, ["full_legal_name", "organisation_name"] + ) + or record.registry_organization.full_name + or record.registry_organization.name, + "inn": cls._digits(record.inn), + "ogrn": cls._digits(record.ogrn), + "address": cls._payload_lookup(payload, ["address"]) + or record.registry_organization.legal_address, + } + ) + return items @classmethod def _serialize_industrial_products( @@ -424,7 +480,7 @@ class StateCorpExchangeService: queryset = IndustrialProductRecord.objects.filter( inn__in=allowed_inns ).order_by("id") - return [ + items = [ { "organization_inn": cls._digits(record.inn), "product_name": record.product_name, @@ -437,6 +493,27 @@ class StateCorpExchangeService: } for record in queryset ] + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.INDUSTRIAL_PRODUCTS], + ): + payload = cls._record_payload(record) + product_name = cls._payload_lookup(payload, ["product_name"]) + items.append( + { + "organization_inn": cls._digits(record.inn), + "product_name": product_name or record.title, + "product_class": cls._payload_lookup( + payload, ["product_model", "regulatory_document"] + ) + or "Промышленная продукция", + "okpd2_code": cls._payload_lookup(payload, ["okpd2_code"]), + "tnved_code": cls._payload_lookup(payload, ["tnved_code"]), + "registry_number": cls._payload_lookup(payload, ["registry_number"]) + or record.external_id, + } + ) + return items @classmethod def _serialize_prosecutor_checks( @@ -463,6 +540,36 @@ class StateCorpExchangeService: "status": record.status, } ) + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.INSPECTIONS], + ): + payload = cls._record_payload(record) + start_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["start_date_normalized", "start_date"]) + or record.record_date, + ) + if start_date is None: + continue + items.append( + { + "organization_inn": cls._digits(record.inn), + "registration_number": cls._payload_lookup( + payload, ["registration_number"] + ) + or record.external_id, + "law_type": cls._payload_lookup(payload, ["legal_basis"]), + "control_authority": cls._payload_lookup( + payload, ["control_authority"] + ), + "prosecutor_office": cls._payload_lookup( + payload, ["prosecutor_office"] + ), + "start_date": start_date.isoformat(), + "status": record.status or cls._payload_lookup(payload, ["status"]), + } + ) return items @classmethod @@ -504,6 +611,7 @@ class StateCorpExchangeService: for record in cls._generic_records( allowed_inns, sources=[ + ParserLoadLog.Source.PROCUREMENTS, ParserLoadLog.Source.PROCUREMENTS_44FZ, ParserLoadLog.Source.PROCUREMENTS_223FZ, ParserLoadLog.Source.CONTRACTS, @@ -585,7 +693,7 @@ class StateCorpExchangeService: .prefetch_related("lines") .order_by("id") ) - return [ + items = [ { "organization_inn": allowed_ogrn_to_inn[report.ogrn], "external_id": report.external_id, @@ -611,6 +719,33 @@ class StateCorpExchangeService: for report in queryset if report.external_id ] + for record in cls._canonical_records( + set(allowed_ogrn_to_inn.values()), + sources=[ParserLoadLog.Source.FNS_REPORTS], + ): + if not record.external_id: + continue + payload = cls._record_payload(record) + items.append( + { + "organization_inn": cls._digits(record.inn), + "external_id": record.external_id, + "ogrn": cls._digits(record.ogrn), + "file_name": cls._payload_lookup(payload, ["file_name"]), + "file_hash": cls._payload_lookup(payload, ["file_hash"]), + "load_batch": record.load_batch, + "status": record.status, + "source": cls._payload_lookup(payload, ["source"]) or record.source, + "error_message": cls._payload_lookup(payload, ["error_message"]), + "lines": [ + cls._serialize_financial_report_line(line) + for line in record.financial_lines.all().order_by( + "year", "form_code", "line_code" + ) + ], + } + ) + return items @staticmethod def _serialize_financial_report_line( @@ -631,11 +766,10 @@ class StateCorpExchangeService: allowed_inns: set[str], ) -> list[dict[str, str]]: items: list[dict[str, str]] = [] - queryset = GenericParserRecord.objects.filter( - source=ParserLoadLog.Source.ARBITRATION, - inn__in=allowed_inns, - ).order_by("id") - for record in queryset: + for record in cls._generic_records( + allowed_inns, + sources=[ParserLoadLog.Source.ARBITRATION], + ): payload = record.payload if isinstance(record.payload, dict) else {} target = payload.get("target") if not isinstance(target, dict): @@ -843,7 +977,7 @@ class StateCorpExchangeService: return items @staticmethod - def _record_payload(record: GenericParserRecord) -> dict[str, Any]: + def _record_payload(record: Any) -> dict[str, Any]: return record.payload if isinstance(record.payload, dict) else {} @classmethod @@ -854,11 +988,35 @@ class StateCorpExchangeService: sources: list[str], ): if not allowed_inns: - return GenericParserRecord.objects.none() - return GenericParserRecord.objects.filter( - source__in=sources, - inn__in=allowed_inns, - ).order_by("id") + return [] + legacy_records = list( + GenericParserRecord.objects.filter( + source__in=sources, + inn__in=allowed_inns, + ).order_by("id") + ) + return legacy_records + list( + cls._canonical_records(allowed_inns, sources=sources) + ) + + @staticmethod + def _canonical_records( + allowed_inns: set[str], + *, + sources: list[str], + ): + """Return canonical-only records, excluding legacy-backed mirrors.""" + if not allowed_inns: + return OrganizationSourceRecord.objects.none() + return ( + OrganizationSourceRecord.objects.filter( + source__in=sources, + extension__organization__inn__in=allowed_inns, + legacy_model="", + ) + .select_related("extension__organization") + .order_by("created_at", "uid") + ) @staticmethod def _payload_lookup(payload: dict[str, Any], candidates: list[str]) -> str: diff --git a/src/apps/parsers/clients/base.py b/src/apps/parsers/clients/base.py index af399c5..d308ac1 100644 --- a/src/apps/parsers/clients/base.py +++ b/src/apps/parsers/clients/base.py @@ -39,7 +39,15 @@ class ConnectionError(HTTPClientError): class HTTPError(HTTPClientError): """HTTP ошибка (4xx, 5xx).""" - pass + def __init__( + self, + message: str, + status_code: int | None = None, + url: str | None = None, + response_data: Any | None = None, + ): + self.response_data = response_data + super().__init__(message, status_code=status_code, url=url) @dataclass @@ -204,10 +212,15 @@ class BaseHTTPClient: if not response.ok: logger.error("HTTP error %d: %s", response.status_code, url) + response_data = None + if "json" in response.headers.get("Content-Type", "").lower(): + with suppress(ValueError): + response_data = response.json() raise HTTPError( f"HTTP {response.status_code} for {url}", status_code=response.status_code, url=url, + response_data=response_data, ) logger.debug("Response %d from %s", response.status_code, url) diff --git a/src/apps/parsers/clients/checko/client.py b/src/apps/parsers/clients/checko/client.py index 05a0397..fb8710e 100644 --- a/src/apps/parsers/clients/checko/client.py +++ b/src/apps/parsers/clients/checko/client.py @@ -10,7 +10,7 @@ from collections.abc import Iterator from dataclasses import dataclass, field from typing import Any -from apps.parsers.clients.base import BaseHTTPClient +from apps.parsers.clients.base import BaseHTTPClient, HTTPError from apps.parsers.clients.checko.exceptions import ( CheckoAPIError, CheckoConnectionError, @@ -317,33 +317,26 @@ class CheckoClient: try: data = self._http_client.get_json(endpoint, params=params) + except HTTPError as e: + if isinstance(e.response_data, dict): + try: + self._raise_api_error(e.response_data, status_code=e.status_code) + except CheckoRateLimitError: + raise + except CheckoAPIError: + # Preserve the existing classification for non-quota HTTP + # errors while still recovering Checko quota metadata. + pass + logger.error("Checko HTTP request failed with status=%s", e.status_code) + raise CheckoConnectionError( + "Checko API request failed", + url=e.url, + ) from e except Exception as e: logger.error("Connection error: %s", e) raise CheckoConnectionError(f"Failed to connect to Checko API: {e}") from e - # Проверяем статус ответа - meta = data.get("meta", {}) - status = meta.get("status", "error") - - if status != "ok": - message = meta.get("message", "Unknown error") - balance = meta.get("balance") - - # Определяем тип ошибки - if "лимит" in message.lower() or "limit" in message.lower(): - raise CheckoRateLimitError( - message=message, - balance=balance, - ) - if "не найден" in message.lower() or "not found" in message.lower(): - raise CheckoNotFoundError( - message=message, - balance=balance, - ) - raise CheckoAPIError( - message=message, - balance=balance, - ) + self._raise_api_error(data) # Map Russian field names to English if "data" in data: @@ -351,6 +344,44 @@ class CheckoClient: return data + @staticmethod + def _raise_api_error( + data: dict[str, Any], + *, + status_code: int | None = None, + ) -> None: + """Классифицировать ошибку Checko из JSON независимо от HTTP статуса.""" + meta = data.get("meta", {}) + status = meta.get("status", "error") + + if status == "ok": + return + + message = str(meta.get("message", "Unknown error")) + balance = meta.get("balance") + request_count = meta.get("today_request_count") + error_kwargs = { + "message": message, + "status_code": status_code, + "balance": balance, + "request_count": request_count, + } + normalized_message = message.lower() + + if any( + marker in normalized_message + for marker in ("лимит", "limit", "квот", "quota") + ): + raise CheckoRateLimitError(**error_kwargs) + if "не найден" in normalized_message or "not found" in normalized_message: + raise CheckoNotFoundError(**error_kwargs) + raise CheckoAPIError( + message=message, + status_code=status_code, + balance=balance, + request_count=request_count, + ) + # ========================================================================= # Парсинг общих моделей # ========================================================================= diff --git a/src/apps/parsers/clients/common/eis_russian_trusted_ca.pem b/src/apps/parsers/clients/common/eis_russian_trusted_ca.pem new file mode 100644 index 0000000..bf2300c --- /dev/null +++ b/src/apps/parsers/clients/common/eis_russian_trusted_ca.pem @@ -0,0 +1,79 @@ +# Project-scoped CA bundle for HTTPS endpoints at zakupki.gov.ru. +# Sources (official AIA endpoints published by the certificate chain): +# - http://nuc-cdp.digital.gov.ru/cdp/rootca_ssl_rsa2022.crt +# - http://nuc-cdp.digital.gov.ru/cdp/subca_ssl_rsa2024.crt +# SHA-256 fingerprints checked on 2026-07-14: +# - root: D2:6D:2D:02:31:B7:C3:9F:92:CC:73:85:12:BA:54:10:35:19:E4:40:5D:68:B5:BD:70:3E:97:88:CA:8E:CF:31 +# - sub: 21:55:78:50:36:C9:00:DB:B5:F1:BB:2A:15:69:C8:0C:55:59:5B:D6:BF:94:86:7A:29:BB:DD:BC:7D:88:A3:F2 +-----BEGIN CERTIFICATE----- +MIIFwjCCA6qgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx +PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu +ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg +Q0EwHhcNMjIwMzAxMjEwNDE1WhcNMzIwMjI3MjEwNDE1WjBwMQswCQYDVQQGEwJS +VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg +YW5kIENvbW11bmljYXRpb25zMSAwHgYDVQQDDBdSdXNzaWFuIFRydXN0ZWQgUm9v +dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMfFOZ8pUAL3+r2n +qqE0Zp52selXsKGFYoG0GM5bwz1bSFtCt+AZQMhkWQheI3poZAToYJu69pHLKS6Q +XBiwBC1cvzYmUYKMYZC7jE5YhEU2bSL0mX7NaMxMDmH2/NwuOVRj8OImVa5s1F4U +zn4Kv3PFlDBjjSjXKVY9kmjUBsXQrIHeaqmUIsPIlNWUnimXS0I0abExqkbdrXbX +YwCOXhOO2pDUx3ckmJlCMUGacUTnylyQW2VsJIyIGA8V0xzdaeUXg0VZ6ZmNUr5Y +Ber/EAOLPb8NYpsAhJe2mXjMB/J9HNsoFMBFJ0lLOT/+dQvjbdRZoOT8eqJpWnVD +U+QL/qEZnz57N88OWM3rabJkRNdU/Z7x5SFIM9FrqtN8xewsiBWBI0K6XFuOBOTD +4V08o4TzJ8+Ccq5XlCUW2L48pZNCYuBDfBh7FxkB7qDgGDiaftEkZZfApRg2E+M9 +G8wkNKTPLDc4wH0FDTijhgxR3Y4PiS1HL2Zhw7bD3CbslmEGgfnnZojNkJtcLeBH +BLa52/dSwNU4WWLubaYSiAmA9IUMX1/RpfpxOxd4Ykmhz97oFbUaDJFipIggx5sX +ePAlkTdWnv+RWBxlJwMQ25oEHmRguNYf4Zr/Rxr9cS93Y+mdXIZaBEE0KS2iLRqa +OiWBki9IMQU4phqPOBAaG7A+eP8PAgMBAAGjZjBkMB0GA1UdDgQWBBTh0YHlzlpf +BKrS6badZrHF+qwshzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qwshzAS +BgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAgEAALIY1wkilt/urfEVM5vKzr6utOeDWCUczmWX/RX4ljpRdgF+5fAIS4vH +tmXkqpSCOVeWUrJV9QvZn6L227ZwuE15cWi8DCDal3Ue90WgAJJZMfTshN4OI8cq +W9E4EG9wglbEtMnObHlms8F3CHmrw3k6KmUkWGoa+/ENmcVl68u/cMRl1JbW2bM+ +/3A+SAg2c6iPDlehczKx2oa95QW0SkPPWGuNA/CE8CpyANIhu9XFrj3RQ3EqeRcS +AQQod1RNuHpfETLU/A2gMmvn/w/sx7TB3W5BPs6rprOA37tutPq9u6FTZOcG1Oqj +C/B7yTqgI7rbyvox7DEXoX7rIiEqyNNUguTk/u3SZ4VXE2kmxdmSh3TQvybfbnXV +4JbCZVaqiZraqc7oZMnRoWrXRG3ztbnbes/9qhRGI7PqXqeKJBztxRTEVj8ONs1d +WN5szTwaPIvhkhO3CO5ErU2rVdUr89wKpNXbBODFKRtgxUT70YpmJ46VVaqdAhOZ +D9EUUn4YaeLaS8AjSF/h7UkjOibNc4qVDiPP+rkehFWM66PVnP1Msh93tc+taIfC +EYVMxjh8zNbFuoc7fzvvrFILLe7ifvEIUqSVIC/AzplM/Jxw7buXFeGP1qVCBEHq +391d/9RAfaZ12zkwFsl+IKwE/OZxW8AHa9i1p4GO0YSNuczzEm4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIG6DCCBNCgAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx +PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu +ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg +Q0EwHhcNMjQwNzE1MTI1MDQxWhcNMjkwNzE5MTI1MDQxWjBvMQswCQYDVQQGEwJS +VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg +YW5kIENvbW11bmljYXRpb25zMR8wHQYDVQQDDBZSdXNzaWFuIFRydXN0ZWQgU3Vi +IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1j0rkZECOt1S8o7I +JY+4YKAxuEa5xaHKHXT2EpkuC/0krqMOjUy2oPIRNgR5g8X0Jl6jamxeGLc4Q1tf +ju6or9oSRYThIUhRsFDQNBiBBEXoBgWxTfiKB2eyT97+pz5TBtBiRCPaLGRHYLRb +9Jz2HkJlxbtNPjtDrF5DPHym+mZ1M1z3hIQYAqJwLpsEBnsw/VxWMlxqHoeewd0h +uJMd71KQ5vOKlz7KrIZ6EobNNa6wItuvsfj3kYCK7O78uLHGXXFxdr8Hae9lMUmC +8F7AFwa+bO1LRlTlqW7rE3rLf+jj70N01N8T3o22v14YBaFBWQWncAVYD2JuL3tH +252+kdNOERf1fLbLRigJAbd+hOhWYlNf963TFDgnNPliHNIW72SygVBnI2V3JwO1 +dp1hVKpK/zt8ziGdHW4gmOLTsH50YKdR4jNqUgQv4wASlKn9OpN6zHYc5G8h86fY +BM+zxE5ikGI+I/vIqBuI0eaDU92AWN/YjFLpu8tMu9kLRSCf1vug6FIfDPWVo7iP +ac/SI2v8jnnpaW7ph/Pz3WkzaG7ZZJsfFs+8dploWc6LOoDtbFBhMdGMxu024msC +0PSjZb5ODXPIaO2NsA7fMiAtZcoK6anTUJh4zOP/stA9qsJGNxdrEmiPXSmBZY/N +Y0wkZgZ6JTDhw7038bPvctkblJkCAwEAAaOCAYswggGHMB0GA1UdDgQWBBR3Pdk5 +r0K93FvKduru/c4+YSkwXzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qws +hzAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADCBmAYIKwYBBQUH +AQEEgYswgYgwQAYIKwYBBQUHMAKGNGh0dHA6Ly9udWMtY2RwLnZvc2tob2QucnUv +Y2RwL3Jvb3RjYV9zc2xfcnNhMjAyMi5jcnQwRAYIKwYBBQUHMAKGOGh0dHA6Ly9u +dWMtY2RwLmRpZ2l0YWwuZ292LnJ1L2NkcC9yb290Y2Ffc3NsX3JzYTIwMjIuY3J0 +MIGFBgNVHR8EfjB8MDqgOKA2hjRodHRwOi8vbnVjLWNkcC52b3NraG9kLnJ1L2Nk +cC9yb290Y2Ffc3NsX3JzYTIwMjIuY3JsMD6gPKA6hjhodHRwOi8vbnVjLWNkcC5k +aWdpdGFsLmdvdi5ydS9jZHAvcm9vdGNhX3NzbF9yc2EyMDIyLmNybDANBgkqhkiG +9w0BAQsFAAOCAgEAmsINXtQ7wwUWvIeOr80MdJS/5G4xhyZOVEmeUorThquT672y +cCg3XCxc4fwbiZqSSbBqntQ7RtiTAKMYMvBageKoVHbzz+R4jX01tKcTx8cDePrz +dJ73bLNUorE7RU9QsW4KyiUeRmjMDV23AUlEvuQFTwgkHXvbac1BBdPn9CrssQuF +5EGohZKcQPFiAAc4SHbRNhlr7uAwgpc/erzI9EAcvA6BVAXcVKoeGpV01uexUgZ6 +St5RP9UmDWNA7T4yVXWJ233N0Q8bl+6AswINQ3PosPu6yQQHQjr65YS06epK+AeI +6j+oGR4xI7EhTQhQvaobnGmX/8QQ7XDRYCP2HXYxiffnn/CfZ/BVyKLYeY1ZipjE +nzqdQIC2+Q3WtY8jsVRQMP38WFRmtsIt5snehnPTs5bKGVIcYzj3o3Ex/K7agEz0 +zAJ0JR5ivXZOvNkT0g9x1v+S1IkU3e/nX1a+tpRquMtnHX0L2lXArNHUbaOO9EJt +d57WaIpofV5cVhhwShOgAuBc9UMJF3/n4t4RKiPxtsK8P67gcmphMhslj7AMYrYM +ej2NvQZY4m3ub3CPC/PrTjDONvb+8g5xrKtxBjYqC74HSB4dg9G3WimSDUuP2Su6 +G2y2TUeyJuCvCLz289VoO0vg7cNdMobE3KCqAiiNhN2VBFxHAUKmUoRcRdw= +-----END CERTIFICATE----- diff --git a/src/apps/parsers/clients/common/structured.py b/src/apps/parsers/clients/common/structured.py index f53f4ea..d0486e4 100644 --- a/src/apps/parsers/clients/common/structured.py +++ b/src/apps/parsers/clients/common/structured.py @@ -10,8 +10,9 @@ import zipfile from collections import Counter from dataclasses import dataclass, field from decimal import Decimal, InvalidOperation +from pathlib import Path from typing import Any -from urllib.parse import urljoin +from urllib.parse import parse_qs, urljoin, urlsplit from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError from apps.parsers.clients.common.schemas import GenericParserItem @@ -55,15 +56,19 @@ EIS_CARD_SOURCES = { "contracts", "unfair_suppliers", } +EIS_CA_BUNDLE_PATH = Path(__file__).with_name("eis_russian_trusted_ca.pem") ZAKUPKI_BASE_URL = "https://zakupki.gov.ru" +ZAKUPKI_HOST = "zakupki.gov.ru" ZAKUPKI_DETAIL_MAX_FILE_SIZE_BYTES = 2 * 1024 * 1024 ZAKUPKI_DETAIL_URL_MARKERS = ( + "/contract/contractCard/common-info", + "/223/purchase/public/purchase/info/common-info", "/view/common-info", "/card/common-info", "/contractCard/common-info", - "/notice/", "/orderclause/card/", "/contract/contractCard/", + "/notice/", ) INN_RE = re.compile(r"(? BaseHTTPClient: """Ленивая инициализация HTTP клиента.""" if self._http_client is None: + verify_ssl = self.verify_ssl + if verify_ssl is True and self.source in EIS_CARD_SOURCES: + verify_ssl = str(EIS_CA_BUNDLE_PATH) self._http_client = BaseHTTPClient( base_url="", proxies=self.proxies, timeout=self.timeout, - verify_ssl=self.verify_ssl, + verify_ssl=verify_ssl, ) return self._http_client @@ -179,7 +187,6 @@ class StructuredDataClient: "skip": 0, "take": GISP_PRODUCTS_PAGE_SIZE, "requireTotalCount": True, - "sort": [{"selector": "res_date", "desc": True}], } }, ) @@ -481,6 +488,9 @@ class StructuredDataClient: "Номер реестровой записи в ЕРУЗ", "Начальная цена", "Цена контракта", + "Заключение контракта", + "Размещен контракт в реестре контрактов", + "Обновлен контракт в реестре контрактов", "Размещено", "Обновлено", "Окончание подачи заявок", @@ -504,6 +514,9 @@ class StructuredDataClient: if detail_url: row["url"] = detail_url row["detail_url"] = detail_url + identity_url = self._select_zakupki_identity_url(card) + if identity_url: + row["identity_url"] = identity_url if lines[0].endswith("-ФЗ"): row["law"] = lines[0] if card_index < self.max_eis_detail_pages: @@ -520,7 +533,9 @@ class StructuredDataClient: href = str(link.get("href") or "").strip() if not href or href.startswith(("#", "javascript:")): continue - urls.append(urljoin(ZAKUPKI_BASE_URL, href)) + url = self._resolve_zakupki_url(href) + if url: + urls.append(url) for marker in ZAKUPKI_DETAIL_URL_MARKERS: for url in urls: @@ -528,6 +543,27 @@ class StructuredDataClient: return url return urls[0] if urls else "" + def _select_zakupki_identity_url(self, card: Any) -> str: + """Выбрать страницу организации с непустыми реквизитами заказчика.""" + for link in card.find_all("a", href=True): + href = str(link.get("href") or "").strip() + url = self._resolve_zakupki_url(href) + if not url or "/epz/organization/" not in urlsplit(url).path: + continue + query = parse_qs(urlsplit(url).query) + if any(query.get(key) for key in ("organizationCode", "inn", "ogrn")): + return url + return "" + + @staticmethod + def _resolve_zakupki_url(href: str) -> str: + """Разрешить только HTTPS-ссылки официального хоста ЕИС.""" + url = urljoin(ZAKUPKI_BASE_URL, href) + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.hostname != ZAKUPKI_HOST: + return "" + return url + def _enrich_zakupki_card_from_detail(self, row: dict[str, Any]) -> None: """Дозагрузить ИНН/КПП/ОГРН заказчика со страницы карточки ЕИС.""" if not self.enrich_eis_detail_pages: @@ -535,7 +571,9 @@ class StructuredDataClient: if row.get("inn") and row.get("ogrn"): return - detail_url = str(row.get("detail_url") or row.get("url") or "").strip() + detail_url = str( + row.get("identity_url") or row.get("detail_url") or row.get("url") or "" + ).strip() if not detail_url: return @@ -608,12 +646,14 @@ class StructuredDataClient: labels: tuple[str, ...], ) -> str: """Найти текстовое значение рядом с подписью на detail page.""" - for index, line in enumerate(lines[:-1]): - if not self._line_contains_label(line, labels): - continue - value = lines[index + 1].strip() - if value and not self._looks_like_detail_label(value): - return value + for label in labels: + normalized_label = self._normalize_detail_label(label) + for index, line in enumerate(lines[:-1]): + if self._normalize_detail_label(line) != normalized_label: + continue + value = lines[index + 1].strip() + if value and not self._looks_like_detail_label(value): + return value return "" @staticmethod @@ -763,7 +803,7 @@ class StructuredDataClient: ) -> None: """Заполнить пары label -> следующая строка из карточки.""" for index, line in enumerate(lines[:-1]): - if line in labels: + if line in labels and lines[index + 1] not in labels: row[line] = lines[index + 1] def _fill_zakupki_status( @@ -880,6 +920,9 @@ class StructuredDataClient: "объекты закупки", "наименование документа", "наименование закупки", + "наименование фио недобросовестного поставщика", + "полное наименование лица", + "фирменное наименование лица", "предмет", "описание", "должность", @@ -901,6 +944,8 @@ class StructuredDataClient: "дата внесения в реестр", "дата вступления постановления", "размещено", + "размещен контракт в реестре контрактов", + "заключение контракта", "обновлено", "включено", "утверждение", diff --git a/src/apps/parsers/clients/fns/__init__.py b/src/apps/parsers/clients/fns/__init__.py index 7383ac3..af952d0 100644 --- a/src/apps/parsers/clients/fns/__init__.py +++ b/src/apps/parsers/clients/fns/__init__.py @@ -1,10 +1,14 @@ -""" -Парсер бухгалтерской отчетности ФНС. - -Обрабатывает Excel файлы формата fin_{external_id}_{ogrn}.xlsx. -""" +"""Клиенты бухгалтерской отчётности ФНС: ГИР БО API и Excel.""" +from apps.parsers.clients.fns.api import FNSApiClient, FNSApiClientError, FNSApiReport from apps.parsers.clients.fns.parser import FNSExcelParser from apps.parsers.clients.fns.schemas import ParsedReport, ReportLine -__all__ = ["FNSExcelParser", "ParsedReport", "ReportLine"] +__all__ = [ + "FNSApiClient", + "FNSApiClientError", + "FNSApiReport", + "FNSExcelParser", + "ParsedReport", + "ReportLine", +] diff --git a/src/apps/parsers/clients/fns/api.py b/src/apps/parsers/clients/fns/api.py new file mode 100644 index 0000000..3779944 --- /dev/null +++ b/src/apps/parsers/clients/fns/api.py @@ -0,0 +1,303 @@ +"""Клиент публичного API ГИР БО ФНС для точечного получения отчётности.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import asdict, dataclass, field +from decimal import Decimal, InvalidOperation +from typing import Any + +from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError +from apps.parsers.clients.fns.schemas import ReportLine +from requests.adapters import BaseAdapter + +DEFAULT_BASE_URL = "https://bo.nalog.gov.ru" +DEFAULT_MAX_PERIODS = 5 +SEARCH_ENDPOINT = "/advanced-search/organizations/search" +FORM_CODES = { + "balance": "1", + "financialResult": "2", + "capitalChange": "3", + "fundsMovement": "4", + "targetedFundsUsing": "6", +} +FORM_NAMES = { + "1": "Бухгалтерский баланс", + "2": "Отчёт о финансовых результатах", + "3": "Отчёт об изменениях капитала", + "4": "Отчёт о движении денежных средств", + "6": "Отчёт о целевом использовании средств", +} +VALUE_KEY_RE = re.compile(r"^(current|previous|beforePrevious)(\d+)$") +INN_RE = re.compile(r"^(?:\d{10}|\d{12})$") + + +class FNSApiClientError(HTTPClientError): + """Ошибка клиента публичного API ГИР БО.""" + + +@dataclass(frozen=True) +class FNSApiReport: + """Один опубликованный вариант бухгалтерской отчётности ГИР БО.""" + + organization_id: int + external_id: str + inn: str + ogrn: str + organization_name: str + year: int + period_type: int | None + actual_date: str + lines: list[ReportLine] + raw_payload: dict[str, Any] + + @property + def file_name(self) -> str: + """Стабильное логическое имя API payload для существующей модели.""" + safe_external_id = self.external_id.replace(":", "_") + return f"girbo_{self.organization_id}_{self.year}_{safe_external_id}.json" + + @property + def file_hash(self) -> str: + """Детерминированный хеш API payload для идемпотентной загрузки.""" + encoded = json.dumps( + self.raw_payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def to_save_report_kwargs(self, *, batch_id: int) -> dict[str, Any]: + """Подготовить аргументы для ``FNSReportService.save_report``.""" + return { + "external_id": self.external_id, + "ogrn": self.ogrn, + "file_name": self.file_name, + "file_hash": self.file_hash, + "source": "api", + "batch_id": batch_id, + "lines_data": [asdict(line) for line in self.lines], + } + + +@dataclass +class FNSApiClient: + """Получает ГИР БО только для явно заданной организации или ИНН.""" + + base_url: str = DEFAULT_BASE_URL + max_periods: int = DEFAULT_MAX_PERIODS + proxies: list[str] | None = None + timeout: int = 60 + http_adapter: BaseAdapter | None = None + _http_client: BaseHTTPClient | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + if self.max_periods < 1: + raise ValueError("max_periods must be positive") + + @property + def http_client(self) -> BaseHTTPClient: + """Ленивая browser-like HTTP-сессия, обязательная для публичного API.""" + if self._http_client is None: + self._http_client = BaseHTTPClient( + base_url=self.base_url, + proxies=self.proxies, + timeout=self.timeout, + adapter=self.http_adapter, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + "Accept": "application/json, text/plain, */*", + "Referer": f"{self.base_url.rstrip('/')}/", + }, + ) + return self._http_client + + def fetch_reports_by_inn(self, inn: str) -> list[FNSApiReport]: + """Найти точное совпадение по ИНН и получить его отчётность.""" + organization = self.find_organization_by_inn(inn) + if organization is None: + raise FNSApiClientError(f"FNS organization not found for INN {inn}") + return self.fetch_reports_by_organization_id(organization["id"]) + + def find_organization_by_inn(self, inn: str) -> dict[str, Any] | None: + """Вернуть точное совпадение из поиска, не сканируя общую выдачу.""" + normalized_inn = self._normalize_inn(inn) + data = self.http_client.get_json( + SEARCH_ENDPOINT, + params={"query": normalized_inn, "page": 0, "size": 10}, + ) + if not isinstance(data, dict) or not isinstance(data.get("content"), list): + raise FNSApiClientError("FNS search API returned invalid content") + for item in data["content"]: + if not isinstance(item, dict): + continue + candidate_inn = re.sub(r"\D", "", str(item.get("inn") or "")) + if candidate_inn == normalized_inn and self._organization_id(item): + return item + return None + + def fetch_reports_by_organization_id( + self, organization_id: int | str + ) -> list[FNSApiReport]: + """Получить опубликованные периоды для конкретного ID организации.""" + parsed_id = self._normalize_organization_id(organization_id) + organization = self.http_client.get_json(f"/nbo/organizations/{parsed_id}") + bfo = self.http_client.get_json(f"/nbo/organizations/{parsed_id}/bfo/") + if not isinstance(organization, dict): + raise FNSApiClientError("FNS organization API returned invalid payload") + if not isinstance(bfo, list): + raise FNSApiClientError("FNS BFO API returned invalid payload") + return self._map_reports( + organization_id=parsed_id, + organization=organization, + bfo=bfo[: self.max_periods], + ) + + def _map_reports( + self, + *, + organization_id: int, + organization: dict[str, Any], + bfo: list[Any], + ) -> list[FNSApiReport]: + reports: list[FNSApiReport] = [] + inn = re.sub(r"\D", "", str(organization.get("inn") or "")) + ogrn = re.sub(r"\D", "", str(organization.get("ogrn") or "")) + organization_name = str( + organization.get("fullName") or organization.get("shortName") or "" + ).strip() + + for period_payload in bfo: + if not isinstance(period_payload, dict): + continue + year = self._read_year(period_payload.get("period")) + type_corrections = period_payload.get("typeCorrections") + if year is None or not isinstance(type_corrections, list): + continue + for typed_correction in type_corrections: + if not isinstance(typed_correction, dict): + continue + correction = typed_correction.get("correction") + if not isinstance(correction, dict): + continue + correction_id = correction.get("id") + if correction_id in (None, ""): + continue + reports.append( + FNSApiReport( + organization_id=organization_id, + external_id=f"girbo:{correction_id}", + inn=inn, + ogrn=ogrn, + organization_name=organization_name, + year=year, + period_type=self._optional_int(typed_correction.get("type")), + actual_date=str( + period_payload.get("actualBfoDate") + or correction.get("datePresent") + or "" + ), + lines=self._map_lines(correction, year=year), + raw_payload=correction, + ) + ) + return reports + + @classmethod + def _map_lines(cls, correction: dict[str, Any], *, year: int) -> list[ReportLine]: + lines: list[ReportLine] = [] + for report_key, form_code in FORM_CODES.items(): + form = correction.get(report_key) + if not isinstance(form, dict): + continue + values_by_code: dict[str, dict[str, int | None]] = {} + for key, value in form.items(): + match = VALUE_KEY_RE.fullmatch(str(key)) + if match is None: + continue + value_kind, line_code = match.groups() + values_by_code.setdefault(line_code, {})[ + value_kind + ] = cls._optional_int(value) + + for line_code in sorted(values_by_code): + values = values_by_code[line_code] + period_start = values.get("previous") + period_end = values.get("current") + if period_start is None and period_end is None: + continue + lines.append( + ReportLine( + form_code=form_code, + line_code=line_code, + line_name=f"{FORM_NAMES[form_code]}, строка {line_code}", + year=year, + period_start=period_start, + period_end=period_end, + ) + ) + return lines + + @staticmethod + def _normalize_inn(value: str) -> str: + inn = re.sub(r"\D", "", str(value)) + if not INN_RE.fullmatch(inn): + raise ValueError("INN must contain 10 or 12 digits") + return inn + + @staticmethod + def _normalize_organization_id(value: int | str) -> int: + try: + organization_id = int(value) + except (TypeError, ValueError) as exc: + raise ValueError("organization_id must be a positive integer") from exc + if organization_id <= 0: + raise ValueError("organization_id must be a positive integer") + return organization_id + + @staticmethod + def _organization_id(item: dict[str, Any]) -> int | None: + raw_value = item.get("id") + if raw_value is None: + return None + try: + value = int(raw_value) + except (TypeError, ValueError): + return None + return value if value > 0 else None + + @staticmethod + def _read_year(value: Any) -> int | None: + try: + year = int(value) + except (TypeError, ValueError): + return None + return year if 1990 <= year <= 2100 else None + + @staticmethod + def _optional_int(value: Any) -> int | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return int(Decimal(str(value))) + except (InvalidOperation, TypeError, ValueError): + return None + + def close(self) -> None: + """Закрыть HTTP-сессию.""" + if self._http_client is not None: + self._http_client.close() + self._http_client = None + + def __enter__(self) -> FNSApiClient: + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/src/apps/parsers/clients/gisp/__init__.py b/src/apps/parsers/clients/gisp/__init__.py new file mode 100644 index 0000000..5490eb6 --- /dev/null +++ b/src/apps/parsers/clients/gisp/__init__.py @@ -0,0 +1,13 @@ +"""Клиент официального реестра промышленной продукции ГИСП.""" + +from apps.parsers.clients.gisp.client import ( + GispProductsClient, + GispProductsClientError, + GispProductsResult, +) + +__all__ = [ + "GispProductsClient", + "GispProductsClientError", + "GispProductsResult", +] diff --git a/src/apps/parsers/clients/gisp/client.py b/src/apps/parsers/clients/gisp/client.py new file mode 100644 index 0000000..3b5da88 --- /dev/null +++ b/src/apps/parsers/clients/gisp/client.py @@ -0,0 +1,226 @@ +"""Клиент публичного UI API реестра промышленной продукции ГИСП.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError +from apps.parsers.clients.minpromtorg.schemas import IndustrialProduct +from requests.adapters import BaseAdapter + +DEFAULT_BASE_URL = "https://gisp.gov.ru" +PRODUCTS_ENDPOINT = "/pp719v2/pub/prod/b/" +DEFAULT_PAGE_SIZE = 100 +DEFAULT_MAX_PAGES = 10 +DEFAULT_MAX_RECORDS = 1_000 +MAX_PAGE_SIZE = 100 +MAX_PAGES = 100 +MAX_RECORDS = 10_000 + + +class GispProductsClientError(HTTPClientError): + """Ошибка клиента реестра продукции ГИСП.""" + + +@dataclass(frozen=True) +class GispProductsResult: + """Ограниченный результат последовательной загрузки страниц ГИСП.""" + + products: list[IndustrialProduct] + total_count: int | None + pages_fetched: int + items_fetched: int + truncated: bool + + +@dataclass +class GispProductsClient: + """Загружает продукцию ГИСП с явными ограничителями объёма.""" + + base_url: str = DEFAULT_BASE_URL + page_size: int = DEFAULT_PAGE_SIZE + max_pages: int = DEFAULT_MAX_PAGES + max_records: int = DEFAULT_MAX_RECORDS + proxies: list[str] | None = None + timeout: int = 120 + http_adapter: BaseAdapter | None = None + _http_client: BaseHTTPClient | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + if not 1 <= self.page_size <= MAX_PAGE_SIZE: + raise ValueError(f"page_size must be between 1 and {MAX_PAGE_SIZE}") + if not 1 <= self.max_pages <= MAX_PAGES: + raise ValueError(f"max_pages must be between 1 and {MAX_PAGES}") + if not 1 <= self.max_records <= MAX_RECORDS: + raise ValueError(f"max_records must be between 1 and {MAX_RECORDS}") + + @property + def http_client(self) -> BaseHTTPClient: + """Ленивая инициализация HTTP клиента.""" + if self._http_client is None: + self._http_client = BaseHTTPClient( + base_url=self.base_url, + proxies=self.proxies, + timeout=self.timeout, + adapter=self.http_adapter, + headers={ + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json", + "Referer": f"{self.base_url.rstrip('/')}/pp719v2/pub/prod/", + }, + ) + return self._http_client + + def fetch_products(self) -> GispProductsResult: + """Последовательно загрузить страницы, не превышая заданные лимиты.""" + products: list[IndustrialProduct] = [] + total_count: int | None = None + pages_fetched = 0 + items_fetched = 0 + offset = 0 + source_exhausted = False + + for _page in range(self.max_pages): + remaining = self.max_records - items_fetched + if remaining <= 0: + break + take = min(self.page_size, remaining) + items, total_count = self._fetch_page( + offset=offset, + take=take, + previous_total_count=total_count, + ) + pages_fetched += 1 + items_fetched += len(items) + offset += len(items) + + for item in items: + if not isinstance(item, dict): + continue + product = self._map_product(item) + if product is not None: + products.append(product) + + if self._source_exhausted( + items_count=len(items), + requested_count=take, + offset=offset, + total_count=total_count, + ): + source_exhausted = True + break + + truncated = not source_exhausted and ( + total_count is None or items_fetched < total_count + ) + return GispProductsResult( + products=products, + total_count=total_count, + pages_fetched=pages_fetched, + items_fetched=items_fetched, + truncated=truncated, + ) + + def _fetch_page( + self, + *, + offset: int, + take: int, + previous_total_count: int | None, + ) -> tuple[list[Any], int | None]: + data = self.http_client.post_json( + PRODUCTS_ENDPOINT, + payload={ + "opt": { + "skip": offset, + "take": take, + "requireTotalCount": True, + } + }, + ) + if data.get("ok") is False: + raise GispProductsClientError("GISP products API returned ok=false") + items = data.get("items") + if not isinstance(items, list): + raise GispProductsClientError("GISP products API returned no items list") + total_count = self._read_total_count(data, previous=previous_total_count) + return items, total_count + + @staticmethod + def _source_exhausted( + *, + items_count: int, + requested_count: int, + offset: int, + total_count: int | None, + ) -> bool: + if total_count is not None: + if offset >= total_count: + return True + if items_count == 0: + raise GispProductsClientError( + "GISP products API returned an empty page before total_count" + ) + return False + return items_count < requested_count + + @staticmethod + def _read_total_count(data: dict[str, Any], *, previous: int | None) -> int | None: + value = data.get("total_count", data.get("totalCount")) + if value is None: + return previous + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise GispProductsClientError( + "GISP products API returned invalid total_count" + ) from exc + if parsed < 0: + raise GispProductsClientError( + "GISP products API returned negative total_count" + ) + return parsed + + @classmethod + def _map_product(cls, item: dict[str, Any]) -> IndustrialProduct | None: + registry_number = cls._text( + item.get("_product_reg_number_2023") or item.get("_product_reg_number_2022") + ) + product_name = cls._text(item.get("_product_name")) + if not registry_number or not product_name: + return None + + document_parts = [ + cls._text(item.get("_basedondoc_name")), + cls._text(item.get("_basedondoc_num")), + cls._text(item.get("_basedondoc_date")), + ] + regulatory_document = " | ".join(part for part in document_parts if part) + return IndustrialProduct( + full_organisation_name=cls._text(item.get("_org_name")), + inn=cls._text(item.get("_org_inn")), + ogrn=cls._text(item.get("_org_ogrn")), + registry_number=registry_number, + product_name=product_name, + product_model=cls._text(item.get("_product_spec")), + okpd2_code=cls._text(item.get("_product_okpd2")), + tnved_code=cls._text(item.get("_product_tnved")), + regulatory_document=regulatory_document, + ) + + @staticmethod + def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + def close(self) -> None: + """Закрыть HTTP-сессию.""" + if self._http_client is not None: + self._http_client.close() + self._http_client = None + + def __enter__(self) -> GispProductsClient: + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/src/apps/parsers/clients/proverki/client.py b/src/apps/parsers/clients/proverki/client.py index 195e3f2..bfc0df7 100644 --- a/src/apps/parsers/clients/proverki/client.py +++ b/src/apps/parsers/clients/proverki/client.py @@ -301,8 +301,10 @@ class ProverkiClient: if self.use_playwright: if progress_callback: progress_callback(20, "Навигация по порталу...") - content = self._download_from_portal(file_url, progress_callback) - self._close_playwright() + try: + content = self._download_from_portal(file_url, progress_callback) + finally: + self._close_playwright() else: if progress_callback: progress_callback(20, f"Скачивание {file_url}...") @@ -330,10 +332,12 @@ class ProverkiClient: progress_callback( 22, "Сервер требует JavaScript, запускаем браузер..." ) - content = self._download_with_playwright( - file_url, progress_callback - ) - self._close_playwright() + try: + content = self._download_with_playwright( + file_url, progress_callback + ) + finally: + self._close_playwright() else: raise ProverkiClientError( "Сервер вернул HTML вместо данных. " @@ -789,7 +793,7 @@ class ProverkiClient: Адаптируется к различным форматам XML proverki.gov.ru: - Данные могут быть в атрибутах элементов - - Данные могут быть во вложенных элементах (I_SUBJECT, I_AUTHORITY) + - Данные могут быть во вложенных элементах старой и текущей схемы """ try: # Определяем namespace элемента @@ -810,6 +814,10 @@ class ProverkiClient: i_authority = find_child("I_AUTHORITY") # Контролирующий орган i_approve = find_child("I_APPROVE") # Информация об утверждении i_classification = find_child("I_CLASSIFICATION") # Классификация + subject = find_child("SUBJECT") + kno_organization = find_child("KNO_ORGANIZATION") + kind_control = find_child("KIND_CONTROL") + kind_knm = find_child("KIND_KNM") def get_attr_value(attr_names: list[str]) -> str: # noqa: C901 """Найти значение атрибута в элементе или вложенных элементах.""" @@ -824,6 +832,12 @@ class ProverkiClient: if name in i_subject.attrib: return i_subject.attrib[name].strip() + # Ищем в SUBJECT текущей схемы ФЗ-248 (NAME, INN, OGRN) + if subject is not None: + for name in attr_names: + if name in subject.attrib: + return subject.attrib[name].strip() + # Ищем в I_AUTHORITY (FRGU_ORG_NAME) if i_authority is not None: for name in attr_names: @@ -853,6 +867,12 @@ class ProverkiClient: return "" + def get_value_attr(child: ET.Element | None) -> str: + """Прочитать VALUE у справочного элемента текущей схемы.""" + if child is None: + return "" + return child.attrib.get("VALUE", "").strip() + # Маппинг атрибутов на поля Inspection # Используем названия из реального XML proverki.gov.ru # Включая русские теги (КНМ формат) @@ -876,6 +896,7 @@ class ProverkiClient: "ORG_NAME", "FULL_NAME", "SHORT_NAME", + "NAME", "I_NAME", "organisation_name", "org_name", @@ -893,7 +914,7 @@ class ProverkiClient: "authority", "КонтрольныйОрган", ] - ) + ) or get_value_attr(kno_organization) inspection_type = get_attr_value( [ "ITYPE_NAME", @@ -903,7 +924,7 @@ class ProverkiClient: "type", "ТипПроверки", ] - ) + ) or get_value_attr(kind_control) inspection_form = get_attr_value( [ "ICARRYOUT_TYPE_NAME", @@ -913,7 +934,7 @@ class ProverkiClient: "form", "ФормаПроверки", ] - ) + ) or get_value_attr(kind_knm) start_date = get_attr_value( [ "START_DATE", @@ -928,6 +949,7 @@ class ProverkiClient: end_date = get_attr_value( [ "END_DATE", + "STOP_DATE", "I_DATE_END", "DATE_END", "end_date", @@ -1032,6 +1054,10 @@ class ProverkiClient: "playwright install chromium" ) from e except Exception as e: + # sync_playwright().start() owns an asyncio runtime. If the + # browser launch fails, leaving it active makes subsequent + # synchronous Django ORM calls look like async-context calls. + self._close_playwright() raise ProverkiClientError( f"Не удалось запустить Playwright browser: {e}" ) from e diff --git a/src/apps/parsers/clients/trudvsem/client.py b/src/apps/parsers/clients/trudvsem/client.py index ae940e4..3dc6951 100644 --- a/src/apps/parsers/clients/trudvsem/client.py +++ b/src/apps/parsers/clients/trudvsem/client.py @@ -14,7 +14,7 @@ from requests.adapters import BaseAdapter logger = logging.getLogger(__name__) -DEFAULT_BASE_URL = "http://opendata.trudvsem.ru/api/v1" +DEFAULT_BASE_URL = "https://opendata.trudvsem.ru/api/v1" VACANCIES_ENDPOINT = "/vacancies" COMPANY_INN_VACANCIES_ENDPOINT = "/vacancies/company/inn/{inn}" diff --git a/src/apps/parsers/clients/vacancies.py b/src/apps/parsers/clients/vacancies.py index 1f5f330..c9da5b8 100644 --- a/src/apps/parsers/clients/vacancies.py +++ b/src/apps/parsers/clients/vacancies.py @@ -87,6 +87,7 @@ def _stable_external_id(source: str, payload: Mapping[str, Any]) -> str: class HHVacanciesClient: """Клиент публичного API HeadHunter.""" + user_agent: str = "" proxies: list[str] | None = None base_url: str = HH_BASE_URL timeout: int = 120 @@ -97,12 +98,20 @@ class HHVacanciesClient: @property def http_client(self) -> BaseHTTPClient: if self._http_client is None: + user_agent = self.user_agent.strip() + if not user_agent: + raise VacanciesClientError( + "HH_USER_AGENT is required; use 'App/Version (contact email)'" + ) self._http_client = BaseHTTPClient( base_url=self.base_url, proxies=self.proxies, timeout=self.timeout, adapter=self.http_adapter, - headers={"Accept": "application/json"}, + headers={ + "Accept": "application/json", + "HH-User-Agent": user_agent, + }, ) return self._http_client @@ -270,6 +279,7 @@ class VacanciesClient: proxies: list[str] | None = None superjob_app_id: str = "" + hh_user_agent: str = "" sources: list[str] | None = None source_clients: dict[str, VacancyProvider] | None = None _source_clients_cache: dict[str, VacancyProvider] | None = field( @@ -287,7 +297,10 @@ class VacanciesClient: clients: dict[str, VacancyProvider] = { TRUDVSEM_SOURCE: TrudvsemClient(proxies=self.proxies), - HH_SOURCE: HHVacanciesClient(proxies=self.proxies), + HH_SOURCE: HHVacanciesClient( + user_agent=self.hh_user_agent, + proxies=self.proxies, + ), } if self.superjob_app_id: clients[SUPERJOB_SOURCE] = SuperJobVacanciesClient( diff --git a/src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py b/src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py new file mode 100644 index 0000000..5975a8f --- /dev/null +++ b/src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py @@ -0,0 +1,63 @@ +import json + +from django.db import migrations + +CANONICAL_SCHEDULE_NAME = "parser:sync_inspections:weekly-saturday-msk" +CANONICAL_TASK_NAME = "apps.parsers.tasks.sync_inspections" +LEGACY_SCHEDULE_NAMES = ( + "parser:inspections:weekly-saturday-msk", + "parse-inspections-weekly", +) +WEEKLY_MSK_CRON = { + "minute": "0", + "hour": "0", + "day_of_week": "6", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "Europe/Moscow", +} + + +def consolidate_inspections_schedule(apps, schema_editor): + CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + PeriodicTask.objects.filter(name__in=LEGACY_SCHEDULE_NAMES).delete() + + canonical = PeriodicTask.objects.filter(name=CANONICAL_SCHEDULE_NAME).first() + if canonical is not None: + canonical.task = CANONICAL_TASK_NAME + canonical.args = json.dumps([]) + canonical.kwargs = json.dumps({}) + canonical.save(update_fields=["task", "args", "kwargs"]) + return + + crontab, _ = CrontabSchedule.objects.get_or_create(**WEEKLY_MSK_CRON) + field_names = {field.name for field in PeriodicTask._meta.fields} + schedule_fields = {"crontab": crontab} + for field_name in ("interval", "solar", "clocked"): + if field_name in field_names: + schedule_fields[field_name] = None + + PeriodicTask.objects.create( + name=CANONICAL_SCHEDULE_NAME, + task=CANONICAL_TASK_NAME, + args=json.dumps([]), + kwargs=json.dumps({}), + enabled=True, + description="Default inspections sync: weekly on Saturday 00:00 MSK.", + **schedule_fields, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0024_auto_20260607_1017"), + ] + + operations = [ + migrations.RunPython( + consolidate_inspections_schedule, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/src/apps/parsers/migrations/0026_update_fns_financial_schedule.py b/src/apps/parsers/migrations/0026_update_fns_financial_schedule.py new file mode 100644 index 0000000..f74cd6a --- /dev/null +++ b/src/apps/parsers/migrations/0026_update_fns_financial_schedule.py @@ -0,0 +1,51 @@ +import json + +from django.db import migrations + +SCHEDULE_NAME = "parser:fns_financial:weekly-saturday-msk" +TASK_NAME = "apps.parsers.tasks.sync_fns_financial_reports" +WEEKLY_MSK_CRON = { + "minute": "0", + "hour": "0", + "day_of_week": "6", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "Europe/Moscow", +} + + +def update_fns_financial_schedule(apps, schema_editor): + CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + crontab, _ = CrontabSchedule.objects.get_or_create(**WEEKLY_MSK_CRON) + field_names = {field.name for field in PeriodicTask._meta.fields} + schedule_fields = {"crontab": crontab} + for field_name in ("interval", "solar", "clocked"): + if field_name in field_names: + schedule_fields[field_name] = None + + PeriodicTask.objects.update_or_create( + name=SCHEDULE_NAME, + defaults={ + "task": TASK_NAME, + "args": json.dumps([]), + "kwargs": json.dumps({}), + "enabled": True, + "description": "Default FNS API sync: weekly on Saturday 00:00 MSK.", + **schedule_fields, + }, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0025_consolidate_inspections_schedule"), + ] + + operations = [ + migrations.RunPython( + update_fns_financial_schedule, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index 219d45d..93eabd0 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -921,11 +921,11 @@ class SourceCardService: params: dict[str, Any], ) -> list[dict[str, str]]: if definition.slug == "financial-indicators": - from apps.parsers.tasks import scan_fns_directory + from apps.parsers.tasks import sync_fns_financial_reports task_info = cls._enqueue_task( - task=scan_fns_directory, - task_name="apps.parsers.tasks.scan_fns_directory", + task=sync_fns_financial_reports, + task_name="apps.parsers.tasks.sync_fns_financial_reports", requested_by_id=requested_by_id, meta={ "source_card": definition.slug, diff --git a/src/apps/parsers/source_registry.py b/src/apps/parsers/source_registry.py index d7a665c..d9b14e3 100644 --- a/src/apps/parsers/source_registry.py +++ b/src/apps/parsers/source_registry.py @@ -136,7 +136,9 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { task_name="apps.parsers.tasks.parse_procurements_44fz", mode="official_api", status="implemented", - upstream_url="https://zakupki.gov.ru/epz/order/extendedsearch/results.html", + upstream_url=( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html?fz44=on" + ), access_method="eis_official_api", parser_strategy="eis_44fz_search", source_notes="Официальный поиск ЕИС; XML-выгрузки ЕИС требуют отдельного discovery по реестрам.", @@ -151,10 +153,12 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { task_name="apps.parsers.tasks.parse_procurements_223fz", mode="official_api", status="implemented", - upstream_url="https://zakupki.gov.ru/epz/orderclause/search/results.html", + upstream_url=( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html?fz223=on" + ), access_method="eis_official_api", parser_strategy="eis_223fz_search", - source_notes="Официальный реестр положений о закупках 223-ФЗ в ЕИС.", + source_notes="Официальный поиск закупочных процедур 223-ФЗ в ЕИС.", api_route="eis/procurements-223fz", ), "contracts": ParserSourceDescriptor( @@ -205,19 +209,17 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { title="Финансово-экономические показатели", agency="ФНС России", data_scope="Финансово-экономическая выгрузка", - task_name="apps.parsers.tasks.scan_fns_directory", + task_name="apps.parsers.tasks.sync_fns_financial_reports", mode="official_api", status="implemented", owner="Сергей", - upstream_url=( - "https://bo.nalog.gov.ru/advanced-search/organizations/search" - "?query=%D0%9E%D0%9E%D0%9E&page=0&size=100" - ), + upstream_url="https://bo.nalog.gov.ru/advanced-search/organizations/search", access_method="public_web_api", - parser_strategy="fns_bfo_search_and_download", + parser_strategy="fns_bfo_bounded_registry_sync", source_notes=( - "ГИР БО: поиск организаций и скачивание отчетности с ЭП ФНС. " - "Ручная загрузка разрешена для финансовых выгрузок от Сергея." + "ГИР БО: точечная загрузка по ИНН/ID ФНС или ограниченному набору " + "активных организаций реестров; общий каталог не сканируется. " + "Ручная загрузка файлов и файловый scanner сохранены." ), supports_file_upload=True, api_route="fns/reports", diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 0934eb7..ff33f75 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -21,6 +21,7 @@ from apps.core.tasks import PeriodicTask as CorePeriodicTask from apps.parsers.clients.base import HTTPClientError from apps.parsers.clients.checko import ( CheckoClient, + CheckoRateLimitError, CompanyRequest, ContractLaw, ContractsRequest, @@ -32,9 +33,10 @@ from apps.parsers.clients.checko import ( ) from apps.parsers.clients.checko.exceptions import CheckoError from apps.parsers.clients.common import GenericParserItem, StructuredDataClient +from apps.parsers.clients.fns import FNSApiClient, FNSApiReport +from apps.parsers.clients.gisp import GispProductsClient from apps.parsers.clients.minpromtorg import ( IndustrialProductionClient, - IndustrialProductsClient, ManufacturesClient, ) from apps.parsers.clients.proverki import ProverkiClient @@ -58,6 +60,7 @@ from apps.parsers.services import ( from apps.parsers.source_registry import PARSER_SOURCES from celery import shared_task from django.conf import settings +from django.db.models import Q from organizations.models import Organization as SourceOrganization from organizations.services import ( normalize_organization_name as normalize_identity_name, @@ -81,6 +84,10 @@ PARSER_TIME_LIMIT_SECONDS = 20 * 60 INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS = 45 * 60 INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS = 60 * 60 CHECKO_SEARCH_RESULT_LIMIT = 100 +FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT = 25 +FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT = 100 +FNS_API_DEFAULT_MAX_PERIODS = 5 +FNS_API_MAX_PERIODS = 10 class ParserSourceSkipped(Exception): @@ -109,6 +116,16 @@ class FstecIdentityCandidate: provider: str +@dataclass(frozen=True) +class FNSApiFetchResult: + """Итог ограниченной загрузки отчётов из ГИР БО.""" + + reports: list[FNSApiReport] + target_count: int + successful_targets: int + failed_targets: int + + VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION = 100 VACANCY_REGISTRY_TEXT_SEARCH_MAX_PAGES_PER_ORGANIZATION = 1 VACANCY_EMPLOYER_WORD_RE = re.compile(r"[0-9A-Za-zА-Яа-яЁё]+") @@ -410,12 +427,17 @@ def _fetch_fedresurs_bankruptcy_records( """Загрузить банкротства: официальный портал, затем fallback через Checko.""" official_error: Exception | None = None try: - return _fetch_structured_records( + official_records = _fetch_structured_records( source_key="fedresurs_bankruptcy", file_url=file_url, file_path=file_path, proxies=proxies, ) + if official_records or file_url or file_path: + return official_records + logger.warning( + "Fedresurs official source returned no records, falling back to Checko" + ) except Exception as exc: if file_url or file_path: raise @@ -424,15 +446,20 @@ def _fetch_fedresurs_bankruptcy_records( "Fedresurs official source failed, falling back to Checko: %s", exc, ) - records = _fetch_checko_bankruptcy_records(proxies=proxies) - if records: - return records - if isinstance(official_error, HTTPClientError): - raise ParserSourceSkipped( - "fedresurs upstream is unavailable or blocked; " - "Checko fallback returned no bankruptcy records" - ) from official_error + records = _fetch_checko_bankruptcy_records(proxies=proxies) + if records: return records + if official_error is None: + raise ParserSourceSkipped( + "fedresurs official source returned no bankruptcy records; " + "Checko fallback returned no bankruptcy records" + ) + if isinstance(official_error, HTTPClientError): + raise ParserSourceSkipped( + "fedresurs upstream is unavailable or blocked; " + "Checko fallback returned no bankruptcy records" + ) from official_error + raise official_error def _fetch_fstec_records( @@ -805,6 +832,13 @@ def _fetch_checko_bankruptcy_records( response = client.get_company( CompanyRequest(inn=target.inn or None, ogrn=target.ogrn or None) ) + except CheckoRateLimitError as exc: + logger.warning( + "Checko bankruptcy fallback stopped: quota/rate limit reached " + "(status_code=%s)", + exc.status_code, + ) + break except CheckoError as exc: logger.info( "Checko bankruptcy lookup skipped for target=%s: %s", @@ -1878,6 +1912,9 @@ def parse_industrial_products( proxies: list[str] | None = None, client_adapter: BaseAdapter | None = None, requested_by_id: int | None = None, + page_size: int = 100, + max_pages: int = 10, + max_records: int = 1_000, ) -> dict: """ Задача парсинга реестра промышленной продукции. @@ -1885,6 +1922,9 @@ def parse_industrial_products( Args: proxies: Список прокси-серверов (опционально). client_adapter: HTTP-адаптер (опционально). + page_size: Размер страницы ГИСП (не более 100). + max_pages: Максимальное число страниц ГИСП (не более 100). + max_records: Максимальное число записей за запуск (не более 10 000). Returns: Результат: batch_id, saved, status @@ -1916,12 +1956,18 @@ def parse_industrial_products( job.update_progress(0, "Инициализация парсера...") try: - job.update_progress(10, "Загрузка данных с API Минпромторга...") - client_kwargs = {"proxies": proxies} + job.update_progress(10, "Загрузка данных из реестра ГИСП...") + client_kwargs = { + "proxies": proxies, + "page_size": page_size, + "max_pages": max_pages, + "max_records": max_records, + } if client_adapter: client_kwargs["http_adapter"] = client_adapter - with IndustrialProductsClient(**client_kwargs) as client: - products = client.fetch_products() + with GispProductsClient(**client_kwargs) as client: + fetch_result = client.fetch_products() + products = fetch_result.products job.update_progress(50, f"Сохранение {len(products)} записей продукции...") saved_count = IndustrialProductService.save_products( @@ -1935,19 +1981,27 @@ def parse_industrial_products( records_count=saved_count, ) - job.complete(result={"batch_id": batch_id, "saved": saved_count}) - - logger.info( - "Industrial products parsing completed (batch_id=%d, saved=%d)", - batch_id, - saved_count, - ) - - return { + result = { "batch_id": batch_id, "saved": saved_count, "status": "success", + "fetched": fetch_result.items_fetched, + "total": fetch_result.total_count, + "truncated": fetch_result.truncated, } + job.complete(result=result) + + logger.info( + "GISP industrial products parsing completed " + "(batch_id=%d, fetched=%d, total=%s, truncated=%s, saved=%d)", + batch_id, + fetch_result.items_fetched, + fetch_result.total_count, + fetch_result.truncated, + saved_count, + ) + + return result except Exception as e: logger.error("Industrial products parsing failed: %s", e, exc_info=True) @@ -2154,7 +2208,7 @@ def parse_all_sources( manufactures_result = parse_manufactures.apply( kwargs={"proxies": proxies, "client_adapter": client_adapter} ) - inspections_result = parse_inspections.apply( + inspections_result = sync_inspections.apply( kwargs={ "proxies": proxies, "client_adapter": client_adapter, @@ -2175,7 +2229,7 @@ def parse_all_sources( proxies=proxies, client_adapter=client_adapter, ) - inspections_result = parse_inspections.delay( + inspections_result = sync_inspections.delay( proxies=proxies, client_adapter=client_adapter, use_playwright=inspections_use_playwright, @@ -2190,6 +2244,7 @@ def parse_all_sources( "contracts": parse_contracts.delay(proxies=proxies).id, "unfair_suppliers": parse_unfair_suppliers.delay(proxies=proxies).id, "fas_goz": parse_fas_goz_evasion.delay(proxies=proxies).id, + "fns_financial": sync_fns_financial_reports.delay(proxies=proxies).id, "arbitration": parse_arbitration_cases.delay(proxies=proxies).id, "fedresurs_bankruptcy": parse_fedresurs_bankruptcy.delay( proxies=proxies @@ -3162,6 +3217,7 @@ def cleanup_stale_parser_loads(max_age_minutes: int | None = None) -> dict: ) source_values = {descriptor.source for descriptor in PARSER_SOURCES.values()} task_names = {descriptor.task_name for descriptor in PARSER_SOURCES.values()} + task_names.add("apps.parsers.tasks.scan_fns_directory") marked_failed = ParserLoadLogService.mark_stale_in_progress_failed( max_age_minutes=int(max_age_minutes) ) @@ -3243,6 +3299,7 @@ def _fetch_vacancy_records( with VacanciesClient( proxies=proxies, superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""), + hh_user_agent=getattr(settings, "HH_USER_AGENT", ""), sources=vacancy_sources, ) as client: return client.fetch_vacancies( @@ -3468,6 +3525,7 @@ def _fetch_registry_organization_vacancy_records( with VacanciesClient( proxies=proxies, superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""), + hh_user_agent=getattr(settings, "HH_USER_AGENT", ""), sources=vacancy_sources, ) as client: for target in targets: @@ -3518,6 +3576,231 @@ def _fetch_registry_organization_vacancy_records( # ============================================================================= +def _resolve_fns_api_registry_limit(value: int | None) -> int: + """Ограничить точечную синхронизацию безопасным числом организаций.""" + resolved = _resolve_lookup_limit( + value, + default=FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT, + ) + return min(resolved, FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT) + + +def _resolve_fns_api_max_periods(value: int | None) -> int: + """Ограничить число запрашиваемых отчётных периодов одной организации.""" + resolved = _resolve_lookup_limit(value, default=FNS_API_DEFAULT_MAX_PERIODS) + return min(max(resolved, 1), FNS_API_MAX_PERIODS) + + +def _active_fns_registry_inns(*, limit: int) -> list[str]: + """Вернуть ограниченный набор ИНН активных организаций реестров.""" + if limit <= 0: + return [] + + queryset = ( + SourceOrganization.objects.filter( + Q(opk_registry_membership=True) + | Q(goz_participation=True) + | ~Q(ropk_num="") + ) + .exclude(inn="") + .order_by("inn") + .values_list("inn", flat=True) + .distinct() + ) + return [str(value).strip() for value in queryset[:limit] if str(value).strip()] + + +def _fetch_fns_registry_reports( + client: FNSApiClient, + *, + target_inns: list[str], +) -> FNSApiFetchResult: + """Получить отчёты реестровых организаций, изолируя ошибки одной цели.""" + reports: list[FNSApiReport] = [] + successful_targets = 0 + failed_targets = 0 + first_target_error = "" + for target_inn in target_inns: + try: + reports.extend(client.fetch_reports_by_inn(target_inn)) + successful_targets += 1 + except (HTTPClientError, ValueError) as exc: + failed_targets += 1 + if not first_target_error: + first_target_error = str(exc) + logger.warning( + "FNS API sync failed for registry INN %s: %s", + target_inn, + exc, + ) + + if failed_targets and successful_targets == 0: + raise RuntimeError( + "FNS API sync failed for every selected registry organization; " + f"first error: {first_target_error}" + ) + return FNSApiFetchResult( + reports=reports, + target_count=len(target_inns), + successful_targets=successful_targets, + failed_targets=failed_targets, + ) + + +def _fetch_fns_api_reports( + *, + normalized_inn: str, + normalized_organization_id: str, + registry_limit: int, + max_periods: int, + proxies: list[str] | None, +) -> FNSApiFetchResult: + """Выбрать только явную цель или ограниченный набор реестровых ИНН.""" + target_inns: list[str] | None = None + if not normalized_inn and not normalized_organization_id: + target_inns = _active_fns_registry_inns(limit=registry_limit) + if not target_inns: + return FNSApiFetchResult([], 0, 0, 0) + + with FNSApiClient(max_periods=max_periods, proxies=proxies) as client: + if normalized_organization_id: + reports = client.fetch_reports_by_organization_id( + normalized_organization_id + ) + return FNSApiFetchResult(reports, 1, 1, 0) + if normalized_inn: + reports = client.fetch_reports_by_inn(normalized_inn) + return FNSApiFetchResult(reports, 1, 1, 0) + return _fetch_fns_registry_reports(client, target_inns=target_inns or []) + + +def _save_fns_api_reports( + reports: list[FNSApiReport], + *, + batch_id: int, +) -> tuple[int, int, int]: + """Сохранить API-отчёты через общую FNS ingestion boundary.""" + saved_reports = 0 + saved_lines = 0 + skipped_reports = 0 + for report in reports: + try: + FNSReportService.save_report( + **report.to_save_report_kwargs(batch_id=batch_id) + ) + except FNSReportOrganizationResolutionSkipped as exc: + skipped_reports += 1 + logger.warning( + "FNS API report %s skipped: %s", + report.external_id, + exc.reason, + ) + continue + saved_reports += 1 + saved_lines += len(report.lines) + return saved_reports, saved_lines, skipped_reports + + +@shared_task( + bind=True, + soft_time_limit=INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS, + time_limit=INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS, +) +def sync_fns_financial_reports( + self, + *, + inn: str | None = None, + organization_id: int | str | None = None, + registry_organization_limit: int | None = None, + max_periods: int | None = None, + proxies: list[str] | None = None, + requested_by_id: int | None = None, +) -> dict: + """Синхронизировать ГИР БО точечно, не сканируя общую выдачу ФНС.""" + normalized_inn = "" if inn is None else str(inn).strip() + normalized_organization_id = ( + "" if organization_id is None else str(organization_id).strip() + ) + if normalized_inn and normalized_organization_id: + raise ValueError("Provide either inn or organization_id, not both") + + registry_limit = _resolve_fns_api_registry_limit(registry_organization_limit) + resolved_max_periods = _resolve_fns_api_max_periods(max_periods) + resolved_proxies = _resolve_proxies(proxies) + if normalized_organization_id: + sync_mode = "organization_id" + elif normalized_inn: + sync_mode = "inn" + else: + sync_mode = "active_registry" + + source = ParserLoadLog.Source.FNS_REPORTS + task_name = "apps.parsers.tasks.sync_fns_financial_reports" + load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( + source=source, + status="in_progress", + ) + task_id = self.request.id or str(uuid.uuid4()) + job = _get_or_create_background_job( + task_id=task_id, + task_name=task_name, + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, + meta={ + "source_key": "fns_financial", + "sync_mode": sync_mode, + "max_periods": resolved_max_periods, + }, + ) + job.mark_started() + job.update_progress(0, "Подготовка точечной синхронизации ГИР БО...") + + try: + job.update_progress(15, "Загрузка отчётности выбранных организаций...") + fetch_result = _fetch_fns_api_reports( + normalized_inn=normalized_inn, + normalized_organization_id=normalized_organization_id, + registry_limit=registry_limit, + max_periods=resolved_max_periods, + proxies=resolved_proxies, + ) + job.update_progress( + 70, + f"Сохранение {len(fetch_result.reports)} отчётов ГИР БО...", + ) + saved_reports, saved_lines, skipped_reports = _save_fns_api_reports( + fetch_result.reports, + batch_id=batch_id, + ) + + ParserLoadLogService.update( + load_log, + status="success", + records_count=saved_lines, + ) + result = { + "batch_id": batch_id, + "status": "success", + "sync_mode": sync_mode, + "targets": fetch_result.target_count, + "successful_targets": fetch_result.successful_targets, + "failed_targets": fetch_result.failed_targets, + "fetched_reports": len(fetch_result.reports), + "saved": saved_reports, + "saved_reports": saved_reports, + "saved_lines": saved_lines, + "skipped_reports": skipped_reports, + } + job.complete(result=result) + return result + except Exception as exc: + logger.error("%s failed: %s", task_name, exc, exc_info=True) + ParserLoadLogService.mark_failed(load_log, str(exc)) + job.fail(error=str(exc)) + raise + + @shared_task(bind=True) def scan_fns_directory( self, diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 12f158d..0574db9 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -138,6 +138,7 @@ TASKS_BY_NAME = { "apps.parsers.tasks.parse_unfair_suppliers": tasks.parse_unfair_suppliers, "apps.parsers.tasks.parse_fas_goz_evasion": tasks.parse_fas_goz_evasion, "apps.parsers.tasks.scan_fns_directory": tasks.scan_fns_directory, + "apps.parsers.tasks.sync_fns_financial_reports": (tasks.sync_fns_financial_reports), "apps.parsers.tasks.parse_arbitration_cases": tasks.parse_arbitration_cases, "apps.parsers.tasks.parse_registry_inspections": tasks.parse_registry_inspections, "apps.parsers.tasks.parse_fedresurs_bankruptcy": tasks.parse_fedresurs_bankruptcy, diff --git a/src/core/celery.py b/src/core/celery.py index 7f477b6..6ad336e 100644 --- a/src/core/celery.py +++ b/src/core/celery.py @@ -70,10 +70,6 @@ app.conf.beat_schedule = { "task": "apps.parsers.tasks.parse_industrial_products", "schedule": 7 * 24 * 60 * 60, # Every 7 days }, - "parse-inspections-weekly": { - "task": "apps.parsers.tasks.parse_inspections", - "schedule": 7 * 24 * 60 * 60, # Every 7 days - }, "sync-ru-proxies-hourly": { "task": "apps.parsers.tasks.sync_ru_proxies", "schedule": getattr(settings, "PROXY_TOOLS_SYNC_INTERVAL_SECONDS", 3600), diff --git a/src/organizations/cache.py b/src/organizations/cache.py index 5023bb9..9532f1a 100644 --- a/src/organizations/cache.py +++ b/src/organizations/cache.py @@ -7,6 +7,7 @@ import time from django.core.cache import cache ORGANIZATION_API_CACHE_PREFIX = "api:v2:organizations" +ORGANIZATION_API_CACHE_CONTRACT_VERSION = 2 ORGANIZATION_API_CACHE_VERSION_KEY = f"{ORGANIZATION_API_CACHE_PREFIX}:version" DEFAULT_ORGANIZATION_API_CACHE_VERSION = 1 DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS = 24 * 60 * 60 diff --git a/src/organizations/filters.py b/src/organizations/filters.py index c69d0c3..92ce9d3 100644 --- a/src/organizations/filters.py +++ b/src/organizations/filters.py @@ -45,7 +45,6 @@ class OrganizationFilter(filters.FilterSet): ) registry = filters.CharFilter(method="filter_registry") registry_name = filters.CharFilter(method="filter_registry_name") - has_registry = filters.BooleanFilter(method="filter_has_registry") source_group = filters.CharFilter(method="filter_source_group") has_financial_indicators = filters.BooleanFilter(method="filter_source_presence") @@ -83,7 +82,6 @@ class OrganizationFilter(filters.FilterSet): "identity_status", "registry", "registry_name", - "has_registry", "source_group", ] @@ -93,9 +91,6 @@ class OrganizationFilter(filters.FilterSet): def filter_registry_name(self, queryset, _name, value): return self._filter_by_registry_membership(queryset, registry_name=value) - def filter_has_registry(self, queryset, _name, value): - return self._filter_by_registry_membership(queryset, has_registry=value) - def filter_source_group(self, queryset, _name, value): source_group = SOURCE_FILTER_ALIASES.get(str(value), str(value)) return self._filter_by_source_group(queryset, source_group, True) @@ -123,15 +118,12 @@ class OrganizationFilter(filters.FilterSet): *, registry_id: str | None = None, registry_name: str | None = None, - has_registry: bool = True, ): query = cls._registry_directory_query( registry_id=registry_id, registry_name=registry_name, ) - if has_registry: - return queryset.filter(query) - return queryset.exclude(query) + return queryset.filter(query) @staticmethod def _registry_directory_query( @@ -139,11 +131,7 @@ class OrganizationFilter(filters.FilterSet): registry_id: str | None = None, registry_name: str | None = None, ) -> Q: - query = ( - Q(opk_registry_membership=True) - | Q(goz_participation=True) - | ~Q(ropk_num="") - ) + query = Q(opk_registry_membership=True) if registry_id: query &= Q(ropk_razdel_num=str(registry_id)) | Q(gk_code=str(registry_id)) if registry_name: @@ -153,20 +141,3 @@ class OrganizationFilter(filters.FilterSet): | Q(business_activity__icontains=registry_name) ) return query - - @staticmethod - def _registry_identity_value_querysets( - *, - registry_id: str | None = None, - registry_name: str | None = None, - ): - organizations = Organization.objects.filter( - OrganizationFilter._registry_directory_query( - registry_id=registry_id, - registry_name=registry_name, - ) - ) - return ( - organizations.exclude(inn="").values_list("inn", flat=True), - organizations.exclude(ogrn="").values_list("ogrn", flat=True), - ) diff --git a/src/organizations/management/commands/create_test_companies.py b/src/organizations/management/commands/create_test_companies.py new file mode 100644 index 0000000..bff9dba --- /dev/null +++ b/src/organizations/management/commands/create_test_companies.py @@ -0,0 +1,29 @@ +"""Create or refresh the fixed frontend test-company dataset.""" + +import json + +from apps.core.management.commands.base import BaseAppCommand + +from organizations.test_companies import TestCompanyDatasetService + + +class Command(BaseAppCommand): + """Create twenty deterministic organizations with every source dataset.""" + + help = "Создает или обновляет 20 тестовых компаний со всеми наборами данных" + use_transaction = True + + def execute_command(self, *args, **options) -> str: + result = TestCompanyDatasetService.create_or_update() + rendered = json.dumps( + { + "organizations_created": result.organizations_created, + "organizations_updated": result.organizations_updated, + "extensions": result.extensions, + "records": result.records, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/organizations/management/commands/delete_test_companies.py b/src/organizations/management/commands/delete_test_companies.py new file mode 100644 index 0000000..44f3ccd --- /dev/null +++ b/src/organizations/management/commands/delete_test_companies.py @@ -0,0 +1,24 @@ +"""Delete the fixed frontend test-company dataset.""" + +import json + +from apps.core.management.commands.base import BaseAppCommand + +from organizations.test_companies import TestCompanyDatasetService + + +class Command(BaseAppCommand): + """Delete only the twenty deterministic test organizations.""" + + help = "Удаляет 20 фиксированных тестовых компаний и связанные данные" + use_transaction = True + + def execute_command(self, *args, **options) -> str: + result = TestCompanyDatasetService.delete() + rendered = json.dumps( + {"organizations_deleted": result.organizations_deleted}, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/organizations/test_companies.py b/src/organizations/test_companies.py new file mode 100644 index 0000000..1fea671 --- /dev/null +++ b/src/organizations/test_companies.py @@ -0,0 +1,539 @@ +"""Deterministic, realistic test-company dataset for frontend demonstrations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date +from decimal import Decimal +from hashlib import sha256 +from uuid import UUID, uuid5 + +from apps.parsers.models import ParserLoadLog +from django.db.models import Count, Max, Min +from django.utils import timezone + +from organizations.cache import invalidate_organization_api_cache +from organizations.models import ( + Organization, + OrganizationSourceFinancialLine, + OrganizationSourceRecord, + SourceExtensionStatus, +) +from organizations.source_cache import invalidate_source_data_cache +from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS + +TEST_COMPANY_COUNT = 20 +TEST_COMPANY_NAMESPACE = UUID("59b36ae9-bcf8-4b7c-b77a-77578f485a01") +TEST_RECORD_PREFIX = "mostovik-test-company" + + +@dataclass(frozen=True) +class TestCompanyDatasetResult: + """Counters returned by create and delete operations.""" + + organizations_created: int = 0 + organizations_updated: int = 0 + organizations_deleted: int = 0 + extensions: int = 0 + records: int = 0 + + +class TestCompanyDatasetService: + """Create, refresh, and remove the fixed frontend demonstration dataset.""" + + @classmethod + def company_uids(cls) -> list[UUID]: + return [ + uuid5(TEST_COMPANY_NAMESPACE, f"company:{index}") + for index in range(1, TEST_COMPANY_COUNT + 1) + ] + + @classmethod + def create_or_update(cls) -> TestCompanyDatasetResult: + now = timezone.now() + created_count = 0 + updated_count = 0 + total_extensions = 0 + total_records = 0 + + descriptors_by_group = cls._descriptors_by_group() + for index, uid in enumerate(cls.company_uids(), start=1): + organization, created = Organization.objects.update_or_create( + uid=uid, + defaults=cls._organization_defaults(index=index, now=now), + ) + created_count += int(created) + updated_count += int(not created) + + extensions = {} + for source_group, descriptors in descriptors_by_group.items(): + descriptor = descriptors[0] + extension, _ = descriptor.extension_model.objects.update_or_create( + organization=organization, + defaults={ + "title": descriptor.title, + "status": SourceExtensionStatus.ACTIVE, + "first_seen_at": now, + "last_seen_at": now, + "last_load_batch": 9_000_000 + index, + "metadata": { + "dataset": TEST_RECORD_PREFIX, + "sources": [item.source for item in descriptors], + }, + }, + ) + extensions[source_group] = extension + + expected_external_ids = [] + for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items(): + if source not in ParserLoadLog.Source.values: + continue + external_id = f"{TEST_RECORD_PREFIX}:{index:02d}:{source}" + expected_external_ids.append(external_id) + record_defaults = cls._record_defaults( + source=source, + index=index, + organization=organization, + ) + record, _ = OrganizationSourceRecord.objects.update_or_create( + source=source, + external_id=external_id, + defaults={ + "extension": extensions[descriptor.source_group], + "record_type": descriptor.record_type, + "load_batch": 9_000_000 + index, + **record_defaults, + }, + ) + if source == ParserLoadLog.Source.FNS_REPORTS: + cls._refresh_financial_lines(record=record, index=index) + + OrganizationSourceRecord.objects.filter( + extension__organization=organization, + external_id__startswith=f"{TEST_RECORD_PREFIX}:", + ).exclude(external_id__in=expected_external_ids).delete() + cls._refresh_extension_counters(organization=organization) + total_extensions += len(extensions) + total_records += len(expected_external_ids) + + cls._invalidate_caches() + return TestCompanyDatasetResult( + organizations_created=created_count, + organizations_updated=updated_count, + extensions=total_extensions, + records=total_records, + ) + + @classmethod + def delete(cls) -> TestCompanyDatasetResult: + company_uids = cls.company_uids() + queryset = Organization.objects.filter(uid__in=company_uids) + deleted_count = queryset.count() + extension_models = { + descriptor.extension_model + for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items() + if source in ParserLoadLog.Source.values + } + for extension_model in extension_models: + extension_model.objects.filter(organization_id__in=company_uids).delete() + queryset.delete() + cls._invalidate_caches() + return TestCompanyDatasetResult(organizations_deleted=deleted_count) + + @staticmethod + def _descriptors_by_group(): + descriptors = {} + for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items(): + if source in ParserLoadLog.Source.values: + descriptors.setdefault(descriptor.source_group, []).append(descriptor) + return descriptors + + @classmethod + def _organization_defaults(cls, *, index: int, now): + is_rosatom = index <= 10 + corporation_name = ( + 'Госкорпорация "Росатом"' if is_rosatom else 'Госкорпорация "Роскосмос"' + ) + corporation_ministry = ( + 'Государственная корпорация по атомной энергии "Росатом"' + if is_rosatom + else 'Государственная корпорация по космической деятельности "Роскосмос"' + ) + industry = ( + "Атомная промышленность" + if is_rosatom + else "Ракетно-космическая промышленность" + ) + name = f"Тестовая компания {index}" + inn = cls._legal_entity_inn(index) + ogrn = cls._legal_entity_ogrn(index) + return { + "rn": 9_900_000 + index, + "gk_code": "2" if is_rosatom else "1", + "gk_name": corporation_name, + "in_korp_code": "1", + "in_korp_name": "Входит в состав", + "name": name, + "full_name": f'Акционерное общество "{name}"', + "short_name": name, + "pn_name": name, + "pn_name_en": f"Test Company {index}", + "inn": inn, + "kpp": f"7709{index:02d}001", + "ogrn": ogrn, + "okpo": f"90{index:06d}", + "filial": ".F.", + "is_branch": False, + "registration_date": date(2010 + index % 10, (index - 1) % 12 + 1, 15), + "create_date": str(2010 + index % 10), + "organizational_legal_form": "12267", + "organizational_legal_form1": "Непубличные акционерные общества", + "ownership_form": "61" if is_rosatom else "16", + "ownership_form1": ( + "Собственность государственных корпораций" + if is_rosatom + else "Частная собственность" + ), + "authorized_capital": Decimal(1_000_000 + index * 100_000), + "legal_address": ( + f"{101000 + index}, г. Москва, Тестовый проезд, д. {index}" + ), + "business_act_cod": "4" if is_rosatom else "2", + "business_activity": "Прочая" if is_rosatom else "Производственная", + "general_director": f"Иванов Иван Иванович {index}", + "general_director_tax_id": f"770100{index:06d}", + "appointment_date": date(2024, 1, min(index, 28)), + "akc_fs": "0", + "akc_sf": "0", + "re_za": False, + "re_zasf": False, + "goz_participation": True, + "opk_registry_membership": True, + "ropk_num": f"ТЕСТ-ОПК-{index:04d}", + "ropk_razdel_num": "3" if is_rosatom else "2", + "ropk_razdel_name": ( + "Организации, подведомственные и находящиеся в сфере деятельности " + f"{corporation_name}" + ), + "min": corporation_ministry, + "dep": "Департамент тестовых данных", + "otr": industry, + "integrated_structure": f"Тестовая интегрированная структура {index}", + "state_sector_code": "2", + "state_sector_name": ( + "контролируются РФ косвенно или совместно с организациями госсектора" + ), + "directory_source_file_hash": sha256( + f"{TEST_RECORD_PREFIX}:{index}".encode() + ).hexdigest(), + "directory_source_row_number": index, + "directory_imported_at": now, + } + + @classmethod + def _record_defaults(cls, *, source: str, index: int, organization): + common = { + "inn": organization.inn, + "ogrn": organization.ogrn, + "organisation_name": organization.name, + "source": source, + "load_batch": 9_000_000 + index, + } + url = f"https://example.test/{source}/{index}" + values = { + ParserLoadLog.Source.FNS_REPORTS: { + "title": f"Бухгалтерская отчетность за 2025 год — {organization.name}", + "record_date": "2025", + "status": "processed", + "payload": { + **common, + "file_name": f"fin_test_{organization.ogrn}.xlsx", + "file_hash": sha256( + f"financial:{organization.ogrn}".encode() + ).hexdigest(), + "lines_count": 4, + }, + }, + ParserLoadLog.Source.PROCUREMENTS: cls._procurement_values( + common, index, organization, law="44-ФЗ/223-ФЗ", source=source + ), + ParserLoadLog.Source.PROCUREMENTS_44FZ: cls._procurement_values( + common, index, organization, law="44-ФЗ", source=source + ), + ParserLoadLog.Source.PROCUREMENTS_223FZ: cls._procurement_values( + common, index, organization, law="223-ФЗ", source=source + ), + ParserLoadLog.Source.CONTRACTS: { + "title": f"Контракт на поставку оборудования № {index}", + "record_date": "15.03.2026", + "amount": Decimal(3_000_000 + index * 25_000), + "status": "Исполнение", + "url": url, + "payload": { + **common, + "contract_number": f"КОНТРАКТ-{index:05d}", + "subject": "Поставка испытательного оборудования", + "customer": organization.name, + "price": str(3_000_000 + index * 25_000), + "law": "44-ФЗ", + "status": "Исполнение", + "url": url, + }, + }, + ParserLoadLog.Source.INDUSTRIAL: { + "title": f"Сертификат промышленного производства {index:05d}/26", + "record_date": "15.01.2026", + "url": url, + "payload": { + **common, + "certificate_number": f"{index:05d}/26", + "issue_date": "15.01.2026", + "issue_date_normalized": "2026-01-15", + "expiry_date": "15.01.2029", + "expiry_date_normalized": "2029-01-15", + "certificate_file_url": url, + }, + }, + ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: { + "title": f"Испытательный комплекс ТК-{index}", + "payload": { + **common, + "full_organisation_name": organization.full_name, + "registry_number": f"ПРОД-{index:06d}", + "product_name": f"Испытательный комплекс ТК-{index}", + "product_model": f"ТК-{index}.2026", + "okpd2_code": "28.99.39.190", + "tnved_code": "8479899707", + "regulatory_document": "ТУ 28.99.39-001-2026", + }, + }, + ParserLoadLog.Source.MANUFACTURES: { + "title": organization.full_name, + "payload": { + **common, + "full_legal_name": organization.full_name, + "address": organization.legal_address, + }, + }, + ParserLoadLog.Source.INSPECTIONS: { + "title": f"Плановая выездная проверка {organization.name}", + "record_date": "01.06.2026", + "status": "Запланирована", + "payload": { + **common, + "registration_number": f"24800000{index:04d}", + "control_authority": "Ростехнадзор", + "inspection_type": "scheduled", + "inspection_form": "documentary_and_on_site", + "legal_basis": "Федеральный закон № 248-ФЗ", + "start_date": "01.06.2026", + "start_date_normalized": "2026-06-01", + "end_date": "15.06.2026", + "end_date_normalized": "2026-06-15", + "data_year": 2026, + "data_month": 6, + "status": "Запланирована", + }, + }, + ParserLoadLog.Source.FEDRESURS_BANKRUPTCY: { + "title": f"Сообщение по делу А40-{10000 + index}/2026", + "record_date": "01.05.2026", + "status": "Наблюдение", + "url": url, + "payload": { + **common, + "case_number": f"А40-{10000 + index}/2026", + "message_type": "Введение наблюдения", + "message_date": "01.05.2026", + "messages_count": 3, + "messages": [ + { + "type": "Введение наблюдения", + "date": "01.05.2026", + } + ], + "url": url, + }, + }, + ParserLoadLog.Source.UNFAIR_SUPPLIERS: { + "title": f"Реестровая запись РНП-{index:05d}", + "record_date": "20.01.2026", + "status": "Включен", + "url": url, + "payload": { + **common, + "supplier": organization.name, + "registry_number": f"РНП-{index:05d}", + "included_at": "20.01.2026", + "reason": "Нарушение условий закупки", + "url": url, + }, + }, + ParserLoadLog.Source.FAS_GOZ: { + "title": f"Постановление ФАС по ГОЗ № {index}", + "record_date": "20.01.2026", + "status": "Исполнено", + "payload": { + **common, + "registry_number": str(index), + "authority": "ФАС России", + "decision": "Постановление по ГОЗ", + "decision_date": "20.01.2026", + "execution_status": "Исполнено", + }, + }, + ParserLoadLog.Source.ARBITRATION: { + "title": f"Дело А40-{20000 + index}/2026", + "record_date": "10.02.2026", + "amount": Decimal(500_000 + index * 10_000), + "status": "В производстве", + "url": url, + "payload": { + **common, + "case_number": f"А40-{20000 + index}/2026", + "court": "Арбитражный суд города Москвы", + "role": "ответчик", + "claim_amount": str(500_000 + index * 10_000), + "status": "В производстве", + "url": url, + }, + }, + ParserLoadLog.Source.FSTEC: { + "title": f"Сертификат ФСТЭК № ТЕСТ-{index:04d}", + "record_date": "15.01.2026", + "status": "Действует", + "payload": { + **common, + "licensee": organization.name, + "registry_number": f"ТЕСТ-{index:04d}", + "issued_at": "15.01.2026", + "status": "Действует", + "№ сертификата": f"ТЕСТ-{index:04d}", + "Заявитель": organization.name, + "Дата внесения в реестр": "15.01.2026", + "Срок действия сертификата": "15.01.2029", + }, + }, + ParserLoadLog.Source.TRUDVSEM: { + "title": f"Инженер-испытатель {index} категории", + "record_date": "20.05.2026", + "status": "Открыта", + "url": url, + "payload": { + **common, + "id": f"VAC-{index:05d}", + "company": organization.name, + "job-name": f"Инженер-испытатель {index} категории", + "creation-date": "2026-05-20", + "salary": "120000.00", + "salary_min": 110000, + "salary_max": 140000, + "currency": "RUB", + "employment": "Полная занятость", + "schedule": "Полный день", + "region": "Москва", + "status": "Открыта", + "vacancy_source": "trudvsem", + "vac_url": url, + }, + }, + } + return values[source] + + @staticmethod + def _procurement_values(common, index, organization, *, law: str, source: str): + purchase_number = f"03732000000{index:08d}"[:19] + amount = Decimal(1_250_000 + index * 50_000) + url = f"https://example.test/{source}/{index}" + return { + "title": f"Поставка оборудования для {organization.name}", + "record_date": "01.02.2026", + "amount": amount, + "status": "Размещено", + "url": url, + "payload": { + **common, + "purchase_number": purchase_number, + "subject": "Поставка испытательного оборудования", + "customer": organization.name, + "customer_inn": organization.inn, + "price": str(amount), + "law": law, + "published_at": "01.02.2026", + "status": "Размещено", + "region_code": "77", + "data_year": 2026, + "data_month": 2, + "url": url, + }, + } + + @staticmethod + def _refresh_financial_lines(*, record, index: int) -> None: + expected = { + ("1", "1600", "Баланс (актив)", 10_000_000 + index * 100_000), + ("1", "1300", "Капитал и резервы", 4_000_000 + index * 50_000), + ("2", "2110", "Выручка", 20_000_000 + index * 200_000), + ("2", "2400", "Чистая прибыль", 2_000_000 + index * 25_000), + } + expected_keys = [] + for form_code, line_code, line_name, period_end in expected: + expected_keys.append((form_code, line_code, 2025)) + OrganizationSourceFinancialLine.objects.update_or_create( + source_record=record, + form_code=form_code, + line_code=line_code, + year=2025, + defaults={ + "line_name": line_name, + "period_start": period_end - 100_000, + "period_end": period_end, + }, + ) + lines = OrganizationSourceFinancialLine.objects.filter(source_record=record) + for line in lines: + key = (line.form_code, line.line_code, line.year) + if key not in expected_keys: + line.delete() + + @staticmethod + def _refresh_extension_counters(*, organization) -> None: + aggregates = { + item["extension_id"]: item + for item in OrganizationSourceRecord.objects.filter( + extension__organization=organization + ) + .values("extension_id") + .annotate( + records_count=Count("uid"), + first_seen_at=Min("created_at"), + last_seen_at=Max("updated_at"), + ) + } + for extension in organization.source_extensions.all(): + aggregate = aggregates.get(extension.uid, {}) + extension.records_count = aggregate.get("records_count", 0) + extension.first_seen_at = aggregate.get("first_seen_at") + extension.last_seen_at = aggregate.get("last_seen_at") + extension.save( + update_fields=["records_count", "first_seen_at", "last_seen_at"] + ) + + @staticmethod + def _legal_entity_inn(index: int) -> str: + base = f"770900{index:03d}" + weights = (2, 4, 10, 3, 5, 9, 4, 6, 8) + checksum = sum( + int(digit) * weight for digit, weight in zip(base, weights, strict=True) + ) + return f"{base}{checksum % 11 % 10}" + + @staticmethod + def _legal_entity_ogrn(index: int) -> str: + base = f"1267700{index:05d}" + return f"{base}{int(base) % 11 % 10}" + + @staticmethod + def _invalidate_caches() -> None: + invalidate_organization_api_cache() + invalidate_source_data_cache() diff --git a/src/organizations/views.py b/src/organizations/views.py index 101b2ee..851a978 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -29,6 +29,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from organizations.cache import ( DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS, + ORGANIZATION_API_CACHE_CONTRACT_VERSION, ORGANIZATION_API_CACHE_PREFIX, get_organization_api_cache_version, invalidate_organization_api_cache, @@ -56,7 +57,6 @@ from organizations.serializers import ( from organizations.source_record_export import build_source_records_export_archive ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations") -FALSE_QUERY_VALUES = {"0", "false", "no", "off"} def _query_parameter( @@ -80,10 +80,6 @@ def _query_parameter( ) -def _is_truthy_query_value(value: str) -> bool: - return value.strip().lower() not in FALSE_QUERY_VALUES - - SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup] ORGANIZATION_LIST_PARAMS = [ @@ -129,15 +125,6 @@ ORGANIZATION_LIST_PARAMS = [ "registry_name", description="Фильтр по части наименования реестра.", ), - _query_parameter( - "has_registry", - description=( - "Фильтр наличия активного участия в любом реестре; по умолчанию true " - "для list endpoint, если параметр не передан." - ), - param_type=openapi.TYPE_BOOLEAN, - default=True, - ), _query_parameter( "source_group", description="Фильтр по группе источников организации.", @@ -180,11 +167,6 @@ SOURCE_RECORD_LIST_PARAMS = [ ), _query_parameter("source", description="Фильтр по legacy source внутри группы."), _query_parameter("record_type", description="Фильтр по типу записи."), - _query_parameter( - "has_registry", - description="Фильтр наличия активного участия организации записи в любом реестре.", - param_type=openapi.TYPE_BOOLEAN, - ), _query_parameter( "organization", description="UID организации.", @@ -228,6 +210,7 @@ class CachedReadOnlyMixin: cache_version = get_organization_api_cache_version() raw_key = ( + f"c{ORGANIZATION_API_CACHE_CONTRACT_VERSION}:" f"v{cache_version}:{request.method}:" f"{request.get_full_path()}:{user_marker}" ) @@ -287,18 +270,12 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): return super().get_permissions() def get_queryset(self): - queryset = super().get_queryset().prefetch_related("source_extensions") - if self.action != "list" or "has_registry" in self.request.query_params: - return queryset - - filterset = OrganizationFilter( - data={"has_registry": "true"}, - queryset=queryset, - request=self.request, + return ( + super() + .get_queryset() + .filter(opk_registry_membership=True) + .prefetch_related("source_extensions") ) - if filterset.is_valid(): - return filterset.qs - return queryset @swagger_auto_schema( tags=[ORGANIZATIONS_TAG], @@ -306,8 +283,8 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): operation_summary="Список организаций", operation_description=( "Возвращает канонический справочник организаций API v2. " - "По умолчанию показывает только организации с активным участием " - "в реестрах; передайте has_registry=false, чтобы снять это ограничение. " + "Контур frontend всегда ограничен организациями, для которых " + "opk_registry_membership=true; снять это ограничение query-параметром нельзя. " "Данные источников возвращаются компактным списком sources; детальные " "записи доступны через endpoints расширений источников." ), @@ -435,11 +412,13 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): class OrganizationSourceExtensionViewSet(ReadOnlyModelViewSet): """Read-only API for source extensions and their records.""" - queryset = OrganizationSourceExtension.objects.select_related( - "organization" - ).order_by( - "organization__name", - "source_group", + queryset = ( + OrganizationSourceExtension.objects.select_related("organization") + .filter(organization__opk_registry_membership=True) + .order_by( + "organization__name", + "source_group", + ) ) serializer_class = OrganizationSourceExtensionSerializer permission_classes = [IsAuthenticated] @@ -488,6 +467,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): "extension", "extension__organization", ) + .filter(extension__organization__opk_registry_membership=True) .prefetch_related("financial_lines") .order_by("-created_at", "-uid") ) @@ -542,7 +522,6 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): source = params.get("source") record_type = params.get("record_type") organization = params.get("organization") - has_registry = params.get("has_registry") search_terms = SearchFilter().get_search_terms(self.request) if source_group: @@ -553,30 +532,11 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): queryset = queryset.filter(record_type=record_type) if organization: queryset = queryset.filter(extension__organization_id=organization) - if has_registry is not None: - registry_query = self._registry_membership_query() - if _is_truthy_query_value(has_registry): - queryset = queryset.filter(registry_query) - else: - queryset = queryset.exclude(registry_query) if search_terms: queryset = self._filter_search_queryset(queryset, search_terms) return queryset - @staticmethod - def _registry_membership_query(): - ( - inn_values, - ogrn_values, - ) = OrganizationFilter._registry_identity_value_querysets() - - return ( - Q(extension__organization__inn__in=inn_values) - | Q(extension__organization__ogrn__in=ogrn_values) - | Q(extension__organization__ogrip__in=ogrn_values) - ) - @classmethod def _filter_search_queryset(cls, queryset, search_terms: list[str]): queryset = queryset.annotate( diff --git a/src/settings/base.py b/src/settings/base.py index c21761a..2535626 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -198,6 +198,13 @@ TEMPLATES = [ WSGI_APPLICATION = "core.wsgi.application" +# Registry archives can legitimately be close to the parser-level 200 MiB limit. +# Keep Django's request-body limit slightly higher to account for multipart framing; +# uploaded file content itself is still validated by ParserUploadRequestSerializer. +DATA_UPLOAD_MAX_MEMORY_SIZE = int( + os.getenv("DATA_UPLOAD_MAX_MEMORY_SIZE", str(210 * 1024 * 1024)) +) + # Database и Cache определяются в dev.py / production.py # ============================================================================= @@ -206,6 +213,7 @@ WSGI_APPLICATION = "core.wsgi.application" ZAKUPKI_TOKEN = os.getenv("ZAKUPKI_TOKEN", "") SUPERJOB_APP_ID = os.getenv("SUPERJOB_APP_ID", "").strip() +HH_USER_AGENT = os.getenv("HH_USER_AGENT", "").strip() FNS_LOCK_TTL_SECONDS = 3600 PARSER_PROXIES = [ item.strip() for item in os.getenv("PARSER_PROXIES", "").split(",") if item.strip() diff --git a/tests/apps/core/test_celery_module.py b/tests/apps/core/test_celery_module.py index 23eebf3..2d6b274 100644 --- a/tests/apps/core/test_celery_module.py +++ b/tests/apps/core/test_celery_module.py @@ -51,7 +51,7 @@ class CeleryModuleTest(SimpleTestCase): ) self.assertIn("parse-manufactures-daily", module.app.conf.beat_schedule) self.assertIn("parse-industrial-products-daily", module.app.conf.beat_schedule) - self.assertIn("parse-inspections-weekly", module.app.conf.beat_schedule) + self.assertNotIn("parse-inspections-weekly", module.app.conf.beat_schedule) self.assertIn("sync-ru-proxies-hourly", module.app.conf.beat_schedule) def test_startup_refresh_queues_when_lock_acquired(self): diff --git a/tests/apps/organizations/test_api_v2.py b/tests/apps/organizations/test_api_v2.py index 8b70985..6682fd7 100644 --- a/tests/apps/organizations/test_api_v2.py +++ b/tests/apps/organizations/test_api_v2.py @@ -31,6 +31,12 @@ from rest_framework.test import APITestCase from tests.apps.user.factories import UserFactory +def create_frontend_organization(**kwargs): + """Create an organization visible through the frontend API by default.""" + kwargs.setdefault("opk_registry_membership", True) + return Organization.objects.create(**kwargs) + + class OrganizationsApiV2Test(APITestCase): """Checks v2 canonical organizations endpoints.""" @@ -51,13 +57,13 @@ class OrganizationsApiV2Test(APITestCase): worksheet.append([key, value]) def test_list_is_paginated_and_available_only_under_v2(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Альфа"', inn="7707083893", kpp="770701001", ogrn="1027700132195", ) - Organization.objects.create( + create_frontend_organization( name="ИП Иванов Иван Иванович", inn="500100732259", ogrip="304500116000157", @@ -174,16 +180,13 @@ class OrganizationsApiV2Test(APITestCase): "search", "ordering", "registry", - "has_registry", "identity_status", "source_group", "has_financial_indicators", "has_planned_inspections", ): self.assertIn(expected_name, list_parameters) - self.assertIn( - "по умолчанию true", list_parameters["has_registry"]["description"] - ) + self.assertNotIn("has_registry", list_parameters) self.assertIn( "группе источников", list_parameters["source_group"]["description"], @@ -206,7 +209,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertIn("/api/v2/organization-sources/{uid}/records/", paths) def test_retrieve_returns_item_by_uid(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Ромашка"', inn="7712345678", kpp="771201001", @@ -224,7 +227,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["name"], 'ООО "Ромашка"') def test_response_includes_normalized_name(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='АКЦИОНЕРНОЕ ОБЩЕСТВО "СЕВЕРНЫЙ МОСТ"', inn="7712345679", kpp="771201002", @@ -242,7 +245,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["normalized_name"], 'АО "Северный Мост"') def test_list_returns_compact_source_summaries_without_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Легкий список"', inn="7712345682", kpp="771201005", @@ -279,7 +282,7 @@ class OrganizationsApiV2Test(APITestCase): ) def test_list_source_summaries_do_not_load_source_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Легкие источники"', inn="7712345685", kpp="771201008", @@ -316,7 +319,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(source_record_queries, []) def test_source_records_endpoint_returns_requested_source_payload(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Явные данные"', full_name='Общество с ограниченной ответственностью "Явные данные"', short_name='ООО "Явные данные"', @@ -387,7 +390,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(embedded_organization["is_branch"], False) def test_detail_returns_compact_source_summaries_by_default(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Полная карточка"', inn="7712345684", kpp="771201007", @@ -410,13 +413,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["sources"][0]["source_group"], "financial_indicators") def test_source_group_filter_limits_organizations_by_source_extension(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Сводка данных"', inn="7712345681", kpp="771201004", ogrn="1027700132199", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Без промышленности"', inn="7712345686", kpp="771201009", @@ -446,7 +449,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["data"][0]["uid"], str(organization.uid)) def test_normalized_name_compacts_scientific_production_forms(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='Научно-Производственное Предприятие "ИМПУЛЬС"', inn="7712345680", kpp="771201003", @@ -463,13 +466,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["normalized_name"], 'НПП "Импульс"') def test_list_filters_by_identifiers_and_searches_by_name(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Северный мост"', inn="7711111111", kpp="771101001", ogrn="1027700132111", ) - Organization.objects.create( + create_frontend_organization( name='АО "Южный путь"', inn="7722222222", kpp="772201001", @@ -494,13 +497,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(by_search.data["data"][0]["uid"], str(target.uid)) def test_list_searches_by_identifier_fragments_for_dashboard(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Поиск по реквизитам"', inn="7711111122", kpp="771102222", ogrn="1027700132122", ) - Organization.objects.create( + create_frontend_organization( name='ООО "Другая организация"', inn="7722222233", kpp="772203333", @@ -520,7 +523,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["data"][0]["uid"], str(target.uid)) def test_list_response_is_cached_by_query_string(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Кеш"', inn="7733333333", kpp="773301001", @@ -532,7 +535,7 @@ class OrganizationsApiV2Test(APITestCase): url, {"inn": "7733333333", "has_registry": "false"}, ) - Organization.objects.create( + create_frontend_organization( name='ООО "После кеша"', inn="7744444444", kpp="774401001", @@ -555,7 +558,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(different_query_response.data["data"][0]["inn"], "7744444444") def test_list_response_cache_is_invalidated_by_version_bump(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Версия кеша"', inn="7744444445", kpp="774401002", @@ -575,7 +578,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(third_response["X-Cache"], "MISS") def test_source_update_invalidates_organization_cache(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Источник сброса"', inn="7744444446", kpp="774401003", @@ -599,7 +602,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(third_response["X-Cache"], "MISS") def test_explicit_invalidation_refreshes_organization_cache(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Реестр сброса"', inn="7744444447", kpp="774401004", @@ -618,7 +621,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(third_response["X-Cache"], "MISS") def test_retrieve_response_is_cached(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Деталь"', inn="7755555555", kpp="775501001", @@ -639,7 +642,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(second_response.data["name"], 'ООО "Деталь"') def test_list_includes_source_groups_and_active_registries(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Данные"', inn="7777777777", kpp="777701001", @@ -783,8 +786,8 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(records_response.data["data"][0]["external_id"], "fin-1") self.assertEqual(len(records_response.data["data"][0]["financial_lines"]), 3) - def test_filters_by_registry_and_has_registry(self): - with_registry = Organization.objects.create( + def test_frontend_scope_cannot_be_lifted_by_has_registry_parameter(self): + with_registry = create_frontend_organization( name='ООО "В реестре"', inn="7788888888", kpp="778801001", @@ -793,11 +796,32 @@ class OrganizationsApiV2Test(APITestCase): ropk_razdel_num="opk-space", ropk_razdel_name="Роскосмос ОПК", ) - without_registry = Organization.objects.create( + without_registry = create_frontend_organization( name='ООО "Без реестра"', inn="7799999999", kpp="779901001", ogrn="1027700132999", + opk_registry_membership=False, + ) + rosatom_without_opk = create_frontend_organization( + name='АО "Росатом без ОПК"', + inn="7799999998", + kpp="779801001", + ogrn="1027700132998", + gk_code="rosatom", + gk_name="Росатом", + goz_participation=True, + opk_registry_membership=False, + ) + roscosmos_without_opk = create_frontend_organization( + name='АО "Роскосмос без ОПК"', + inn="7799999997", + kpp="779701001", + ogrn="1027700132997", + gk_code="roscosmos", + gk_name="Роскосмос", + ropk_num="historical-number-without-membership", + opk_registry_membership=False, ) by_registry = self.client.get( reverse("api_v2:organizations:organizations-list"), @@ -820,22 +844,31 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(has_registry.data["data"][0]["uid"], str(with_registry.uid)) self.assertEqual(no_registry.data["meta"]["pagination"]["total_count"], 1) - self.assertEqual(no_registry.data["data"][0]["uid"], str(without_registry.uid)) - - def test_has_registry_filter_uses_uncorrelated_identity_subqueries(self): - filterset = OrganizationFilter( - data={"has_registry": "true"}, - queryset=Organization.objects.all(), + self.assertEqual(no_registry.data["data"][0]["uid"], str(with_registry.uid)) + returned_uids = {item["uid"] for item in no_registry.data["data"]} + self.assertTrue( + returned_uids.isdisjoint( + { + str(without_registry.uid), + str(rosatom_without_opk.uid), + str(roscosmos_without_opk.uid), + } + ) ) - self.assertTrue(filterset.is_valid(), filterset.errors) - sql = str(filterset.qs.query).upper() + hidden_detail = self.client.get( + reverse( + "api_v2:organizations:organizations-detail", + args=[rosatom_without_opk.uid], + ) + ) + self.assertEqual(hidden_detail.status_code, status.HTTP_404_NOT_FOUND) - self.assertIn("OPK_REGISTRY_MEMBERSHIP", sql) - self.assertNotIn("EXISTS", sql) + def test_has_registry_is_not_a_public_filter(self): + self.assertNotIn("has_registry", OrganizationFilter.base_filters) def test_list_defaults_to_has_registry_true(self): - with_registry = Organization.objects.create( + with_registry = create_frontend_organization( name='ООО "Дефолтный реестр"', inn="7800000101", kpp="780001101", @@ -844,11 +877,21 @@ class OrganizationsApiV2Test(APITestCase): ropk_razdel_num="default", ropk_razdel_name="Дефолтный реестр", ) - Organization.objects.create( + create_frontend_organization( name='ООО "Скрыто по дефолту"', inn="7800000102", kpp="780001102", ogrn="1027700133102", + opk_registry_membership=False, + ) + create_frontend_organization( + name='АО "Росатом без ОПК скрыт по дефолту"', + inn="7800000103", + kpp="780001103", + ogrn="1027700133103", + gk_name="Росатом", + goz_participation=True, + opk_registry_membership=False, ) default_response = self.client.get( reverse("api_v2:organizations:organizations-list") @@ -869,24 +912,24 @@ class OrganizationsApiV2Test(APITestCase): 1, ) self.assertEqual( - explicit_false_response.data["data"][0]["inn"], - "7800000102", + {item["inn"] for item in explicit_false_response.data["data"]}, + {"7800000101"}, ) def test_filters_by_data_presence(self): - with_industrial = Organization.objects.create( + with_industrial = create_frontend_organization( name='ООО "С промышленностью"', inn="7800000001", kpp="780001001", ogrn="1027700133001", ) - with_fns = Organization.objects.create( + with_fns = create_frontend_organization( name='ООО "С отчетностью"', inn="7800000002", kpp="780001002", ogrn="1027700133002", ) - without_data = Organization.objects.create( + without_data = create_frontend_organization( name='ООО "Без данных"', inn="7800000003", kpp="780001003", @@ -937,7 +980,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(has_fns.data["data"][0]["uid"], str(with_fns.uid)) def test_has_fns_reports_filter_does_not_preload_report_identities(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Отчетность без preload"', inn="7800000004", kpp="780001004", @@ -966,13 +1009,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(distinct_report_queries, []) def test_sources_action_returns_only_current_organization_extensions(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Источник"', inn="7800000201", kpp="780002101", ogrn="1027700133201", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Другой источник"', inn="7800000205", kpp="780002105", @@ -1010,7 +1053,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(sources["industrial_production"]["uid"], str(industrial.uid)) def test_unknown_source_group_filter_returns_empty_page(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Неверная группа"', inn="7800000202", kpp="780002102", @@ -1026,7 +1069,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["meta"]["pagination"]["total_count"], 0) def test_detail_source_summaries_do_not_load_source_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Без N+1"', inn="7800000203", kpp="780002103", @@ -1084,7 +1127,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(source_record_queries, []) def test_trudvsem_filter_alias_uses_vacancies_source_group(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Вакансии"', inn="7800000204", kpp="780002104", @@ -1128,7 +1171,7 @@ class OrganizationsApiV2Test(APITestCase): @override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True) def test_dev_flag_allows_anonymous_access(self): self.client.force_authenticate(user=None) - Organization.objects.create( + create_frontend_organization( name='ООО "Без токена"', inn="7766666666", kpp="776601001", diff --git a/tests/apps/organizations/test_api_v2_source_extensions.py b/tests/apps/organizations/test_api_v2_source_extensions.py index 97ed2ab..6adb849 100644 --- a/tests/apps/organizations/test_api_v2_source_extensions.py +++ b/tests/apps/organizations/test_api_v2_source_extensions.py @@ -13,6 +13,12 @@ from rest_framework.test import APITestCase from tests.apps.user.factories import UserFactory +def create_frontend_organization(**kwargs): + """Create an organization visible through the frontend API by default.""" + kwargs.setdefault("opk_registry_membership", True) + return Organization.objects.create(**kwargs) + + class OrganizationSourceExtensionsApiV2Test(APITestCase): """Checks organization-centric source extension API contract.""" @@ -22,7 +28,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.client.force_authenticate(self.user) def test_list_returns_compact_source_summaries_instead_of_embedded_data(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "API"', inn="7707083810", ogrn="1027700132010", @@ -56,7 +62,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(item["sources"][0]["records_count"], 1) def test_organization_sources_action_returns_extensions(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Sources"', inn="7707083811", ogrn="1027700132011", @@ -77,7 +83,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(response.data[0]["uid"], str(extension.uid)) def test_source_records_endpoint_returns_extension_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Records"', inn="7707083812", ogrn="1027700132012", @@ -110,12 +116,12 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): ) def test_flat_source_records_endpoint_filters_by_source_group(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Flat Records"', inn="7707083813", ogrn="1027700132013", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Other Records"', inn="7707083814", ogrn="1027700132014", @@ -160,18 +166,19 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(record["source_group"], "planned_inspections") self.assertEqual(record["organization"]["uid"], str(target.uid)) - def test_flat_source_records_endpoint_filters_by_has_registry(self): - with_registry = Organization.objects.create( + def test_flat_source_records_scope_cannot_be_lifted_by_has_registry(self): + with_registry = create_frontend_organization( name='ООО "With Registry Source"', inn="7707083815", ogrn="1027700132015", opk_registry_membership=True, ropk_razdel_num="source-records", ) - without_registry = Organization.objects.create( + without_registry = create_frontend_organization( name='ООО "Without Registry Source"', inn="7707083816", ogrn="1027700132016", + opk_registry_membership=False, ) with_registry_extension = PlannedInspectionExtension.objects.create( organization=with_registry, @@ -225,11 +232,11 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): ) self.assertEqual( without_registry_response.data["data"][0]["external_id"], - "INSP-WITHOUT-REGISTRY", + "INSP-WITH-REGISTRY", ) def test_source_record_organization_uses_active_registry_identity(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Реестровое имя"', inn="1800020960", kpp="180001001", @@ -266,7 +273,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(response_organization["ogrn"], organization.ogrn) def test_source_record_organization_keeps_canonical_inn_with_leading_zero(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Башнефть-Добыча"', inn="0277106840", ogrn="1090280032699", @@ -298,12 +305,12 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(response_organization["ogrn"], organization.ogrn) def test_flat_source_records_searches_payload_values_displayed_in_tables(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Поиск по payload"', inn="7707083817", ogrn="1027700132017", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Другая payload"', inn="7707083818", ogrn="1027700132018", @@ -363,7 +370,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): ) def test_flat_source_records_searches_registry_identity_displayed_in_tables(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='АО "Реестровый поиск"', inn="1800020961", kpp="180001002", diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py new file mode 100644 index 0000000..65d525e --- /dev/null +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -0,0 +1,168 @@ +"""Tests for deterministic frontend demo company management commands.""" + +from io import StringIO + +from apps.exchange.state_corp_services import StateCorpExchangeService +from apps.parsers.models import ParserLoadLog +from django.core.management import call_command +from django.test import TestCase, override_settings +from organizations.models import ( + Organization, + OrganizationSourceExtension, + OrganizationSourceFinancialLine, + OrganizationSourceRecord, + SourceGroup, +) + +TEST_EXCHANGE_TOKEN = "test-exchange-token" # noqa: S105 + + +class TestCompaniesCommandsTest(TestCase): + """Checks creation, refresh, and removal of the fixed demo dataset.""" + + def test_create_builds_twenty_companies_with_every_source_dataset(self): + call_command("create_test_companies", stdout=StringIO()) + + companies = Organization.objects.filter(name__startswith="Тестовая компания ") + self.assertEqual(companies.count(), 20) + self.assertEqual( + set(companies.values_list("name", flat=True)), + {f"Тестовая компания {index}" for index in range(1, 21)}, + ) + self.assertEqual(companies.filter(opk_registry_membership=True).count(), 20) + self.assertEqual( + companies.filter( + gk_code="2", + gk_name='Госкорпорация "Росатом"', + ).count(), + 10, + ) + self.assertEqual( + companies.filter( + gk_code="1", + gk_name='Госкорпорация "Роскосмос"', + ).count(), + 10, + ) + + expected_groups = {choice.value for choice in SourceGroup} + expected_sources = {choice.value for choice in ParserLoadLog.Source} + for company in companies: + self.assertEqual( + set(company.source_extensions.values_list("source_group", flat=True)), + expected_groups, + ) + self.assertEqual( + set( + OrganizationSourceRecord.objects.filter( + extension__organization=company + ).values_list("source", flat=True) + ), + expected_sources, + ) + + self.assertEqual( + OrganizationSourceExtension.objects.filter( + organization__in=companies + ).count(), + 20 * len(expected_groups), + ) + self.assertEqual( + OrganizationSourceRecord.objects.filter( + extension__organization__in=companies + ).count(), + 20 * len(expected_sources), + ) + self.assertEqual( + OrganizationSourceFinancialLine.objects.filter( + source_record__extension__organization__in=companies + ).count(), + 20 * 4, + ) + + def test_create_updates_the_fixed_dataset_without_duplicates(self): + call_command("create_test_companies", stdout=StringIO()) + company_uids = set( + Organization.objects.filter( + name__startswith="Тестовая компания " + ).values_list("uid", flat=True) + ) + first_company = Organization.objects.get(name="Тестовая компания 1") + first_company.name = "Поврежденное тестовое имя" + first_company.gk_name = "Неверная корпорация" + first_company.save(update_fields=["name", "gk_name"]) + record = OrganizationSourceRecord.objects.filter( + extension__organization=first_company, + source=ParserLoadLog.Source.INSPECTIONS, + ).get() + record.title = "Устаревшие тестовые данные" + record.payload = {"stale": True} + record.save(update_fields=["title", "payload"]) + + call_command("create_test_companies", stdout=StringIO()) + + refreshed_uids = set( + Organization.objects.filter( + name__startswith="Тестовая компания " + ).values_list("uid", flat=True) + ) + self.assertEqual(refreshed_uids, company_uids) + self.assertEqual(len(refreshed_uids), 20) + first_company.refresh_from_db() + self.assertEqual(first_company.name, "Тестовая компания 1") + self.assertEqual(first_company.gk_name, 'Госкорпорация "Росатом"') + record.refresh_from_db() + self.assertNotEqual(record.title, "Устаревшие тестовые данные") + self.assertIn("registration_number", record.payload) + self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15) + + @override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN) + def test_created_source_records_are_exported_to_state_corp_package(self): + call_command("create_test_companies", stdout=StringIO()) + + company_inns = list( + Organization.objects.filter(name__startswith="Тестовая компания ") + .order_by("rn") + .values_list("inn", flat=True) + ) + package = StateCorpExchangeService.build_package( + organization_inns=company_inns + ) + + self.assertEqual( + package.payload_counts, + { + "organizations": 20, + "industrial_certificates": 20, + "manufacturers": 20, + "industrial_products": 20, + "prosecutor_checks": 20, + "public_procurements": 80, + "financial_reports": 20, + "arbitration_cases": 20, + "bankruptcy_procedures": 20, + "defense_unreliable_suppliers": 40, + "information_security_registries": 20, + "labor_vacancies": 20, + }, + ) + + def test_delete_removes_only_the_fixed_twenty_companies(self): + untouched = Organization.objects.create( + name="Тестовая компания 99", + inn="7700000099", + ogrn="1027700000099", + opk_registry_membership=True, + ) + call_command("create_test_companies", stdout=StringIO()) + + call_command("delete_test_companies", stdout=StringIO()) + + self.assertFalse( + Organization.objects.filter( + name__in=[f"Тестовая компания {index}" for index in range(1, 21)] + ).exists() + ) + self.assertTrue(Organization.objects.filter(uid=untouched.uid).exists()) + self.assertEqual(OrganizationSourceExtension.objects.count(), 0) + self.assertEqual(OrganizationSourceRecord.objects.count(), 0) diff --git a/tests/apps/parsers/test_checko.py b/tests/apps/parsers/test_checko.py index 11f7038..46f9f48 100644 --- a/tests/apps/parsers/test_checko.py +++ b/tests/apps/parsers/test_checko.py @@ -493,6 +493,41 @@ class CheckoClientApiTest(SimpleTestCase): ) ) + def test_http_403_quota_error_keeps_checko_rate_limit_classification(self): + with TestHTTPServer() as server: + server.add_json( + "/v2/company", + { + "meta": { + "status": "error", + "message": ( + "Превышен суточный лимит запросов для бесплатного тарифа" + ), + "balance": 0, + "today_request_count": 100, + } + }, + status=403, + ) + client = _client_for(server) + with self.assertRaises(CheckoRateLimitError) as context: + client.get_company(CompanyRequest(inn=_digits(10))) + + self.assertEqual(context.exception.status_code, 403) + self.assertEqual(context.exception.balance, 0) + self.assertEqual(context.exception.request_count, 100) + + def test_non_quota_http_error_remains_connection_error(self): + with TestHTTPServer() as server: + server.add_json( + "/v2/company", + {"meta": {"status": "error", "message": "Доступ запрещён"}}, + status=403, + ) + client = _client_for(server) + with self.assertRaises(CheckoConnectionError): + client.get_company(CompanyRequest(inn=_digits(10))) + def test_not_found_error_handling(self): with TestHTTPServer() as server: server.add_json( diff --git a/tests/apps/parsers/test_fns_api_client.py b/tests/apps/parsers/test_fns_api_client.py new file mode 100644 index 0000000..5af5020 --- /dev/null +++ b/tests/apps/parsers/test_fns_api_client.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from urllib.parse import parse_qs + +from apps.parsers.clients.fns import FNSApiClient, FNSApiClientError +from django.test import SimpleTestCase + +from tests.utils import Response +from tests.utils import TestHTTPServer as HTTPTestServer + + +class FNSApiClientTest(SimpleTestCase): + def test_fetch_reports_by_inn_uses_exact_search_and_maps_financial_lines(self): + search_queries: list[dict[str, list[str]]] = [] + + def search(_request, _body: bytes) -> Response: + search_queries.append(parse_qs(_request.query)) + return Response( + body=( + b'{"content":[' + b'{"id":1,"inn":"7700000000","ogrn":"1027700000000"},' + b'{"id":42,"inn":"7701000001",' + b'"ogrn":"1027700000001"}]}' + ), + headers={"Content-Type": "application/json"}, + ) + + bfo = [ + { + "period": "2025", + "actualBfoDate": "2026-03-10", + "typeCorrections": [ + { + "type": 12, + "correction": { + "id": 555, + "datePresent": "2026-03-10T12:00:00", + "balance": { + "id": 555, + "okud": "0710001", + "current1600": 120, + "previous1600": 100, + "beforePrevious1600": 90, + }, + "financialResult": { + "id": 555, + "okud": "0710002", + "current2110": 300, + "previous2110": 250, + "current2400": 50, + "previous2400": 40, + }, + }, + } + ], + }, + { + "period": "2024", + "typeCorrections": [], + }, + ] + + with HTTPTestServer() as server: + server.add_route("GET", "/advanced-search/organizations/search", search) + server.add_json( + "/nbo/organizations/42", + { + "id": 42, + "inn": "7701000001", + "ogrn": "1027700000001", + "fullName": "АО Завод", + }, + ) + server.add_json("/nbo/organizations/42/bfo/", bfo) + with FNSApiClient( + base_url=server.base_url, + max_periods=1, + http_adapter=server.adapter, + ) as client: + self.assertIn( + "Mozilla/5.0", client.http_client.session.headers["User-Agent"] + ) + reports = client.fetch_reports_by_inn("7701000001") + + self.assertEqual( + search_queries, + [{"query": ["7701000001"], "page": ["0"], "size": ["10"]}], + ) + self.assertEqual(len(reports), 1) + report = reports[0] + self.assertEqual(report.external_id, "girbo:555") + self.assertEqual(report.ogrn, "1027700000001") + self.assertEqual(report.year, 2025) + self.assertEqual(report.period_type, 12) + self.assertEqual( + [ + ( + line.form_code, + line.line_code, + line.period_start, + line.period_end, + ) + for line in report.lines + ], + [ + ("1", "1600", 100, 120), + ("2", "2110", 250, 300), + ("2", "2400", 40, 50), + ], + ) + kwargs = report.to_save_report_kwargs(batch_id=7) + self.assertEqual(kwargs["source"], "api") + self.assertEqual(kwargs["batch_id"], 7) + self.assertEqual(kwargs["file_hash"], report.file_hash) + self.assertEqual(len(kwargs["lines_data"]), 3) + + def test_fetch_reports_by_organization_id_does_not_run_search(self): + with HTTPTestServer() as server: + server.add_json( + "/nbo/organizations/42", + { + "id": 42, + "inn": "7701000001", + "ogrn": "1027700000001", + "fullName": "АО Завод", + }, + ) + server.add_json("/nbo/organizations/42/bfo/", []) + client = FNSApiClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + reports = client.fetch_reports_by_organization_id(42) + + self.assertEqual(reports, []) + + def test_fetch_reports_by_inn_fails_when_exact_match_is_absent(self): + with HTTPTestServer() as server: + server.add_json( + "/advanced-search/organizations/search", + {"content": [{"id": 1, "inn": "7700000000"}]}, + ) + client = FNSApiClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage(FNSApiClientError, "not found"): + client.fetch_reports_by_inn("7701000001") + + def test_rejects_invalid_inn_before_request(self): + client = FNSApiClient() + + with self.assertRaisesMessage(ValueError, "10 or 12 digits"): + client.fetch_reports_by_inn("not-an-inn") diff --git a/tests/apps/parsers/test_gisp_client.py b/tests/apps/parsers/test_gisp_client.py new file mode 100644 index 0000000..0c59d8f --- /dev/null +++ b/tests/apps/parsers/test_gisp_client.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import json + +from apps.parsers.clients.gisp import GispProductsClient, GispProductsClientError +from django.test import SimpleTestCase + +from tests.utils import Response +from tests.utils import TestHTTPServer as HTTPTestServer + + +def _gisp_item(number: int) -> dict: + return { + "_product_reg_number_2023": str(number), + "_product_reg_number_2022": f"old-{number}", + "_res_number": f"resolution-{number}", + "_org_inn": "7701000001", + "_org_ogrn": "1027700000001", + "_org_name": "АО Завод", + "_product_name": f"Станок {number}", + "_product_spec": f"Модель {number}", + "_product_okpd2": "28.41.1", + "_product_tnved": "8456", + "_basedondoc_name": "Заключение", + "_basedondoc_num": "42", + "_basedondoc_date": "2026-01-15", + } + + +class GispProductsClientTest(SimpleTestCase): + def test_rejects_page_size_above_safe_bound(self): + with self.assertRaisesMessage( + ValueError, + "page_size must be between 1 and 100", + ): + GispProductsClient(page_size=101) + + def test_rejects_max_pages_above_safe_bound(self): + with self.assertRaisesMessage( + ValueError, + "max_pages must be between 1 and 100", + ): + GispProductsClient(max_pages=101) + + def test_rejects_max_records_above_safe_bound(self): + with self.assertRaisesMessage( + ValueError, + "max_records must be between 1 and 10000", + ): + GispProductsClient(max_records=10_001) + + def test_fetch_products_follows_skip_pagination_and_maps_items(self): + requests: list[dict] = [] + + def handle_page(_request, body: bytes) -> Response: + payload = json.loads(body) + requests.append(payload) + options = payload["opt"] + skip = options["skip"] + take = options["take"] + items = [_gisp_item(number) for number in range(skip + 1, 6)][:take] + return Response( + body=json.dumps( + {"ok": True, "total_count": 5, "items": items} + ).encode(), + headers={"Content-Type": "application/json"}, + ) + + with HTTPTestServer() as server: + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_page) + with GispProductsClient( + base_url=server.base_url, + page_size=2, + max_pages=10, + max_records=5, + http_adapter=server.adapter, + ) as client: + result = client.fetch_products() + + self.assertEqual( + [payload["opt"] for payload in requests], + [ + {"skip": 0, "take": 2, "requireTotalCount": True}, + {"skip": 2, "take": 2, "requireTotalCount": True}, + {"skip": 4, "take": 1, "requireTotalCount": True}, + ], + ) + self.assertEqual(result.total_count, 5) + self.assertEqual(result.pages_fetched, 3) + self.assertEqual(result.items_fetched, 5) + self.assertFalse(result.truncated) + self.assertEqual(len(result.products), 5) + self.assertEqual(result.products[0].registry_number, "1") + self.assertEqual(result.products[0].inn, "7701000001") + self.assertEqual(result.products[0].product_name, "Станок 1") + self.assertEqual( + result.products[0].regulatory_document, + "Заключение | 42 | 2026-01-15", + ) + + def test_fetch_products_stops_at_max_pages_and_reports_truncation(self): + calls = 0 + + def handle_page(_request, body: bytes) -> Response: + nonlocal calls + calls += 1 + options = json.loads(body)["opt"] + items = [ + _gisp_item(number) + for number in range(options["skip"] + 1, options["skip"] + 3) + ] + return Response( + body=json.dumps( + {"ok": True, "total_count": 100, "items": items} + ).encode(), + headers={"Content-Type": "application/json"}, + ) + + with HTTPTestServer() as server: + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_page) + with GispProductsClient( + base_url=server.base_url, + page_size=2, + max_pages=2, + max_records=100, + http_adapter=server.adapter, + ) as client: + result = client.fetch_products() + + self.assertEqual(calls, 2) + self.assertEqual(result.items_fetched, 4) + self.assertTrue(result.truncated) + + def test_fetch_products_rejects_invalid_response_shape(self): + with HTTPTestServer() as server: + server.add_route( + "POST", + "/pp719v2/pub/prod/b/", + lambda _request, _body: Response( + body=b'{"ok":true,"total_count":1}', + headers={"Content-Type": "application/json"}, + ), + ) + client = GispProductsClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage( + GispProductsClientError, + "returned no items list", + ): + client.fetch_products() + + def test_fetch_products_rejects_empty_page_before_known_total(self): + with HTTPTestServer() as server: + server.add_route( + "POST", + "/pp719v2/pub/prod/b/", + lambda _request, _body: Response( + body=b'{"ok":true,"total_count":10,"items":[]}', + headers={"Content-Type": "application/json"}, + ), + ) + client = GispProductsClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage( + GispProductsClientError, + "empty page before total_count", + ): + client.fetch_products() diff --git a/tests/apps/parsers/test_parser_schedules.py b/tests/apps/parsers/test_parser_schedules.py new file mode 100644 index 0000000..2f23ee5 --- /dev/null +++ b/tests/apps/parsers/test_parser_schedules.py @@ -0,0 +1,104 @@ +import importlib + +from django.apps import apps +from django.test import TestCase +from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask + +schedule_migration = importlib.import_module( + "apps.parsers.migrations.0025_consolidate_inspections_schedule" +) + + +class DefaultParserScheduleTest(TestCase): + def test_inspections_have_one_canonical_sync_schedule(self): + crontab = CrontabSchedule.objects.create( + minute="0", + hour="0", + day_of_week="6", + day_of_month="*", + month_of_year="*", + timezone="Europe/Moscow", + ) + interval = IntervalSchedule.objects.create( + every=7, + period=IntervalSchedule.DAYS, + ) + PeriodicTask.objects.create( + name="parser:inspections:weekly-saturday-msk", + task="apps.parsers.tasks.parse_inspections", + crontab=crontab, + enabled=True, + ) + PeriodicTask.objects.create( + name="parse-inspections-weekly", + task="apps.parsers.tasks.parse_inspections", + interval=interval, + enabled=True, + ) + PeriodicTask.objects.create( + name="parser:sync_inspections:weekly-saturday-msk", + task="apps.parsers.tasks.parse_inspections", + args='["stale"]', + kwargs='{"year": 2020}', + crontab=crontab, + enabled=True, + ) + + schedule_migration.consolidate_inspections_schedule(apps, None) + schedule_migration.consolidate_inspections_schedule(apps, None) + + schedule_names = { + "parser:inspections:weekly-saturday-msk", + "parser:sync_inspections:weekly-saturday-msk", + "parse-inspections-weekly", + } + + schedules = list( + PeriodicTask.objects.filter(name__in=schedule_names) + .order_by("name") + .values_list("name", "task", "enabled", "args", "kwargs") + ) + + self.assertEqual( + schedules, + [ + ( + "parser:sync_inspections:weekly-saturday-msk", + "apps.parsers.tasks.sync_inspections", + True, + "[]", + "{}", + ) + ], + ) + + def test_fns_weekly_schedule_runs_bounded_api_sync(self): + fns_schedule_migration = importlib.import_module( + "apps.parsers.migrations.0026_update_fns_financial_schedule" + ) + interval = IntervalSchedule.objects.create( + every=7, + period=IntervalSchedule.DAYS, + ) + PeriodicTask.objects.create( + name="parser:fns_financial:weekly-saturday-msk", + task="apps.parsers.tasks.scan_fns_directory", + args='["stale"]', + kwargs='{"stale": true}', + interval=interval, + enabled=True, + ) + + fns_schedule_migration.update_fns_financial_schedule(apps, None) + fns_schedule_migration.update_fns_financial_schedule(apps, None) + + task = PeriodicTask.objects.get(name="parser:fns_financial:weekly-saturday-msk") + self.assertEqual( + task.task, + "apps.parsers.tasks.sync_fns_financial_reports", + ) + self.assertEqual(task.args, "[]") + self.assertEqual(task.kwargs, "{}") + self.assertTrue(task.enabled) + self.assertIsNotNone(task.crontab_id) + self.assertIsNone(task.interval_id) diff --git a/tests/apps/parsers/test_proverki_client.py b/tests/apps/parsers/test_proverki_client.py index a8b6aae..2bd8a65 100644 --- a/tests/apps/parsers/test_proverki_client.py +++ b/tests/apps/parsers/test_proverki_client.py @@ -332,6 +332,25 @@ class ProverkiDownloadParseTest(SimpleTestCase): ) self.assertEqual(len(inspections), 1) + def test_download_and_parse_closes_playwright_when_portal_fails(self): + class _PortalClient(ProverkiClient): + playwright_closed = False + + def _download_from_portal(self, *args, **kwargs): # type: ignore[override] + raise RuntimeError("navigation failed") + + def _close_playwright(self): # type: ignore[override] + self.playwright_closed = True + + client = _PortalClient(use_playwright=True) + + with self.assertRaises(RuntimeError): + client._download_and_parse( + "http://portal.example.com", file_format="portal" + ) + + self.assertTrue(client.playwright_closed) + def test_download_from_portal_does_not_wait_for_networkidle(self): archive = build_zip( [("data.xml", _xml_with_tag("INSPECTION", _inspection_attrs()))] @@ -654,6 +673,63 @@ class ProverkiParseXMLTest(SimpleTestCase): self.assertEqual(inspections[0].inspection_type, inspection_type) self.assertEqual(inspections[0].status, status) + def test_parse_xml_record_fz248_current_production_shape(self): + xml = b""" + + + + + + + + +""" + client = ProverkiClient() + + inspections = client._parse_xml_content(xml) + + self.assertEqual(len(inspections), 1) + inspection = inspections[0] + self.assertEqual(inspection.registration_number, "248000000000") + self.assertEqual(inspection.inn, "7700000000") + self.assertEqual(inspection.ogrn, "1027700000000") + self.assertEqual(inspection.organisation_name, "Example Industrial Company") + self.assertEqual( + inspection.control_authority, + "Example Supervisory Authority", + ) + self.assertEqual(inspection.inspection_type, "Unscheduled inspection") + self.assertEqual(inspection.inspection_form, "On-site inspection") + self.assertEqual(inspection.start_date, "2026-07-01") + self.assertEqual(inspection.end_date, "2026-07-15") + self.assertEqual(inspection.status, "Completed") + + def test_parse_xml_record_fz248_kind_control_is_type_fallback(self): + element = ET.fromstring( # noqa: S314 + """ + + + + """ + ) + client = ProverkiClient() + + inspection = client._parse_xml_record(element) + + self.assertIsNotNone(inspection) + self.assertEqual( + inspection.inspection_type, + "Federal industrial supervision", + ) + def test_parse_xml_record_namespace_text_child_fallback(self): ns = "http://example.com/ns" inn = _digits(10) @@ -1171,3 +1247,42 @@ class ProverkiPlaywrightStubTest(SimpleTestCase): sys.modules.pop("playwright.sync_api", None) else: sys.modules["playwright.sync_api"] = original_module + + def test_get_browser_launch_error_stops_playwright(self): + class _BrokenChromium: + def launch(self, **_kwargs): + raise RuntimeError("browser missing") + + class _FakePlaywright: + chromium = _BrokenChromium() + + def __init__(self): + self.stopped = False + + def stop(self): + self.stopped = True + + playwright = _FakePlaywright() + + class _FakeSyncPlaywright: + def start(self): + return playwright + + fake_module = types.SimpleNamespace( + sync_playwright=lambda: _FakeSyncPlaywright() + ) + client = ProverkiClient() + original_module = sys.modules.get("playwright.sync_api") + sys.modules["playwright.sync_api"] = fake_module + try: + with self.assertRaises(ProverkiClientError): + client._get_browser() + finally: + if original_module is None: + sys.modules.pop("playwright.sync_api", None) + else: + sys.modules["playwright.sync_api"] = original_module + + self.assertTrue(playwright.stopped) + self.assertIsNone(client._playwright) + self.assertIsNone(client._browser) diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index 55d21eb..a6049e8 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -176,6 +176,29 @@ class SourceCardServiceUnitTest(SimpleTestCase): ) self.assertEqual(enqueue_mock.call_count, 3) + @patch( + "apps.parsers.source_cards.SourceCardService._enqueue_task", + return_value={ + "task_id": "task-fns", + "task_name": "apps.parsers.tasks.sync_fns_financial_reports", + }, + ) + def test_refresh_card_for_financial_indicators_uses_api_sync(self, enqueue_mock): + result = SourceCardService.refresh_card( + slug="financial-indicators", + requested_by_id=12, + ) + + self.assertEqual(result["tasks"][0]["task_id"], "task-fns") + self.assertEqual( + enqueue_mock.call_args.kwargs["task_name"], + "apps.parsers.tasks.sync_fns_financial_reports", + ) + self.assertEqual( + enqueue_mock.call_args.kwargs["kwargs"], + {"requested_by_id": 12}, + ) + @patch( "apps.parsers.source_cards.SourceCardService._enqueue_task", return_value={ diff --git a/tests/apps/parsers/test_source_cards_views.py b/tests/apps/parsers/test_source_cards_views.py index da040fa..7b65485 100644 --- a/tests/apps/parsers/test_source_cards_views.py +++ b/tests/apps/parsers/test_source_cards_views.py @@ -3,8 +3,6 @@ from __future__ import annotations import hashlib -from pathlib import Path -from tempfile import TemporaryDirectory from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import ParserLoadLog @@ -236,21 +234,17 @@ class SourceCardsApiTestCase(APITestCase): self.assertIn("status_label", row) self.assertIn("active_tasks", row) + @override_settings(CELERY_TASK_ALWAYS_EAGER=False) def test_refresh_creates_background_job_and_returns_task(self): self.client.force_authenticate(self.admin) - with TemporaryDirectory() as tmp_dir, override_settings( - FNS_WATCH_DIRECTORY=str(Path(tmp_dir) / "watch"), - FNS_PROCESSED_DIRECTORY=str(Path(tmp_dir) / "processed"), - FNS_FAILED_DIRECTORY=str(Path(tmp_dir) / "failed"), - ): - response = self.client.post( - reverse( - "api_v1:sources:source-cards-refresh", - kwargs={"slug": "financial-indicators"}, - ), - {}, - format="json", - ) + response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "financial-indicators"}, + ), + {}, + format="json", + ) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(response.data["status"], "accepted") @@ -260,7 +254,7 @@ class SourceCardsApiTestCase(APITestCase): self.assertTrue( BackgroundJob.objects.filter( task_id=task_id, - task_name="apps.parsers.tasks.scan_fns_directory", + task_name="apps.parsers.tasks.sync_fns_financial_reports", user_id=self.admin.id, ).exists() ) diff --git a/tests/apps/parsers/test_source_registry.py b/tests/apps/parsers/test_source_registry.py new file mode 100644 index 0000000..fd47794 --- /dev/null +++ b/tests/apps/parsers/test_source_registry.py @@ -0,0 +1,42 @@ +from apps.parsers import tasks +from apps.parsers.clients.common.structured import MAX_FILE_SIZE_BYTES +from apps.parsers.source_registry import PARSER_SOURCES +from apps.parsers.views import TASKS_BY_NAME +from django.conf import settings +from django.test import SimpleTestCase + + +class ParserSourceRegistryEisTest(SimpleTestCase): + def test_procurement_urls_select_the_expected_law(self): + common_search_url = ( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html" + ) + + self.assertEqual( + PARSER_SOURCES["procurements_44fz"].upstream_url, + f"{common_search_url}?fz44=on", + ) + self.assertEqual( + PARSER_SOURCES["procurements_223fz"].upstream_url, + f"{common_search_url}?fz223=on", + ) + + +class ParserSourceRegistryFNSTest(SimpleTestCase): + def test_request_body_limit_allows_maximum_registry_archive(self): + self.assertGreater(settings.DATA_UPLOAD_MAX_MEMORY_SIZE, MAX_FILE_SIZE_BYTES) + + def test_fns_refresh_uses_bounded_api_task_and_preserves_file_upload(self): + source = PARSER_SOURCES["fns_financial"] + + self.assertEqual( + source.task_name, + "apps.parsers.tasks.sync_fns_financial_reports", + ) + self.assertEqual(source.parser_strategy, "fns_bfo_bounded_registry_sync") + self.assertTrue(source.supports_file_upload) + self.assertNotIn("query=", source.upstream_url) + self.assertIs( + TASKS_BY_NAME[source.task_name], + tasks.sync_fns_financial_reports, + ) diff --git a/tests/apps/parsers/test_structured_data_client.py b/tests/apps/parsers/test_structured_data_client.py index 4f506de..6fb65b0 100644 --- a/tests/apps/parsers/test_structured_data_client.py +++ b/tests/apps/parsers/test_structured_data_client.py @@ -1,7 +1,10 @@ from __future__ import annotations from apps.parsers.clients.base import HTTPClientError -from apps.parsers.clients.common.structured import StructuredDataClient +from apps.parsers.clients.common.structured import ( + EIS_CA_BUNDLE_PATH, + StructuredDataClient, +) from django.test import SimpleTestCase @@ -18,7 +21,84 @@ class _FakeHTTPClient: return response +class _FakeGispHTTPClient: + def __init__(self) -> None: + self.endpoint = "" + self.payload: dict | None = None + + def post_json( + self, + endpoint: str, + *, + payload: dict | None = None, + **_kwargs, + ) -> dict: + self.endpoint = endpoint + self.payload = payload + return {"items": [{"res_number": "123", "_product_name": "Станок"}]} + + +class StructuredDataClientGispTest(SimpleTestCase): + def test_products_request_omits_unsupported_res_date_sort(self): + client = StructuredDataClient(source="mpt_products") + fake_http_client = _FakeGispHTTPClient() + client._http_client = fake_http_client + + records = client.fetch_records(file_url="https://gisp.gov.ru/pp719v2/pub/prod/") + + self.assertEqual(len(records), 1) + self.assertEqual(records[0].external_id, "123") + self.assertEqual( + fake_http_client.endpoint, + "https://gisp.gov.ru/pp719v2/pub/prod/b/", + ) + self.assertEqual( + fake_http_client.payload, + { + "opt": { + "skip": 0, + "take": 100, + "requireTotalCount": True, + } + }, + ) + + +class StructuredDataClientFasGozParsingTest(SimpleTestCase): + def test_person_name_populates_record_title(self): + client = StructuredDataClient(source="fas_goz") + + records = client.fetch_records( + content=""" + + + + + + + + + + + + +
РГОЗ-001ФАС РоссииПостановление № 114.07.2026ИсполненоООО ПРОГРЕССПРОГРЕССг. Москва7707083893
+ """.encode(), + file_name="register.html", + ) + + self.assertEqual(len(records), 1) + self.assertEqual(records[0].organisation_name, "ООО ПРОГРЕСС") + self.assertEqual(records[0].title, "ООО ПРОГРЕСС") + + class StructuredDataClientEisCardEnrichmentTest(SimpleTestCase): + def test_eis_client_uses_project_scoped_trusted_ca_bundle(self): + client = StructuredDataClient(source="procurements_44fz") + + self.assertTrue(EIS_CA_BUNDLE_PATH.is_file()) + self.assertEqual(client.http_client.verify_ssl, str(EIS_CA_BUNDLE_PATH)) + def test_procurement_card_enriches_customer_identity_from_detail_page(self): detail_url = ( "https://zakupki.gov.ru/epz/order/notice/ea20/view/" @@ -104,3 +184,170 @@ class StructuredDataClientEisCardEnrichmentTest(SimpleTestCase): self.assertEqual(records[0].inn, "") self.assertEqual(records[0].ogrn, "") self.assertEqual(records[0].organisation_name, "ФЕДЕРАЛЬНОЕ ГБУ НАУКИ") + + def test_procurement_card_prefers_organization_page_for_identity(self): + detail_url = ( + "https://zakupki.gov.ru/epz/order/notice/zk20/view/" + "common-info.html?regNumber=0322300001726000667" + ) + identity_url = ( + "https://zakupki.gov.ru/epz/organization/view/" + "info.html?organizationCode=03223000017" + ) + client = StructuredDataClient(source="procurements_44fz") + client._http_client = _FakeHTTPClient( + { + identity_url: """ +
Заказчик
+
21.07.2016
+
ОГРН
1022701405737
+
ИНН
2725006476
+
КПП
272501001
+
Полное наименование
КГБУЗ БОЛЬНИЦА
+ """.encode() + } + ) + + records = client.fetch_records( + content=f""" +
+
44-ФЗ
+ № 0322300001726000667 +
Объект закупки
Поставка оборудования
+
Заказчик
КГБУЗ БОЛЬНИЦА
+ КГБУЗ БОЛЬНИЦА +
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual(records[0].url, detail_url) + self.assertEqual(records[0].inn, "2725006476") + self.assertEqual(records[0].ogrn, "1022701405737") + self.assertEqual(records[0].organisation_name, "КГБУЗ БОЛЬНИЦА") + self.assertEqual(client._http_client.downloaded_urls, [identity_url]) + + def test_procurement_card_rejects_untrusted_detail_urls(self): + loopback_identity_url = ( + "http://127.0.0.1/epz/organization/view/" "info.html?inn=2725006476" + ) + foreign_detail_url = ( + "https://example.test/epz/order/notice/zk20/view/" + "common-info.html?regNumber=0322300001726000667" + ) + client = StructuredDataClient(source="procurements_44fz") + client._http_client = _FakeHTTPClient( + { + loopback_identity_url: b"", + foreign_detail_url: b"", + } + ) + + records = client.fetch_records( + content=f""" +
+
44-ФЗ
+ № 0322300001726000667 +
Объект закупки
Поставка оборудования
+
Заказчик
КГБУЗ БОЛЬНИЦА
+ КГБУЗ БОЛЬНИЦА +
+ """.encode(), + file_name="results.html", + ) + + self.assertNotIn("detail_url", records[0].payload) + self.assertNotIn("identity_url", records[0].payload) + self.assertEqual(client._http_client.downloaded_urls, []) + + def test_procurement_card_resolves_relative_official_urls(self): + detail_href = ( + "/epz/order/notice/zk20/view/" + "common-info.html?regNumber=0322300001726000667" + ) + identity_href = "/epz/organization/view/info.html?organizationCode=03223000017" + client = StructuredDataClient( + source="procurements_44fz", + enrich_eis_detail_pages=False, + ) + + records = client.fetch_records( + content=f""" +
+
44-ФЗ
+ № 0322300001726000667 +
Объект закупки
Поставка оборудования
+
Заказчик
КГБУЗ БОЛЬНИЦА
+ КГБУЗ БОЛЬНИЦА +
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual( + records[0].payload["detail_url"], + f"https://zakupki.gov.ru{detail_href}", + ) + self.assertEqual( + records[0].payload["identity_url"], + f"https://zakupki.gov.ru{identity_href}", + ) + + +class StructuredDataClientEisCardParsingTest(SimpleTestCase): + def test_contract_card_prefers_contract_details_over_order_notice(self): + contract_url = ( + "https://zakupki.gov.ru/epz/contract/contractCard/" + "common-info.html?reestrNumber=3310300251025000007" + ) + notice_url = ( + "https://zakupki.gov.ru/epz/order/notice/view/" + "common-info.html?regNumber=0126300014725000080" + ) + client = StructuredDataClient( + source="contracts", + enrich_eis_detail_pages=False, + ) + + records = client.fetch_records( + content=f""" +
+ № 3310300251025000007 +
Заказчик
МБДОУ СКАЗКА
+
Объекты закупки
Мясо птицы
+
Размещен контракт в реестре контрактов
+
04.12.2025
+ Сведения закупки +
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual(records[0].url, contract_url) + self.assertEqual(records[0].organisation_name, "МБДОУ СКАЗКА") + self.assertEqual(records[0].record_date, "04.12.2025") + + def test_unfair_supplier_skips_empty_placed_label(self): + client = StructuredDataClient( + source="unfair_suppliers", + enrich_eis_detail_pages=False, + ) + + records = client.fetch_records( + content=""" +
+
44-ФЗ
+
№ 26007353
+
Размещено
+
Наименование (ФИО) недобросовестного поставщика
+
ООО ПРОГРЕСС
+
ИНН (аналог ИНН)
9728104322
+
Включено
14.07.2026
+
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual(records[0].title, "ООО ПРОГРЕСС") + self.assertEqual(records[0].organisation_name, "ООО ПРОГРЕСС") + self.assertEqual(records[0].record_date, "14.07.2026") diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 9c773f5..05286de 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -4,10 +4,12 @@ from __future__ import annotations import hashlib import io +import json import os import tempfile import threading import unittest +from contextlib import ExitStack from datetime import date from pathlib import Path from types import SimpleNamespace @@ -17,7 +19,9 @@ from urllib.parse import urlparse from apps.core.services import BackgroundJobService from apps.parsers import tasks as parser_tasks from apps.parsers.clients.base import HTTPError +from apps.parsers.clients.checko import CheckoRateLimitError from apps.parsers.clients.common.schemas import GenericParserItem +from apps.parsers.clients.gisp import GispProductsClient, GispProductsClientError from apps.parsers.clients.minpromtorg.industrial import ( IndustrialProductionClient, IndustrialProductionClientError, @@ -26,10 +30,6 @@ from apps.parsers.clients.minpromtorg.manufactures import ( ManufacturesClient, ManufacturesClientError, ) -from apps.parsers.clients.minpromtorg.products import ( - IndustrialProductsClient, - IndustrialProductsClientError, -) from apps.parsers.clients.proverki.client import ProverkiClientError from apps.parsers.clients.zakupki import ZakupkiClientError from apps.parsers.models import ( @@ -38,6 +38,8 @@ from apps.parsers.models import ( ) from apps.parsers.services import FNSReportService, ParserLoadLogService from apps.parsers.tasks import ( + FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT, + FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT, INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS, INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS, _move_to_dir, @@ -55,11 +57,12 @@ from apps.parsers.tasks import ( parse_trudvsem_vacancies, process_fns_files_batch, scan_fns_directory, + sync_fns_financial_reports, sync_inspections, sync_procurements, sync_ru_proxies, ) -from django.test import TestCase, override_settings +from django.test import SimpleTestCase, TestCase, override_settings from openpyxl import Workbook from organizations.models import Organization, OrganizationSourceRecord @@ -71,7 +74,7 @@ from tests.apps.parsers.factories import ( ProxyFactory, ) from tests.apps.parsers.organization_helpers import create_directory_organization -from tests.utils import TestHTTPServer +from tests.utils import Response, TestHTTPServer from tests.utils.fixtures import ( build_minpromtorg_certificates_excel, build_minpromtorg_manufacturers_excel, @@ -277,6 +280,69 @@ class SyncRuProxiesTaskTestCase(TestCase): sync_mock.assert_called_once_with() +class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase): + @override_settings( + CHECKO_API_KEY="test-key", + FEDRESURS_CHECKO_FALLBACK_LIMIT=100, + ) + def test_rate_limit_stops_remaining_checko_lookups(self): + targets = [ + parser_tasks.RegistryLookupTarget( + organization_id=str(index), + inn=f"77010000{index:02d}", + ogrn=f"10277000000{index:02d}", + name=f"Организация {index}", + ) + for index in range(3) + ] + + class _RateLimitedCheckoClient: + calls = 0 + + def __init__(self, **_kwargs): + return + + def get_company(self, _request): + self.__class__.calls += 1 + raise CheckoRateLimitError( + "Превышен суточный лимит запросов", + status_code=403, + ) + + with ( + patch.object( + parser_tasks, + "_active_registry_lookup_targets", + return_value=targets, + ), + patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient), + ): + records = parser_tasks._fetch_checko_bankruptcy_records(proxies=None) + + self.assertEqual(records, []) + self.assertEqual(_RateLimitedCheckoClient.calls, 1) + + def test_non_http_official_error_is_not_silenced_when_fallback_is_empty(self): + with ( + patch.object( + parser_tasks, + "_fetch_structured_records", + side_effect=ValueError("invalid official payload"), + ), + patch.object( + parser_tasks, + "_fetch_checko_bankruptcy_records", + return_value=[], + ), + self.assertRaisesRegex(ValueError, "invalid official payload"), + ): + parser_tasks._fetch_fedresurs_bankruptcy_records( + file_url=None, + file_path=None, + proxies=None, + ) + + class GenericSourceFetchTestCase(TestCase): """Tests for source-specific generic fetch configuration.""" @@ -561,6 +627,52 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(captured_inns, [str(organization.mn_inn)]) self.assertNotIn(str(no_membership.mn_inn), captured_inns) + def test_fedresurs_falls_back_to_checko_when_official_source_is_empty(self): + fallback_records = [SimpleNamespace(source="fedresurs_bankruptcy")] + + with ( + patch.object( + parser_tasks, + "_fetch_structured_records", + return_value=[], + ), + patch.object( + parser_tasks, + "_fetch_checko_bankruptcy_records", + return_value=fallback_records, + ) as fallback, + ): + records = parser_tasks._fetch_fedresurs_bankruptcy_records( + file_url=None, + file_path=None, + proxies=[], + ) + + self.assertIs(records, fallback_records) + fallback.assert_called_once_with(proxies=[]) + + def test_fedresurs_skips_when_official_source_and_fallback_are_empty(self): + with ( + patch.object( + parser_tasks, + "_fetch_structured_records", + return_value=[], + ), + patch.object( + parser_tasks, + "_fetch_checko_bankruptcy_records", + return_value=[], + ), + ): + result = parser_tasks.parse_fedresurs_bankruptcy(proxies=[]) + + log = ParserLoadLog.objects.get( + source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY + ) + self.assertEqual(result["status"], "skipped") + self.assertEqual(log.status, ParserLoadLog.Status.SKIPPED) + self.assertIn("returned no bankruptcy records", log.error_message) + def test_checko_bankruptcy_items_group_messages_by_procedure(self): company = SimpleNamespace( ogrn="1052452047450", @@ -961,6 +1073,14 @@ class GenericSourceFetchTestCase(TestCase): cleanup_mock.assert_called_once_with(max_age_minutes=45) jobs_cleanup_mock.assert_called_once() self.assertEqual(jobs_cleanup_mock.call_args.kwargs["max_age_minutes"], 45) + self.assertIn( + "apps.parsers.tasks.scan_fns_directory", + jobs_cleanup_mock.call_args.kwargs["task_names"], + ) + self.assertIn( + "apps.parsers.tasks.sync_fns_financial_reports", + jobs_cleanup_mock.call_args.kwargs["task_names"], + ) self.assertEqual(result["status"], "success") self.assertEqual(result["marked_failed"], 2) self.assertEqual(result["marked_jobs_failed"], 3) @@ -992,6 +1112,186 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(job.meta["batch_id"], 7) +class FNSApiSyncTaskTestCase(TestCase): + @staticmethod + def _report(*, external_id: str = "girbo:101", lines_count: int = 2): + lines = [SimpleNamespace() for _ in range(lines_count)] + + def _save_kwargs(*, batch_id: int) -> dict: + return { + "external_id": external_id, + "ogrn": "1027700132195", + "file_name": f"{external_id}.json", + "file_hash": "a" * 64, + "source": "api", + "batch_id": batch_id, + "lines_data": [], + } + + return SimpleNamespace( + external_id=external_id, + lines=lines, + to_save_report_kwargs=_save_kwargs, + ) + + def test_sync_by_explicit_inn_saves_reports_through_fns_service(self): + report = self._report() + with ( + patch.object(parser_tasks, "FNSApiClient") as client_class, + patch.object( + parser_tasks.FNSReportService, + "save_report", + return_value=SimpleNamespace(uid=fake.uuid4()), + ) as save_report, + patch.object(parser_tasks, "_active_fns_registry_inns") as registry_inns, + ): + client = client_class.return_value.__enter__.return_value + client.fetch_reports_by_inn.return_value = [report] + + result = sync_fns_financial_reports( + inn="7707083893", + max_periods=3, + proxies=[], + ) + + client_class.assert_called_once_with(max_periods=3, proxies=[]) + client.fetch_reports_by_inn.assert_called_once_with("7707083893") + registry_inns.assert_not_called() + save_kwargs = save_report.call_args.kwargs + self.assertEqual(save_kwargs["external_id"], report.external_id) + self.assertEqual(save_kwargs["batch_id"], result["batch_id"]) + self.assertEqual(result["saved_reports"], 1) + self.assertEqual(result["saved_lines"], 2) + self.assertEqual(result["sync_mode"], "inn") + + def test_sync_by_explicit_fns_organization_id_bypasses_search(self): + report = self._report(lines_count=1) + with ( + patch.object(parser_tasks, "FNSApiClient") as client_class, + patch.object(parser_tasks.FNSReportService, "save_report"), + patch.object(parser_tasks, "_active_fns_registry_inns") as registry_inns, + ): + client = client_class.return_value.__enter__.return_value + client.fetch_reports_by_organization_id.return_value = [report] + + result = sync_fns_financial_reports( + organization_id=12345, + proxies=[], + ) + + client.fetch_reports_by_organization_id.assert_called_once_with("12345") + client.fetch_reports_by_inn.assert_not_called() + registry_inns.assert_not_called() + self.assertEqual(result["sync_mode"], "organization_id") + self.assertEqual(result["targets"], 1) + + def test_registry_sync_clamps_limit_and_continues_after_target_error(self): + report = self._report(lines_count=1) + with ( + patch.object( + parser_tasks, + "_active_fns_registry_inns", + return_value=["7707083893", "7736050003"], + ) as registry_inns, + patch.object(parser_tasks, "FNSApiClient") as client_class, + patch.object(parser_tasks.FNSReportService, "save_report"), + ): + client = client_class.return_value.__enter__.return_value + client.fetch_reports_by_inn.side_effect = [ + HTTPError("organization unavailable"), + [report], + ] + + result = sync_fns_financial_reports( + registry_organization_limit=2_600_000, + proxies=[], + ) + + registry_inns.assert_called_once_with( + limit=FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT + ) + self.assertEqual(result["targets"], 2) + self.assertEqual(result["successful_targets"], 1) + self.assertEqual(result["failed_targets"], 1) + self.assertEqual(result["saved_reports"], 1) + + def test_registry_sync_uses_bounded_default(self): + with ( + patch.object( + parser_tasks, + "_active_fns_registry_inns", + return_value=[], + ) as registry_inns, + patch.object(parser_tasks, "FNSApiClient") as client_class, + ): + result = sync_fns_financial_reports(proxies=[]) + + registry_inns.assert_called_once_with( + limit=FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT + ) + client_class.assert_not_called() + self.assertEqual(result["targets"], 0) + self.assertEqual(result["saved_reports"], 0) + + def test_rejects_combined_explicit_selectors_before_creating_load(self): + with self.assertRaisesRegex(ValueError, "either inn or organization_id"): + sync_fns_financial_reports( + inn="7707083893", + organization_id=12345, + proxies=[], + ) + + self.assertFalse( + ParserLoadLog.objects.filter( + source=ParserLoadLog.Source.FNS_REPORTS + ).exists() + ) + + def test_zero_organization_id_is_rejected_instead_of_running_registry_sync(self): + with ( + patch.object( + parser_tasks, + "_active_fns_registry_inns", + ) as registry_inns, + self.assertRaisesRegex(ValueError, "positive integer"), + ): + sync_fns_financial_reports(organization_id=0, proxies=[]) + + registry_inns.assert_not_called() + self.assertTrue( + ParserLoadLog.objects.filter( + source=ParserLoadLog.Source.FNS_REPORTS, + status=ParserLoadLog.Status.FAILED, + ).exists() + ) + + def test_active_registry_targets_include_all_registry_signals_and_respect_limit( + self, + ): + OrganizationFactory( + inn="1000000001", + opk_registry_membership=True, + ) + OrganizationFactory( + inn="1000000002", + goz_participation=True, + ) + OrganizationFactory( + inn="1000000003", + ropk_num="РОПК-3", + ) + OrganizationFactory( + inn="1000000000", + opk_registry_membership=False, + goz_participation=False, + ropk_num="", + ) + + targets = parser_tasks._active_fns_registry_inns(limit=2) + + self.assertEqual(targets, ["1000000001", "1000000002"]) + + @override_settings( CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, @@ -1378,7 +1678,7 @@ class MinpromtorgTasksTestCase(TestCase): def _add_minpromtorg_routes(self, server: TestHTTPServer): certificates_bytes, cert_rows = build_minpromtorg_certificates_excel(count=2) manufacturers_bytes, manuf_rows = build_minpromtorg_manufacturers_excel(count=2) - products_bytes, product_rows = build_minpromtorg_products_excel(count=2) + _products_bytes, product_rows = build_minpromtorg_products_excel(count=2) _ensure_directory_organizations_for_rows(cert_rows) _ensure_directory_organizations_for_rows(manuf_rows) _ensure_directory_organizations_for_rows(product_rows) @@ -1388,7 +1688,6 @@ class MinpromtorgTasksTestCase(TestCase): ) cert_file = f"data_resolutions_{date_str}.xlsx" manuf_file = f"data_orgs_{date_str}.xlsx" - products_file = f"industrial_products_{date_str}.xlsx" server.add_json( "/api/kss-document-preview", @@ -1402,15 +1701,6 @@ class MinpromtorgTasksTestCase(TestCase): "name": ManufacturesClient().query, "files": [{"name": manuf_file, "url": f"/files/{manuf_file}"}], }, - { - "name": IndustrialProductsClient().query, - "files": [ - { - "name": products_file, - "url": f"/files/{products_file}", - } - ], - }, ] }, ) @@ -1428,13 +1718,37 @@ class MinpromtorgTasksTestCase(TestCase): "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ), ) - server.add_bytes( - f"/files/{products_file}", - products_bytes, - content_type=( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ), - ) + gisp_items = [ + { + "_product_reg_number_2023": row.registry_number, + "_org_inn": row.inn, + "_org_ogrn": row.ogrn, + "_org_name": row.full_organisation_name, + "_product_name": row.product_name, + "_product_spec": row.product_model, + "_product_okpd2": row.okpd2_code, + "_product_tnved": row.tnved_code, + "_basedondoc_name": row.regulatory_document, + } + for row in product_rows + ] + + def handle_gisp_products(_request, body: bytes) -> Response: + options = json.loads(body)["opt"] + skip = options["skip"] + take = options["take"] + return Response( + body=json.dumps( + { + "ok": True, + "total_count": len(gisp_items), + "items": gisp_items[skip : skip + take], + } + ).encode(), + headers={"Content-Type": "application/json"}, + ) + + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_gisp_products) return cert_rows, product_rows, manuf_rows def test_parse_all_minpromtorg_success(self): @@ -1470,11 +1784,24 @@ class MinpromtorgTasksTestCase(TestCase): def test_parse_all_sources_success(self): with TestHTTPServer() as server: cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) - result = parse_all_sources( - proxies=[], - client_adapter=server.adapter, - inspections_use_playwright=False, - ) + with patch.object( + parser_tasks.sync_inspections, + "apply", + return_value=SimpleNamespace(id="inspections-test-task"), + ) as inspections_apply: + result = parse_all_sources( + proxies=[], + client_adapter=server.adapter, + inspections_use_playwright=False, + ) + + inspections_apply.assert_called_once_with( + kwargs={ + "proxies": [], + "client_adapter": server.adapter, + "use_playwright": False, + } + ) self.assertIn("industrial", result) self.assertIn("industrial_products", result) @@ -1505,6 +1832,44 @@ class MinpromtorgTasksTestCase(TestCase): 0, ) + @override_settings(CELERY_TASK_ALWAYS_EAGER=False) + def test_parse_all_sources_queues_fns_api_sync(self): + task_names = ( + "parse_industrial_production", + "parse_industrial_products", + "parse_manufactures", + "sync_inspections", + "parse_procurements_44fz", + "parse_procurements_223fz", + "parse_contracts", + "parse_unfair_suppliers", + "parse_fas_goz_evasion", + "sync_fns_financial_reports", + "parse_arbitration_cases", + "parse_fedresurs_bankruptcy", + "parse_fstec_registers", + "parse_trudvsem_vacancies", + ) + with ExitStack() as stack: + delays = { + task_name: stack.enter_context( + patch.object( + getattr(parser_tasks, task_name), + "delay", + return_value=SimpleNamespace(id=f"{task_name}-id"), + ) + ) + for task_name in task_names + } + + result = parse_all_sources(proxies=[]) + + self.assertEqual( + result["fns_financial"], + "sync_fns_financial_reports-id", + ) + delays["sync_fns_financial_reports"].assert_called_once_with(proxies=[]) + def test_parse_all_minpromtorg_without_adapter(self): with TestHTTPServer() as server: cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) @@ -1514,7 +1879,7 @@ class MinpromtorgTasksTestCase(TestCase): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) - class _LocalIndustrialProductsClient(IndustrialProductsClient): + class _LocalIndustrialProductsClient(GispProductsClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) @@ -1525,7 +1890,7 @@ class MinpromtorgTasksTestCase(TestCase): super().__init__(*args, **kwargs) original_industrial = parser_tasks.IndustrialProductionClient - original_industrial_products = parser_tasks.IndustrialProductsClient + original_industrial_products = parser_tasks.GispProductsClient original_manufactures = parser_tasks.ManufacturesClient original_industrial_delay = parser_tasks.parse_industrial_production.delay original_industrial_products_delay = ( @@ -1533,7 +1898,7 @@ class MinpromtorgTasksTestCase(TestCase): ) original_manufactures_delay = parser_tasks.parse_manufactures.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient - parser_tasks.IndustrialProductsClient = _LocalIndustrialProductsClient + parser_tasks.GispProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): @@ -1563,7 +1928,7 @@ class MinpromtorgTasksTestCase(TestCase): result = parse_all_minpromtorg(proxies=[]) finally: parser_tasks.IndustrialProductionClient = original_industrial - parser_tasks.IndustrialProductsClient = original_industrial_products + parser_tasks.GispProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures parser_tasks.parse_industrial_production.delay = ( original_industrial_delay @@ -1604,7 +1969,7 @@ class MinpromtorgTasksTestCase(TestCase): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) - class _LocalIndustrialProductsClient(IndustrialProductsClient): + class _LocalIndustrialProductsClient(GispProductsClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) @@ -1615,16 +1980,16 @@ class MinpromtorgTasksTestCase(TestCase): super().__init__(*args, **kwargs) original_industrial = parser_tasks.IndustrialProductionClient - original_industrial_products = parser_tasks.IndustrialProductsClient + original_industrial_products = parser_tasks.GispProductsClient original_manufactures = parser_tasks.ManufacturesClient original_industrial_delay = parser_tasks.parse_industrial_production.delay original_industrial_products_delay = ( parser_tasks.parse_industrial_products.delay ) original_manufactures_delay = parser_tasks.parse_manufactures.delay - original_inspections_delay = parser_tasks.parse_inspections.delay + original_sync_inspections_delay = parser_tasks.sync_inspections.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient - parser_tasks.IndustrialProductsClient = _LocalIndustrialProductsClient + parser_tasks.GispProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): @@ -1653,12 +2018,12 @@ class MinpromtorgTasksTestCase(TestCase): _industrial_products_eager_delay ) parser_tasks.parse_manufactures.delay = _manufactures_eager_delay - parser_tasks.parse_inspections.delay = _inspections_stub_delay + parser_tasks.sync_inspections.delay = _inspections_stub_delay try: result = parse_all_sources(proxies=[], inspections_use_playwright=None) finally: parser_tasks.IndustrialProductionClient = original_industrial - parser_tasks.IndustrialProductsClient = original_industrial_products + parser_tasks.GispProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures parser_tasks.parse_industrial_production.delay = ( original_industrial_delay @@ -1667,7 +2032,7 @@ class MinpromtorgTasksTestCase(TestCase): original_industrial_products_delay ) parser_tasks.parse_manufactures.delay = original_manufactures_delay - parser_tasks.parse_inspections.delay = original_inspections_delay + parser_tasks.sync_inspections.delay = original_sync_inspections_delay self.assertIn("industrial", result) self.assertIn("industrial_products", result) @@ -1733,30 +2098,16 @@ class MinpromtorgTasksTestCase(TestCase): ) def test_parse_industrial_products_failure(self): - date_str = fake.date_between(start_date="-30d", end_date="today").strftime( - "%Y%m%d" - ) - products_file = f"industrial_products_{date_str}.xlsx" - with TestHTTPServer() as server: - server.add_json( - "/api/kss-document-preview", - { - "data": [ - { - "name": IndustrialProductsClient().query, - "files": [ - { - "name": products_file, - "url": f"/files/{products_file}", - } - ], - } - ] - }, + server.add_route( + "POST", + "/pp719v2/pub/prod/b/", + lambda _request, _body: Response( + body=b'{"ok":true,"total_count":1}', + headers={"Content-Type": "application/json"}, + ), ) - server.add_bytes("/files/" + products_file, b"not-an-excel") - with self.assertRaises(IndustrialProductsClientError): + with self.assertRaises(GispProductsClientError): parse_industrial_products(client_adapter=server.adapter) def test_parse_industrial_products_with_default_proxies(self): @@ -1765,6 +2116,9 @@ class MinpromtorgTasksTestCase(TestCase): result = parse_industrial_products(client_adapter=server.adapter) self.assertEqual(result["status"], "success") + self.assertEqual(result["fetched"], len(product_rows)) + self.assertEqual(result["total"], len(product_rows)) + self.assertFalse(result["truncated"]) self.assertEqual( OrganizationSourceRecord.objects.filter( source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS @@ -1772,6 +2126,35 @@ class MinpromtorgTasksTestCase(TestCase): len(product_rows), ) + def test_parse_industrial_products_applies_bounded_pagination(self): + with TestHTTPServer() as server: + _cert_rows, _product_rows, _manuf_rows = self._add_minpromtorg_routes( + server + ) + result = parse_industrial_products( + client_adapter=server.adapter, + page_size=1, + max_pages=1, + max_records=1, + ) + + self.assertEqual(result["fetched"], 1) + self.assertEqual(result["total"], 2) + self.assertTrue(result["truncated"]) + self.assertEqual(result["saved"], 1) + self.assertEqual( + OrganizationSourceRecord.objects.filter( + source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS + ).count(), + 1, + ) + + def test_parse_industrial_products_keeps_public_task_name(self): + self.assertEqual( + parse_industrial_products.name, + "apps.parsers.tasks.parse_industrial_products", + ) + def test_parse_manufactures_failure(self): date_str = fake.date_between(start_date="-30d", end_date="today").strftime( "%Y%m%d" @@ -2782,7 +3165,10 @@ class ParseVacanciesTaskTestCase(TestCase): self.assertEqual(result["saved"], 1) self.assertEqual(captured_offsets, [0]) - @override_settings(SUPERJOB_APP_ID="test-superjob-app-id") + @override_settings( + SUPERJOB_APP_ID="test-superjob-app-id", + HH_USER_AGENT="Mostovik/1.0 (ops@example.test)", + ) def test_parse_trudvsem_vacancies_uses_combined_vacancies_client(self): for inn in ("7701000401", "7701000402", "7701000403"): _ensure_directory_organization(inn=inn) @@ -2842,6 +3228,10 @@ class ParseVacanciesTaskTestCase(TestCase): self.assertEqual(result["status"], "success") self.assertEqual(result["saved"], 3) self.assertEqual(captured_kwargs["superjob_app_id"], "test-superjob-app-id") + self.assertEqual( + captured_kwargs["hh_user_agent"], + "Mostovik/1.0 (ops@example.test)", + ) self.assertEqual(captured_kwargs["sources"], ["trudvsem", "hh", "superjob"]) self.assertEqual(captured_fetch_kwargs["limit"], 25) self.assertEqual(captured_fetch_kwargs["offset"], 5) diff --git a/tests/apps/parsers/test_vacancy_clients.py b/tests/apps/parsers/test_vacancy_clients.py index 1124466..2b4672f 100644 --- a/tests/apps/parsers/test_vacancy_clients.py +++ b/tests/apps/parsers/test_vacancy_clients.py @@ -17,6 +17,15 @@ from tests.utils import TestHTTPServer class TrudvsemClientTest(SimpleTestCase): + def test_default_base_url_uses_https_transport(self): + with TestHTTPServer() as server: + server.add_json("/api/v1/vacancies", {"results": {"vacancies": []}}) + with TrudvsemClient(http_adapter=server.adapter) as client: + records = client.fetch_vacancies(limit=1) + + self.assertEqual(records, []) + self.assertEqual(client.base_url, "https://opendata.trudvsem.ru/api/v1") + def test_fetch_vacancies_by_company_inn_uses_company_endpoint(self): with TestHTTPServer() as server: server.add_json( @@ -78,6 +87,7 @@ class HHVacanciesClientTest(SimpleTestCase): }, ) client = HHVacanciesClient( + user_agent="Mostovik/1.0 (ops@example.test)", base_url=server.base_url, http_adapter=server.adapter, ) @@ -93,6 +103,10 @@ class HHVacanciesClientTest(SimpleTestCase): self.assertEqual(record.amount, Decimal("120000")) self.assertEqual(record.url, "https://hh.ru/vacancy/123") self.assertEqual(record.payload["vacancy_source"], "hh") + self.assertEqual( + client.http_client.session.headers["HH-User-Agent"], + "Mostovik/1.0 (ops@example.test)", + ) def test_fetch_vacancies_tolerates_unexpected_nested_shapes(self): with TestHTTPServer() as server: @@ -111,6 +125,7 @@ class HHVacanciesClientTest(SimpleTestCase): }, ) client = HHVacanciesClient( + user_agent="Mostovik/1.0 (ops@example.test)", base_url=server.base_url, http_adapter=server.adapter, ) @@ -122,6 +137,21 @@ class HHVacanciesClientTest(SimpleTestCase): self.assertEqual(records[0].organisation_name, "") self.assertIsNone(records[0].amount) + def test_fetch_vacancies_requires_hh_user_agent_before_request(self): + with TestHTTPServer() as server: + server.add_json("/vacancies", {"items": []}) + client = HHVacanciesClient( + user_agent=" ", + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage( + VacanciesClientError, + "HH_USER_AGENT is required", + ): + client.fetch_vacancies(limit=1) + class SuperJobVacanciesClientTest(SimpleTestCase): def test_fetch_vacancies_maps_superjob_objects_to_generic_records(self): -- 2.39.5 From 63f59b3799f3c6d43332209841a13f30baeb0b62 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Thu, 16 Jul 2026 15:42:35 +0200 Subject: [PATCH 2/6] fix: serve collected static files with whitenoise --- docker/scripts/start-web.sh | 1 + src/settings/base.py | 1 + tests/test_runtime_static.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 tests/test_runtime_static.py diff --git a/docker/scripts/start-web.sh b/docker/scripts/start-web.sh index 7aeeb55..ac501b7 100755 --- a/docker/scripts/start-web.sh +++ b/docker/scripts/start-web.sh @@ -18,6 +18,7 @@ esac /app/docker/scripts/check-deps.sh python src/manage.py migrate --noinput +python src/manage.py collectstatic --noinput exec gunicorn core.wsgi:application \ --bind "0.0.0.0:${PORT:-8000}" \ diff --git a/src/settings/base.py b/src/settings/base.py index 2535626..0d1e19c 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -284,6 +284,7 @@ USE_TZ = True STATIC_URL = "/static/" STATIC_ROOT = PROJECT_ROOT / "staticfiles" STATICFILES_DIRS = [BASE_DIR / "static"] +STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" # Media files MEDIA_URL = "/media/" diff --git a/tests/test_runtime_static.py b/tests/test_runtime_static.py new file mode 100644 index 0000000..eccf7aa --- /dev/null +++ b/tests/test_runtime_static.py @@ -0,0 +1,31 @@ +"""Regression checks for production static-file delivery.""" + +from pathlib import Path + +from django.core.management import call_command +from django.test import Client, override_settings +from settings import base + + +def test_whitenoise_uses_compressed_manifest_storage(): + assert ( + base.STATICFILES_STORAGE + == "whitenoise.storage.CompressedManifestStaticFilesStorage" + ) + + +def test_web_startup_collects_static_files_before_gunicorn(): + script_path = Path(__file__).parents[1] / "docker" / "scripts" / "start-web.sh" + script = script_path.read_text(encoding="utf-8") + + assert "python src/manage.py collectstatic --noinput" in script + assert script.index("collectstatic --noinput") < script.index("exec gunicorn") + + +def test_whitenoise_serves_collected_swagger_asset(tmp_path): + with override_settings(STATIC_ROOT=tmp_path): + call_command("collectstatic", interactive=False, verbosity=0) + response = Client().get("/static/drf-yasg/style.css") + + assert response.status_code == 200 + assert response["Content-Type"].startswith("text/css") -- 2.39.5 From 76aed1dca3b597f53275d9d889752706524a06cd Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Sun, 19 Jul 2026 13:32:46 +0200 Subject: [PATCH 3/6] feat: address QA feedback and limit Checko collection --- src/apps/exchange/state_corp_services.py | 3 - src/apps/parsers/checko_collection.py | 56 +++++ .../0027_checkocollectionattempt.py | 116 +++++++++++ src/apps/parsers/models.py | 57 ++++++ src/apps/parsers/tasks.py | 135 +++++++++--- src/organizations/directory_import.py | 5 +- src/organizations/test_companies.py | 193 +++++++++++++++++- src/organizations/views.py | 2 - src/settings/base.py | 6 +- src/settings/test.py | 3 +- src/user/views.py | 11 + .../apps/exchange/test_state_corp_services.py | 16 ++ .../organizations/test_directory_import.py | 18 +- .../test_test_companies_commands.py | 27 ++- tests/apps/parsers/test_checko_collection.py | 92 +++++++++ tests/apps/parsers/test_tasks.py | 20 ++ tests/apps/user/test_views.py | 11 + 17 files changed, 730 insertions(+), 41 deletions(-) create mode 100644 src/apps/parsers/checko_collection.py create mode 100644 src/apps/parsers/migrations/0027_checkocollectionattempt.py create mode 100644 tests/apps/parsers/test_checko_collection.py diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index b02cdd4..a6ecf75 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -614,7 +614,6 @@ class StateCorpExchangeService: ParserLoadLog.Source.PROCUREMENTS, ParserLoadLog.Source.PROCUREMENTS_44FZ, ParserLoadLog.Source.PROCUREMENTS_223FZ, - ParserLoadLog.Source.CONTRACTS, ], ): item = cls._serialize_generic_public_procurement(record) @@ -676,8 +675,6 @@ class StateCorpExchangeService: return "44-ФЗ" if source == ParserLoadLog.Source.PROCUREMENTS_223FZ: return "223-ФЗ" - if source == ParserLoadLog.Source.CONTRACTS: - return "contract" return "" @classmethod diff --git a/src/apps/parsers/checko_collection.py b/src/apps/parsers/checko_collection.py new file mode 100644 index 0000000..ab04a0b --- /dev/null +++ b/src/apps/parsers/checko_collection.py @@ -0,0 +1,56 @@ +"""Quota-safe monthly claims for organization lookups in Checko.""" + +from datetime import date, datetime + +from apps.parsers.models import CheckoCollectionAttempt +from django.utils import timezone + + +def collection_period_month(at: date | datetime | None = None) -> date: + if at is None: + local_date = timezone.localdate() + elif isinstance(at, datetime): + local_date = timezone.localtime(at).date() if timezone.is_aware(at) else at.date() + else: + local_date = at + return local_date.replace(day=1) + + +def claim_monthly_collection( + *, + organization_id: str, + source: str, + at: date | datetime | None = None, +) -> CheckoCollectionAttempt | None: + """Atomically claim this month's only allowed Checko collection attempt.""" + attempt, created = CheckoCollectionAttempt.objects.get_or_create( + organization_id=organization_id, + source=source, + period_month=collection_period_month(at), + defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS}, + ) + return attempt if created else None + + +def finish_collection( + attempt: CheckoCollectionAttempt, + *, + records_count: int, + error: Exception | str | None = None, +) -> None: + """Persist the outcome without releasing the monthly claim.""" + attempt.records_count = max(int(records_count), 0) + attempt.error_message = str(error or "") + attempt.status = ( + CheckoCollectionAttempt.Status.FAILED + if error is not None + else CheckoCollectionAttempt.Status.SUCCESS + ) + attempt.save( + update_fields=[ + "records_count", + "error_message", + "status", + "updated_at", + ] + ) diff --git a/src/apps/parsers/migrations/0027_checkocollectionattempt.py b/src/apps/parsers/migrations/0027_checkocollectionattempt.py new file mode 100644 index 0000000..2f8ab58 --- /dev/null +++ b/src/apps/parsers/migrations/0027_checkocollectionattempt.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.9 on 2026-07-19 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("organizations", "0007_auto_20260607_1017"), + ("parsers", "0026_update_fns_financial_schedule"), + ] + + operations = [ + migrations.CreateModel( + name="CheckoCollectionAttempt", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Дата и время создания записи", + verbose_name="создано", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="Дата и время последнего обновления", + verbose_name="обновлено", + ), + ), + ( + "source", + models.CharField( + choices=[ + ("arbitration", "Арбитражные дела"), + ("bankruptcy", "Банкротства"), + ("contracts", "Контракты"), + ("inspections", "Проверки"), + ], + max_length=32, + verbose_name="источник", + ), + ), + ( + "period_month", + models.DateField( + help_text="Первый день календарного месяца, за который занята попытка", + verbose_name="месяц сбора", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("in_progress", "В процессе"), + ("success", "Успешно"), + ("failed", "Ошибка"), + ], + default="in_progress", + max_length=20, + verbose_name="статус", + ), + ), + ( + "records_count", + models.PositiveIntegerField( + default=0, + verbose_name="количество записей", + ), + ), + ( + "error_message", + models.TextField(blank=True, verbose_name="сообщение об ошибке"), + ), + ( + "organization", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="checko_collection_attempts", + to="organizations.organization", + verbose_name="организация", + ), + ), + ], + options={ + "verbose_name": "попытка сбора Checko", + "verbose_name_plural": "попытки сбора Checko", + "db_table": "parsers_checko_collection_attempt", + "ordering": ["-period_month", "source", "organization_id"], + "indexes": [ + models.Index( + fields=["source", "period_month"], + name="parsers_che_source_0ca426_idx", + ) + ], + "constraints": [ + models.UniqueConstraint( + fields=("organization", "source", "period_month"), + name="unique_checko_collection_per_org_source_month", + ) + ], + }, + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 70433ca..771f669 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -117,6 +117,63 @@ class ParserBatchSequence(TimestampMixin, models.Model): return f"{self.source}: next batch {self.next_batch_id}" +class CheckoCollectionAttempt(TimestampMixin, models.Model): + """Monthly Checko collection claim for one organization and source.""" + + class Source(models.TextChoices): + ARBITRATION = "arbitration", _("Арбитражные дела") + BANKRUPTCY = "bankruptcy", _("Банкротства") + CONTRACTS = "contracts", _("Контракты") + INSPECTIONS = "inspections", _("Проверки") + + class Status(models.TextChoices): + IN_PROGRESS = "in_progress", _("В процессе") + SUCCESS = "success", _("Успешно") + FAILED = "failed", _("Ошибка") + + organization = models.ForeignKey( + "organizations.Organization", + on_delete=models.CASCADE, + related_name="checko_collection_attempts", + verbose_name=_("организация"), + ) + source = models.CharField( + _("источник"), + max_length=32, + choices=Source.choices, + ) + period_month = models.DateField( + _("месяц сбора"), + help_text=_("Первый день календарного месяца, за который занята попытка"), + ) + status = models.CharField( + _("статус"), + max_length=20, + choices=Status.choices, + default=Status.IN_PROGRESS, + ) + records_count = models.PositiveIntegerField(_("количество записей"), default=0) + error_message = models.TextField(_("сообщение об ошибке"), blank=True) + + class Meta: + db_table = "parsers_checko_collection_attempt" + verbose_name = _("попытка сбора Checko") + verbose_name_plural = _("попытки сбора Checko") + ordering = ["-period_month", "source", "organization_id"] + constraints = [ + models.UniqueConstraint( + fields=["organization", "source", "period_month"], + name="unique_checko_collection_per_org_source_month", + ), + ] + indexes = [ + models.Index(fields=["source", "period_month"]), + ] + + def __str__(self) -> str: + return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})" + + GENERIC_RECORD_SOURCE_CHOICES = [ *ParserLoadLog.Source.choices, ("hh", _("Вакансии HeadHunter")), diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index ff33f75..4c1dda9 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -18,6 +18,11 @@ from pathlib import Path from apps.core.services import BackgroundJobService from apps.core.tasks import PeriodicTask as CorePeriodicTask +from apps.parsers.checko_collection import ( + claim_monthly_collection, + collection_period_month, + finish_collection, +) from apps.parsers.clients.base import HTTPClientError from apps.parsers.clients.checko import ( CheckoClient, @@ -43,7 +48,7 @@ from apps.parsers.clients.proverki import ProverkiClient from apps.parsers.clients.proverki.schemas import Inspection as ProverkiInspection from apps.parsers.clients.vacancies import VacanciesClient from apps.parsers.clients.zakupki import ZakupkiClient -from apps.parsers.models import ParserLoadLog +from apps.parsers.models import CheckoCollectionAttempt, ParserLoadLog from apps.parsers.services import ( FNSReportOrganizationResolutionSkipped, FNSReportService, @@ -171,11 +176,17 @@ def _resolve_lookup_limit( def _active_registry_lookup_targets( *, limit: int | None = None, + checko_source: str | None = None, ) -> list[RegistryLookupTarget]: """Вернуть организации, которые сейчас состоят хотя бы в одном реестре.""" + queryset = SourceOrganization.objects.filter(opk_registry_membership=True) + if checko_source is not None: + queryset = queryset.exclude( + checko_collection_attempts__source=checko_source, + checko_collection_attempts__period_month=collection_period_month(), + ) queryset = ( - SourceOrganization.objects.filter(opk_registry_membership=True) - .order_by("inn", "ogrn", "uid") + queryset.order_by("inn", "ogrn", "uid") .values( "uid", "inn", @@ -817,9 +828,14 @@ def _fetch_checko_bankruptcy_records( if limit <= 0: logger.info("Fedresurs Checko fallback is disabled by limit=%s", limit) return [] - targets = _active_registry_lookup_targets(limit=limit) + targets = _active_registry_lookup_targets( + limit=limit, + checko_source=CheckoCollectionAttempt.Source.BANKRUPTCY, + ) if not targets: - logger.info("No active registry organizations found for Fedresurs fallback") + logger.info( + "No active registry organizations are due for monthly Fedresurs fallback" + ) return [] checko_proxies = ( @@ -828,11 +844,19 @@ def _fetch_checko_bankruptcy_records( client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30) records: list[GenericParserItem] = [] for target in targets: + attempt = claim_monthly_collection( + organization_id=target.organization_id, + source=CheckoCollectionAttempt.Source.BANKRUPTCY, + ) + if attempt is None: + continue + records_before = len(records) try: response = client.get_company( CompanyRequest(inn=target.inn or None, ogrn=target.ogrn or None) ) except CheckoRateLimitError as exc: + finish_collection(attempt, records_count=0, error=exc) logger.warning( "Checko bankruptcy fallback stopped: quota/rate limit reached " "(status_code=%s)", @@ -840,6 +864,7 @@ def _fetch_checko_bankruptcy_records( ) break except CheckoError as exc: + finish_collection(attempt, records_count=0, error=exc) logger.info( "Checko bankruptcy lookup skipped for target=%s: %s", target.inn or target.ogrn, @@ -847,15 +872,18 @@ def _fetch_checko_bankruptcy_records( ) continue company = response.data - if company is None: - continue - records.extend( - _checko_bankruptcy_items( - company=company, - fallback_inn=target.inn, - fallback_ogrn=target.ogrn, - fallback_name=target.name, + if company is not None: + records.extend( + _checko_bankruptcy_items( + company=company, + fallback_inn=target.inn, + fallback_ogrn=target.ogrn, + fallback_name=target.name, + ) ) + finish_collection( + attempt, + records_count=len(records) - records_before, ) logger.info("Fetched %d bankruptcy records through Checko fallback", len(records)) return records @@ -990,7 +1018,10 @@ def _add_arbitration_subject( def _arbitration_subjects(limit: int) -> list[ArbitrationSubject]: """Собрать активные организации из реестров для арбитражного lookup.""" subjects: dict[str, ArbitrationSubject] = {} - for target in _active_registry_lookup_targets(limit=limit): + for target in _active_registry_lookup_targets( + limit=limit, + checko_source=CheckoCollectionAttempt.Source.ARBITRATION, + ): _add_arbitration_subject( subjects, inn=target.inn, @@ -1031,7 +1062,7 @@ def _fetch_checko_arbitration_records( subjects = _arbitration_subjects(resolved_limit) if not subjects: raise ParserSourceSkipped( - "no active registry organizations found for arbitration" + "no active registry organizations are due for monthly arbitration lookup" ) checko_proxies = ( @@ -1040,7 +1071,16 @@ def _fetch_checko_arbitration_records( client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30) records: list[GenericParserItem] = [] failed_lookups = 0 + attempted_lookups = 0 for subject in subjects: + attempt = claim_monthly_collection( + organization_id=subject.registry_organization_id, + source=CheckoCollectionAttempt.Source.ARBITRATION, + ) + if attempt is None: + continue + attempted_lookups += 1 + records_before = len(records) try: request = LegalCasesRequest( inn=subject.inn or None, @@ -1051,13 +1091,19 @@ def _fetch_checko_arbitration_records( records.append(_checko_arbitration_item(legal_case, subject=subject)) except CheckoError as exc: failed_lookups += 1 + finish_collection(attempt, records_count=0, error=exc) logger.info( "Checko arbitration lookup skipped for subject=%s: %s", _arbitration_subject_key(subject), exc, ) + else: + finish_collection( + attempt, + records_count=len(records) - records_before, + ) - if failed_lookups == len(subjects) and not records: + if attempted_lookups and failed_lookups == attempted_lookups and not records: raise ParserSourceSkipped("Checko arbitration lookups failed for all subjects") logger.info( @@ -1273,10 +1319,13 @@ def _fetch_checko_registry_inspections( logger.info("Registry inspections Checko parser is disabled by limit=%s", limit) return [] - targets = _active_registry_lookup_targets(limit=resolved_limit) + targets = _active_registry_lookup_targets( + limit=resolved_limit, + checko_source=CheckoCollectionAttempt.Source.INSPECTIONS, + ) if not targets: raise ParserSourceSkipped( - "no active registry organizations found for registry inspections" + "no active registry organizations are due for monthly inspections lookup" ) checko_proxies = ( @@ -1285,7 +1334,16 @@ def _fetch_checko_registry_inspections( client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30) records: list[ProverkiInspection] = [] failed_lookups = 0 + attempted_lookups = 0 for target in targets: + attempt = claim_monthly_collection( + organization_id=target.organization_id, + source=CheckoCollectionAttempt.Source.INSPECTIONS, + ) + if attempt is None: + continue + attempted_lookups += 1 + records_before = len(records) try: request = InspectionsRequest( inn=target.inn or None, @@ -1296,13 +1354,19 @@ def _fetch_checko_registry_inspections( records.append(_checko_inspection_item(inspection, target=target)) except CheckoError as exc: failed_lookups += 1 + finish_collection(attempt, records_count=0, error=exc) logger.info( "Checko inspections lookup skipped for target=%s: %s", target.inn or target.ogrn, exc, ) + else: + finish_collection( + attempt, + records_count=len(records) - records_before, + ) - if failed_lookups == len(targets) and not records: + if attempted_lookups and failed_lookups == attempted_lookups and not records: raise ParserSourceSkipped("Checko inspections lookups failed for all targets") logger.info( @@ -1429,10 +1493,13 @@ def _fetch_checko_registry_contract_records( logger.info("Registry contracts Checko parser is disabled by limit=%s", limit) return [] - targets = _active_registry_lookup_targets(limit=resolved_limit) + targets = _active_registry_lookup_targets( + limit=resolved_limit, + checko_source=CheckoCollectionAttempt.Source.CONTRACTS, + ) if not targets: raise ParserSourceSkipped( - "no active registry organizations found for registry contracts" + "no active registry organizations are due for monthly contracts lookup" ) checko_proxies = ( @@ -1441,7 +1508,17 @@ def _fetch_checko_registry_contract_records( client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30) records: list[GenericParserItem] = [] failed_lookups = 0 + attempted_lookups = 0 for target in targets: + attempt = claim_monthly_collection( + organization_id=target.organization_id, + source=CheckoCollectionAttempt.Source.CONTRACTS, + ) + if attempt is None: + continue + attempted_lookups += 1 + records_before = len(records) + target_failures: list[CheckoError] = [] for law in (ContractLaw.FZ44, ContractLaw.FZ223): try: request = ContractsRequest( @@ -1456,15 +1533,21 @@ def _fetch_checko_registry_contract_records( ) except CheckoError as exc: failed_lookups += 1 + target_failures.append(exc) logger.info( "Checko contracts lookup skipped for target=%s law=%s: %s", target.inn or target.ogrn, law.value, exc, ) + finish_collection( + attempt, + records_count=len(records) - records_before, + error=target_failures[0] if len(target_failures) == 2 else None, + ) - expected_lookups = len(targets) * 2 - if failed_lookups == expected_lookups and not records: + expected_lookups = attempted_lookups * 2 + if expected_lookups and failed_lookups == expected_lookups and not records: raise ParserSourceSkipped("Checko contracts lookups failed for all targets") logger.info( @@ -2242,6 +2325,9 @@ def parse_all_sources( proxies=proxies ).id, "contracts": parse_contracts.delay(proxies=proxies).id, + "registry_contracts": parse_registry_contracts.delay( + proxies=proxies + ).id, "unfair_suppliers": parse_unfair_suppliers.delay(proxies=proxies).id, "fas_goz": parse_fas_goz_evasion.delay(proxies=proxies).id, "fns_financial": sync_fns_financial_reports.delay(proxies=proxies).id, @@ -2249,6 +2335,9 @@ def parse_all_sources( "fedresurs_bankruptcy": parse_fedresurs_bankruptcy.delay( proxies=proxies ).id, + "registry_inspections": parse_registry_inspections.delay( + proxies=proxies + ).id, "fstec": parse_fstec_registers.delay(proxies=proxies).id, "trudvsem": parse_trudvsem_vacancies.delay(proxies=proxies).id, } diff --git a/src/organizations/directory_import.py b/src/organizations/directory_import.py index e213219..dcab28c 100644 --- a/src/organizations/directory_import.py +++ b/src/organizations/directory_import.py @@ -14,6 +14,7 @@ from django.db import transaction from django.utils import timezone from openpyxl import load_workbook +from organizations.cache import invalidate_organization_api_cache from organizations.models import Organization DIRECTORY_SHEET = "Лист1" @@ -142,12 +143,14 @@ class OrganizationDirectoryImportService: else: updated += 1 - return OrganizationDirectoryImportResult( + result = OrganizationDirectoryImportResult( scanned=scanned, created=created, updated=updated, skipped=skipped, ) + transaction.on_commit(invalidate_organization_api_cache) + return result @classmethod def _organization_defaults( diff --git a/src/organizations/test_companies.py b/src/organizations/test_companies.py index 1fea671..8a48023 100644 --- a/src/organizations/test_companies.py +++ b/src/organizations/test_companies.py @@ -8,7 +8,16 @@ from decimal import Decimal from hashlib import sha256 from uuid import UUID, uuid5 -from apps.parsers.models import ParserLoadLog +from apps.parsers.models import ( + FinancialReport, + GenericParserRecord, + IndustrialCertificateRecord, + IndustrialProductRecord, + InspectionRecord, + ManufacturerRecord, + ParserLoadLog, + ProcurementRecord, +) from django.db.models import Count, Max, Min from django.utils import timezone @@ -105,6 +114,21 @@ class TestCompanyDatasetService: **record_defaults, }, ) + legacy_record = cls._sync_parser_result_record( + source=source, + external_id=external_id, + organization=organization, + defaults=record_defaults, + index=index, + ) + legacy_module = legacy_record.__class__.__module__.removesuffix( + ".models" + ) + record.legacy_model = ( + f"{legacy_module}.{legacy_record.__class__.__name__}" + ) + record.legacy_pk = str(legacy_record.pk) + record.save(update_fields=["legacy_model", "legacy_pk", "updated_at"]) if source == ParserLoadLog.Source.FNS_REPORTS: cls._refresh_financial_lines(record=record, index=index) @@ -129,6 +153,7 @@ class TestCompanyDatasetService: company_uids = cls.company_uids() queryset = Organization.objects.filter(uid__in=company_uids) deleted_count = queryset.count() + cls._delete_parser_result_records(company_uids) extension_models = { descriptor.extension_model for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items() @@ -140,6 +165,172 @@ class TestCompanyDatasetService: cls._invalidate_caches() return TestCompanyDatasetResult(organizations_deleted=deleted_count) + @classmethod + def _sync_parser_result_record( + cls, + *, + source: str, + external_id: str, + organization: Organization, + defaults: dict, + index: int, + ): + """Mirror demo rows into the parser tables served by public source APIs.""" + payload = defaults.get("payload", {}) + load_batch = 9_000_000 + index + common = { + "load_batch": load_batch, + "registry_organization": organization, + } + + if source == ParserLoadLog.Source.INDUSTRIAL: + legacy_record, _ = IndustrialCertificateRecord.objects.update_or_create( + certificate_number=payload["certificate_number"], + defaults={ + **common, + "issue_date": payload["issue_date"], + "issue_date_normalized": date.fromisoformat( + payload["issue_date_normalized"] + ), + "expiry_date": payload["expiry_date"], + "expiry_date_normalized": date.fromisoformat( + payload["expiry_date_normalized"] + ), + "certificate_file_url": payload["certificate_file_url"], + "organisation_name": organization.name, + "inn": organization.inn, + "ogrn": organization.ogrn, + }, + ) + return legacy_record + + if source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: + legacy_record, _ = IndustrialProductRecord.objects.update_or_create( + registry_number=payload["registry_number"], + defaults={ + **common, + "full_organisation_name": payload["full_organisation_name"], + "ogrn": organization.ogrn, + "inn": organization.inn, + "product_name": payload["product_name"], + "product_model": payload["product_model"], + "okpd2_code": payload["okpd2_code"], + "tnved_code": payload["tnved_code"], + "regulatory_document": payload["regulatory_document"], + }, + ) + return legacy_record + + if source == ParserLoadLog.Source.MANUFACTURES: + legacy_record, _ = ManufacturerRecord.objects.update_or_create( + inn=organization.inn, + defaults={ + **common, + "full_legal_name": payload["full_legal_name"], + "ogrn": organization.ogrn, + "address": payload["address"], + }, + ) + return legacy_record + + if source == ParserLoadLog.Source.INSPECTIONS: + legacy_record, _ = InspectionRecord.objects.update_or_create( + registration_number=payload["registration_number"], + defaults={ + **common, + "inn": organization.inn, + "ogrn": organization.ogrn, + "organisation_name": organization.name, + "control_authority": payload["control_authority"], + "inspection_type": payload["inspection_type"], + "inspection_form": payload["inspection_form"], + "start_date": payload["start_date"], + "start_date_normalized": date.fromisoformat( + payload["start_date_normalized"] + ), + "end_date": payload["end_date"], + "end_date_normalized": date.fromisoformat( + payload["end_date_normalized"] + ), + "status": payload["status"], + "legal_basis": payload["legal_basis"], + "is_federal_law_248": True, + "data_year": payload["data_year"], + "data_month": payload["data_month"], + }, + ) + return legacy_record + + if source == ParserLoadLog.Source.PROCUREMENTS: + legacy_record, _ = ProcurementRecord.objects.update_or_create( + purchase_number=payload["purchase_number"], + defaults={ + **common, + "purchase_name": payload["subject"], + "customer_inn": organization.inn, + "customer_kpp": organization.kpp, + "customer_ogrn": organization.ogrn, + "customer_name": organization.name, + "max_price": payload["price"], + "max_price_amount": defaults["amount"], + "publish_date": payload["published_at"], + "publish_date_normalized": date(2026, 2, 1), + "status": payload["status"], + "law_type": payload["law"], + "purchase_object_info": payload["subject"], + "href": payload["url"], + "region_code": payload["region_code"], + "data_year": payload["data_year"], + "data_month": payload["data_month"], + }, + ) + return legacy_record + + if source == ParserLoadLog.Source.FNS_REPORTS: + legacy_record, _ = FinancialReport.objects.update_or_create( + external_id=f"test-company-{index:02d}", + defaults={ + **common, + "ogrn": organization.ogrn, + "file_name": payload["file_name"], + "file_hash": payload["file_hash"], + "status": FinancialReport.Status.SUCCESS, + "source": FinancialReport.SourceType.API, + }, + ) + return legacy_record + + legacy_record, _ = GenericParserRecord.objects.update_or_create( + source=source, + external_id=external_id, + defaults={ + **common, + "inn": organization.inn, + "ogrn": organization.ogrn, + "organisation_name": organization.name, + "title": defaults.get("title", ""), + "record_date": defaults.get("record_date", ""), + "amount": defaults.get("amount"), + "status": defaults.get("status", ""), + "url": defaults.get("url", ""), + "payload": payload, + }, + ) + return legacy_record + + @staticmethod + def _delete_parser_result_records(company_uids: list[UUID]) -> None: + for model in ( + GenericParserRecord, + IndustrialCertificateRecord, + IndustrialProductRecord, + ManufacturerRecord, + InspectionRecord, + ProcurementRecord, + FinancialReport, + ): + model.objects.filter(registry_organization_id__in=company_uids).delete() + @staticmethod def _descriptors_by_group(): descriptors = {} diff --git a/src/organizations/views.py b/src/organizations/views.py index 851a978..3dcfbf1 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -32,7 +32,6 @@ from organizations.cache import ( ORGANIZATION_API_CACHE_CONTRACT_VERSION, ORGANIZATION_API_CACHE_PREFIX, get_organization_api_cache_version, - invalidate_organization_api_cache, ) from organizations.directory_import import ( OrganizationDirectoryImportError, @@ -395,7 +394,6 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): with suppress(FileNotFoundError): os.unlink(temp_path) - invalidate_organization_api_cache() return Response( { "success": True, diff --git a/src/settings/base.py b/src/settings/base.py index 0d1e19c..9322919 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -334,8 +334,10 @@ REST_FRAMEWORK = { SIMPLE_JWT = { "ACCESS_TOKEN_LIFETIME": timedelta(minutes=60), "REFRESH_TOKEN_LIFETIME": timedelta(days=7), - "ROTATE_REFRESH_TOKENS": True, - "BLACKLIST_AFTER_ROTATION": True, + # The frontend can issue concurrent refresh requests. Reusing one refresh token + # avoids a race where the first response blacklists the token used by the second. + "ROTATE_REFRESH_TOKENS": False, + "BLACKLIST_AFTER_ROTATION": False, "UPDATE_LAST_LOGIN": True, "ALGORITHM": "HS256", "VERIFYING_KEY": None, diff --git a/src/settings/test.py b/src/settings/test.py index c1bef6d..b8a2693 100644 --- a/src/settings/test.py +++ b/src/settings/test.py @@ -108,5 +108,6 @@ SIMPLE_JWT = { **globals().get("SIMPLE_JWT", {}), "ACCESS_TOKEN_LIFETIME": timedelta(minutes=5), "REFRESH_TOKEN_LIFETIME": timedelta(days=1), - "ROTATE_REFRESH_TOKENS": True, + "ROTATE_REFRESH_TOKENS": False, + "BLACKLIST_AFTER_ROTATION": False, } diff --git a/src/user/views.py b/src/user/views.py index 84b4a30..5642eb2 100644 --- a/src/user/views.py +++ b/src/user/views.py @@ -13,6 +13,7 @@ from rest_framework.exceptions import ValidationError from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework_simplejwt.serializers import TokenRefreshSerializer from rest_framework_simplejwt.views import TokenRefreshView as SimpleJWTTokenRefreshView from rest_framework_simplejwt.views import TokenVerifyView as SimpleJWTTokenVerifyView @@ -646,10 +647,20 @@ def user_profile_detail(request): return Response(profile_data) +class ReusableTokenRefreshSerializer(TokenRefreshSerializer): + """Return the unchanged refresh token for concurrent frontend requests.""" + + def validate(self, attrs): + payload = super().validate(attrs) + payload["refresh"] = attrs["refresh"] + return payload + + class TokenRefreshView(SimpleJWTTokenRefreshView): """Обновление access токена через refresh токен.""" permission_classes = [AllowAny] + serializer_class = ReusableTokenRefreshSerializer @swagger_auto_schema( tags=[AUTH_TAG], diff --git a/tests/apps/exchange/test_state_corp_services.py b/tests/apps/exchange/test_state_corp_services.py index 1498163..3720266 100644 --- a/tests/apps/exchange/test_state_corp_services.py +++ b/tests/apps/exchange/test_state_corp_services.py @@ -201,6 +201,18 @@ class StateCorpExchangeServiceTest(TestCase): "Окончание подачи заявок": "20.02.2026", }, ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.CONTRACTS, + load_batch=1, + external_id="contract-not-a-procurement", + inn=organization.inn, + ogrn=organization.ogrn, + title="Исполненный контракт", + record_date="16.02.2026", + amount="1000000.00", + status="completed", + payload={"registry_number": "contract-not-a-procurement"}, + ) GenericParserRecord.objects.create( source=ParserLoadLog.Source.ARBITRATION, load_batch=1, @@ -294,6 +306,10 @@ class StateCorpExchangeServiceTest(TestCase): self.assertEqual(package.payload_counts["labor_vacancies"], 1) payload = _decode_package_payload(package) + self.assertNotIn( + "contract-not-a-procurement", + {row["purchase_number"] for row in payload["data"]["public_procurements"]}, + ) self.assertEqual(payload["format"], StateCorpExchangeService.PAYLOAD_FORMAT) self.assertEqual(payload["schema_version"], 3) diff --git a/tests/apps/organizations/test_directory_import.py b/tests/apps/organizations/test_directory_import.py index 739abb3..6fd3ce9 100644 --- a/tests/apps/organizations/test_directory_import.py +++ b/tests/apps/organizations/test_directory_import.py @@ -2,6 +2,7 @@ from decimal import Decimal from tempfile import NamedTemporaryFile +from unittest.mock import patch from django.test import TestCase from openpyxl import Workbook @@ -61,6 +62,19 @@ class OrganizationDirectoryImportServiceTest(TestCase): self.assertTrue(organization.goz_participation) self.assertFalse(organization.opk_registry_membership) + def test_import_xlsx_invalidates_organization_api_cache_after_commit(self): + path = self._workbook_path([self._row(rn="10")]) + + with ( + patch( + "organizations.directory_import.invalidate_organization_api_cache" + ) as invalidate_cache, + self.captureOnCommitCallbacks(execute=True), + ): + OrganizationDirectoryImportService.import_xlsx(path) + + invalidate_cache.assert_called_once_with() + def test_import_xlsx_rejects_missing_required_column(self): path = self._workbook_path([], headers=SOURCE_HEADERS[:-1]) @@ -111,9 +125,7 @@ class OrganizationDirectoryImportServiceTest(TestCase): ] embedded_value = "\t".join(str(value or "") for value in first_tail) embedded_value = ( - embedded_value - + "_x000D_\n" - + "\t".join(["21", "0", "0", 'ООО "Вторая"']) + embedded_value + "_x000D_\n" + "\t".join(["21", "0", "0", 'ООО "Вторая"']) ) second_row = self._row( diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py index 65d525e..6848af0 100644 --- a/tests/apps/organizations/test_test_companies_commands.py +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -3,7 +3,16 @@ from io import StringIO from apps.exchange.state_corp_services import StateCorpExchangeService -from apps.parsers.models import ParserLoadLog +from apps.parsers.models import ( + FinancialReport, + GenericParserRecord, + IndustrialCertificateRecord, + IndustrialProductRecord, + InspectionRecord, + ManufacturerRecord, + ParserLoadLog, + ProcurementRecord, +) from django.core.management import call_command from django.test import TestCase, override_settings from organizations.models import ( @@ -79,6 +88,13 @@ class TestCompaniesCommandsTest(TestCase): ).count(), 20 * 4, ) + self.assertEqual(IndustrialCertificateRecord.objects.count(), 20) + self.assertEqual(IndustrialProductRecord.objects.count(), 20) + self.assertEqual(ManufacturerRecord.objects.count(), 20) + self.assertEqual(InspectionRecord.objects.count(), 20) + self.assertEqual(ProcurementRecord.objects.count(), 20) + self.assertEqual(FinancialReport.objects.count(), 20) + self.assertEqual(GenericParserRecord.objects.count(), 20 * 9) def test_create_updates_the_fixed_dataset_without_duplicates(self): call_command("create_test_companies", stdout=StringIO()) @@ -125,9 +141,7 @@ class TestCompaniesCommandsTest(TestCase): .order_by("rn") .values_list("inn", flat=True) ) - package = StateCorpExchangeService.build_package( - organization_inns=company_inns - ) + package = StateCorpExchangeService.build_package(organization_inns=company_inns) self.assertEqual( package.payload_counts, @@ -137,7 +151,7 @@ class TestCompaniesCommandsTest(TestCase): "manufacturers": 20, "industrial_products": 20, "prosecutor_checks": 20, - "public_procurements": 80, + "public_procurements": 60, "financial_reports": 20, "arbitration_cases": 20, "bankruptcy_procedures": 20, @@ -166,3 +180,6 @@ class TestCompaniesCommandsTest(TestCase): self.assertTrue(Organization.objects.filter(uid=untouched.uid).exists()) self.assertEqual(OrganizationSourceExtension.objects.count(), 0) self.assertEqual(OrganizationSourceRecord.objects.count(), 0) + self.assertEqual(GenericParserRecord.objects.count(), 0) + self.assertEqual(IndustrialProductRecord.objects.count(), 0) + self.assertEqual(FinancialReport.objects.count(), 0) diff --git a/tests/apps/parsers/test_checko_collection.py b/tests/apps/parsers/test_checko_collection.py new file mode 100644 index 0000000..b6b55e7 --- /dev/null +++ b/tests/apps/parsers/test_checko_collection.py @@ -0,0 +1,92 @@ +from datetime import date + +from apps.parsers import tasks as parser_tasks +from apps.parsers.checko_collection import claim_monthly_collection, finish_collection +from apps.parsers.models import CheckoCollectionAttempt +from django.test import TestCase + +from tests.apps.parsers.organization_helpers import create_directory_organization + + +class CheckoCollectionClaimTest(TestCase): + def setUp(self): + self.organization = create_directory_organization( + pn_name='ООО "Месячный лимит"', + mn_ogrn=1027700000199, + mn_inn=7701000199, + in_kpp=770101001, + mn_okpo="12345678", + ) + + def test_claim_is_unique_per_organization_source_and_calendar_month(self): + first = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.ARBITRATION, + at=date(2026, 7, 1), + ) + + duplicate = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.ARBITRATION, + at=date(2026, 7, 31), + ) + other_source = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.INSPECTIONS, + at=date(2026, 7, 31), + ) + next_month = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.ARBITRATION, + at=date(2026, 8, 1), + ) + + self.assertIsNotNone(first) + self.assertIsNone(duplicate) + self.assertIsNotNone(other_source) + self.assertIsNotNone(next_month) + + def test_failed_attempt_keeps_monthly_claim(self): + attempt = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.CONTRACTS, + at=date(2026, 7, 19), + ) + assert attempt is not None + finish_collection(attempt, records_count=0, error="quota exhausted") + + repeated = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.CONTRACTS, + at=date(2026, 7, 20), + ) + + attempt.refresh_from_db() + self.assertIsNone(repeated) + self.assertEqual(attempt.status, CheckoCollectionAttempt.Status.FAILED) + self.assertEqual(attempt.error_message, "quota exhausted") + + def test_due_filter_is_applied_before_lookup_limit(self): + self.organization.opk_registry_membership = True + self.organization.save(update_fields=["opk_registry_membership"]) + second = create_directory_organization( + pn_name='ООО "Следующая организация"', + mn_ogrn=1027700000299, + mn_inn=7701000299, + in_kpp=770101001, + mn_okpo="87654321", + ) + second.opk_registry_membership = True + second.save(update_fields=["opk_registry_membership"]) + claimed = claim_monthly_collection( + organization_id=str(self.organization.id), + source=CheckoCollectionAttempt.Source.INSPECTIONS, + ) + self.assertIsNotNone(claimed) + + targets = parser_tasks._active_registry_lookup_targets( + limit=1, + checko_source=CheckoCollectionAttempt.Source.INSPECTIONS, + ) + + self.assertEqual([target.organization_id for target in targets], [str(second.id)]) diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 05286de..38b19d0 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -33,6 +33,7 @@ from apps.parsers.clients.minpromtorg.manufactures import ( from apps.parsers.clients.proverki.client import ProverkiClientError from apps.parsers.clients.zakupki import ZakupkiClientError from apps.parsers.models import ( + CheckoCollectionAttempt, FinancialReport, ParserLoadLog, ) @@ -316,6 +317,12 @@ class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase): return_value=targets, ), patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient), + patch.object( + parser_tasks, + "claim_monthly_collection", + return_value=SimpleNamespace(), + ), + patch.object(parser_tasks, "finish_collection"), ): records = parser_tasks._fetch_checko_bankruptcy_records(proxies=None) @@ -801,6 +808,15 @@ class GenericSourceFetchTestCase(TestCase): [request.inn for request in _CheckoClient.instances[0].requests], [str(organization.mn_inn)], ) + second_result = parser_tasks.parse_arbitration_cases(limit=10, proxies=[]) + self.assertEqual(second_result["status"], "skipped") + self.assertEqual(len(_CheckoClient.instances), 1) + attempt = CheckoCollectionAttempt.objects.get( + organization_id=organization.id, + source=CheckoCollectionAttempt.Source.ARBITRATION, + ) + self.assertEqual(attempt.status, CheckoCollectionAttempt.Status.SUCCESS) + self.assertEqual(attempt.records_count, 1) @override_settings(CHECKO_API_KEY="test-key") def test_arbitration_skips_when_no_active_registry_organizations(self): @@ -1842,11 +1858,13 @@ class MinpromtorgTasksTestCase(TestCase): "parse_procurements_44fz", "parse_procurements_223fz", "parse_contracts", + "parse_registry_contracts", "parse_unfair_suppliers", "parse_fas_goz_evasion", "sync_fns_financial_reports", "parse_arbitration_cases", "parse_fedresurs_bankruptcy", + "parse_registry_inspections", "parse_fstec_registers", "parse_trudvsem_vacancies", ) @@ -1869,6 +1887,8 @@ class MinpromtorgTasksTestCase(TestCase): "sync_fns_financial_reports-id", ) delays["sync_fns_financial_reports"].assert_called_once_with(proxies=[]) + delays["parse_registry_contracts"].assert_called_once_with(proxies=[]) + delays["parse_registry_inspections"].assert_called_once_with(proxies=[]) def test_parse_all_minpromtorg_without_adapter(self): with TestHTTPServer() as server: diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index c668ffd..c8638c1 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -595,6 +595,17 @@ class TokenRefreshViewTest(APITestCase): # New refresh token should be different # Refresh token may be the same or different depending on implementation + def test_same_refresh_token_can_handle_parallel_refresh_requests(self): + data = {"refresh": self.tokens["refresh"]} + + first_response = self.client.post(self.refresh_url, data, format="json") + second_response = self.client.post(self.refresh_url, data, format="json") + + self.assertEqual(first_response.status_code, status.HTTP_200_OK) + self.assertEqual(second_response.status_code, status.HTTP_200_OK) + self.assertEqual(first_response.data["refresh"], self.tokens["refresh"]) + self.assertEqual(second_response.data["refresh"], self.tokens["refresh"]) + def test_refresh_token_invalid(self): """Test token refresh fails with invalid refresh token""" data = {"refresh": fake.pystr(min_chars=20, max_chars=50)} -- 2.39.5 From 06e8c2c3c6df4a8168b685573ae9f3c892480bc9 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Sun, 19 Jul 2026 13:36:20 +0200 Subject: [PATCH 4/6] style: format Checko collection helper --- src/apps/parsers/checko_collection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/apps/parsers/checko_collection.py b/src/apps/parsers/checko_collection.py index ab04a0b..033740f 100644 --- a/src/apps/parsers/checko_collection.py +++ b/src/apps/parsers/checko_collection.py @@ -10,7 +10,9 @@ def collection_period_month(at: date | datetime | None = None) -> date: if at is None: local_date = timezone.localdate() elif isinstance(at, datetime): - local_date = timezone.localtime(at).date() if timezone.is_aware(at) else at.date() + local_date = ( + timezone.localtime(at).date() if timezone.is_aware(at) else at.date() + ) else: local_date = at return local_date.replace(day=1) -- 2.39.5 From 59769be40052fef57e7f8169cce29c0bab0501d2 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Sun, 19 Jul 2026 16:22:13 +0200 Subject: [PATCH 5/6] fix: parse Checko API response fields --- src/apps/parsers/clients/checko/client.py | 134 +++++++++++++++------- tests/apps/parsers/test_checko.py | 130 +++++++++++++++++++++ 2 files changed, 223 insertions(+), 41 deletions(-) diff --git a/src/apps/parsers/clients/checko/client.py b/src/apps/parsers/clients/checko/client.py index fb8710e..3f25cf5 100644 --- a/src/apps/parsers/clients/checko/client.py +++ b/src/apps/parsers/clients/checko/client.py @@ -234,6 +234,14 @@ def _map_ru_keys(data: dict | None) -> dict | None: return result +def _first_present(data: dict, *keys: str, default: Any = None) -> Any: + """Return the first explicitly present value from normalized Checko data.""" + for key in keys: + if key in data: + return data[key] + return default + + @dataclass class CheckoClient: """ @@ -471,7 +479,7 @@ class CheckoClient: def _parse_bankruptcy_message(self, data: dict) -> BankruptcyMessage: """Парсинг сообщения о банкротстве.""" return BankruptcyMessage( - type=data.get("type", ""), + type=_first_present(data, "type", "type_name", default=""), date=data.get("date", ""), case_number=data.get("case_number"), ) @@ -1183,37 +1191,54 @@ class CheckoClient: ogrn=data.get("ogrn"), inn=data.get("inn"), kpp=data.get("kpp"), - name=data.get("name"), + name=_first_present(data, "name", "short_name", "full_name"), region_code=data.get("region_code"), ) def _parse_contract(self, data: dict) -> Contract: """Парсинг контракта.""" + suppliers = _first_present(data, "suppliers", "постав", default=[]) + if isinstance(suppliers, dict): + suppliers = [suppliers] + objects = _first_present(data, "objects", "объекты", default=[]) + subject = data.get("subject") + if not subject and isinstance(objects, list): + subject = ( + "; ".join( + str(item.get("name") or item.get("наим")) + for item in objects + if isinstance(item, dict) and (item.get("name") or item.get("наим")) + ) + or None + ) return Contract( - registry_number=data.get("registry_number", ""), - publish_date=data.get("publish_date"), - sign_date=data.get("sign_date"), - execution_date=data.get("execution_date"), - price=data.get("price"), + registry_number=_first_present( + data, "registry_number", "регномер", default="" + ), + publish_date=_first_present(data, "publish_date", "date"), + sign_date=_first_present(data, "sign_date", "date"), + execution_date=_first_present(data, "execution_date", "датаисп"), + price=_first_present(data, "price", "цена"), currency_code=data.get("currency_code"), status=data.get("status"), - subject=data.get("subject"), + subject=subject, law=data.get("law"), purchase_number=data.get("purchase_number"), - customer=self._parse_contract_party(data.get("customer")), - suppliers=tuple( - self._parse_contract_party(s) for s in data.get("suppliers", []) if s + customer=self._parse_contract_party( + _first_present(data, "customer", "заказ") ), - url=data.get("url"), + suppliers=tuple(self._parse_contract_party(s) for s in suppliers if s), + url=_first_present(data, "url", "стреис"), ) def _parse_contracts_data(self, data: dict | None) -> ContractsData | None: """Парсинг данных о контрактах.""" if not data: return None + records = _first_present(data, "contracts", "records", default=[]) return ContractsData( - contracts=tuple(self._parse_contract(c) for c in data.get("contracts", [])), - pagination=self._parse_pagination(data.get("pagination")), + contracts=tuple(self._parse_contract(c) for c in records), + pagination=self._parse_pagination(data.get("pagination") or data), total_sum=data.get("total_sum"), ) @@ -1278,32 +1303,52 @@ class CheckoClient: def _parse_inspection(self, data: dict) -> Inspection: """Парсинг проверки.""" + authority = _first_present(data, "authority", "оргконтр", default={}) + objects = _first_present(data, "objects", "объекты", default=[]) + results = [ + str(item.get("результ")) + for item in objects + if isinstance(item, dict) and item.get("результ") + ] return Inspection( - id=data.get("id"), - erp_id=data.get("erp_id"), - plan_date_from=data.get("plan_date_from"), - plan_date_to=data.get("plan_date_to"), + id=_first_present(data, "id", "number"), + erp_id=_first_present(data, "erp_id", "number"), + plan_date_from=_first_present(data, "plan_date_from", "start_date"), + plan_date_to=_first_present(data, "plan_date_to", "датаоконч"), actual_date_from=data.get("actual_date_from"), actual_date_to=data.get("actual_date_to"), - type=data.get("type"), - form=data.get("form"), + type=_first_present(data, "type", "типпров"), + form=_first_present(data, "form", "типрасп"), status=data.get("status"), - authority_name=data.get("authority_name"), - authority_ogrn=data.get("authority_ogrn"), - subject=data.get("subject"), - result=data.get("result"), - violations_found=data.get("violations_found", False), + authority_name=_first_present( + data, + "authority_name", + default=authority.get("name") if isinstance(authority, dict) else None, + ), + authority_ogrn=_first_present( + data, + "authority_ogrn", + default=authority.get("ogrn") if isinstance(authority, dict) else None, + ), + subject=_first_present(data, "subject", "цель"), + result=_first_present( + data, + "result", + default="; ".join(results) or None, + ), + violations_found=bool( + _first_present(data, "violations_found", "наруш", default=False) + ), ) def _parse_inspections_data(self, data: dict | None) -> InspectionsData | None: """Парсинг данных о проверках.""" if not data: return None + records = _first_present(data, "inspections", "records", default=[]) return InspectionsData( - inspections=tuple( - self._parse_inspection(i) for i in data.get("inspections", []) - ), - pagination=self._parse_pagination(data.get("pagination")), + inspections=tuple(self._parse_inspection(i) for i in records), + pagination=self._parse_pagination(data.get("pagination") or data), ) def get_inspections(self, request: InspectionsRequest) -> InspectionsResponse: @@ -1465,20 +1510,26 @@ class CheckoClient: def _parse_legal_case(self, data: dict) -> LegalCase: """Парсинг арбитражного дела.""" return LegalCase( - case_number=data.get("case_number", ""), - court_name=data.get("court_name"), - type=data.get("type"), + case_number=_first_present(data, "case_number", "number", default=""), + court_name=_first_present(data, "court_name", "суд"), + type=_first_present(data, "type", "type_name"), category=data.get("category"), - status=data.get("status"), - filing_date=data.get("filing_date"), + status=_first_present( + data, + "status", + default="active" if data.get("практив") else None, + ), + filing_date=_first_present(data, "filing_date", "date"), result_date=data.get("result_date"), - claim_amount=data.get("claim_amount"), + claim_amount=_first_present(data, "claim_amount", "суммиск"), awarded_amount=data.get("awarded_amount"), plaintiffs=tuple( - self._parse_case_party(p) for p in data.get("plaintiffs", []) + self._parse_case_party(p) + for p in _first_present(data, "plaintiffs", "ист", default=[]) ), defendants=tuple( - self._parse_case_party(d) for d in data.get("defendants", []) + self._parse_case_party(d) + for d in _first_present(data, "defendants", "ответ", default=[]) ), third_parties=tuple( self._parse_case_party(t) for t in data.get("third_parties", []) @@ -1486,17 +1537,18 @@ class CheckoClient: instances=tuple( self._parse_case_instance(i) for i in data.get("instances", []) ), - url=data.get("url"), + url=_first_present(data, "url", "стркад"), ) def _parse_legal_cases_data(self, data: dict | None) -> LegalCasesData | None: """Парсинг данных о делах.""" if not data: return None + records = _first_present(data, "cases", "records", default=[]) return LegalCasesData( - cases=tuple(self._parse_legal_case(c) for c in data.get("cases", [])), - pagination=self._parse_pagination(data.get("pagination")), - total_claim_amount=data.get("total_claim_amount"), + cases=tuple(self._parse_legal_case(c) for c in records), + pagination=self._parse_pagination(data.get("pagination") or data), + total_claim_amount=_first_present(data, "total_claim_amount", "общсуммиск"), ) def get_legal_cases(self, request: LegalCasesRequest) -> LegalCasesResponse: diff --git a/tests/apps/parsers/test_checko.py b/tests/apps/parsers/test_checko.py index 46f9f48..9671f7a 100644 --- a/tests/apps/parsers/test_checko.py +++ b/tests/apps/parsers/test_checko.py @@ -436,6 +436,52 @@ class CheckoClientApiTest(SimpleTestCase): self.assertEqual(response.data.contracts[0].registry_number, contract_number) + def test_get_contracts_supports_checko_russian_response(self): + inn = _digits(10) + contract_number = _digits(12) + + with TestHTTPServer() as server: + server.add_json( + "/v2/contracts", + { + "data": { + "ЗапВсего": 1, + "СтрВсего": 1, + "СтрТекущ": 1, + "Записи": [ + { + "РегНомер": contract_number, + "СтрЕИС": "https://zakupki.gov.ru/contract/1", + "Дата": "2026-01-10", + "ДатаИсп": "2026-12-31", + "Цена": 125000, + "Заказ": { + "ИНН": inn, + "НаимПолн": "Тестовый заказчик", + }, + "Постав": { + "ИНН": _digits(10), + "НаимСокр": "Тестовый поставщик", + }, + "Объекты": [{"Наим": "Поставка оборудования"}], + } + ], + }, + "meta": _meta_ok(), + }, + ) + client = _client_for(server) + response = client.get_contracts( + ContractsRequest(inn=inn, law=ContractLaw.FZ44) + ) + + contract = response.data.contracts[0] + self.assertEqual(contract.registry_number, contract_number) + self.assertEqual(contract.subject, "Поставка оборудования") + self.assertEqual(contract.customer.inn, inn) + self.assertEqual(len(contract.suppliers), 1) + self.assertEqual(response.data.pagination.total_records, 1) + def test_get_legal_cases(self): inn = "".join(str(fake.random_int(0, 9)) for _ in range(10)) case_number = fake.bothify(text="A-####/####") @@ -465,6 +511,47 @@ class CheckoClientApiTest(SimpleTestCase): self.assertEqual(response.data.cases[0].case_number, case_number) + def test_get_legal_cases_supports_checko_russian_response(self): + inn = _digits(10) + case_number = "А40-12345/2026" + + with TestHTTPServer() as server: + server.add_json( + "/v2/legal-cases", + { + "data": { + "ЗапВсего": 1, + "СтрВсего": 1, + "СтрТекущ": 1, + "ОбщСуммИск": 250000, + "Записи": [ + { + "Номер": case_number, + "СтрКАД": "https://kad.arbitr.ru/Card/1", + "Дата": "2026-02-03", + "Тип": "Гражданское", + "ПрАктив": True, + "Суд": "Арбитражный суд города Москвы", + "Ист": [{"ИНН": inn, "Наим": "Истец"}], + "Ответ": [{"ИНН": _digits(10), "Наим": "Ответчик"}], + "СуммИск": 250000, + } + ], + }, + "meta": _meta_ok(), + }, + ) + client = _client_for(server) + response = client.get_legal_cases(LegalCasesRequest(inn=inn)) + + legal_case = response.data.cases[0] + self.assertEqual(legal_case.case_number, case_number) + self.assertEqual(legal_case.claim_amount, 250000) + self.assertEqual(legal_case.plaintiffs[0].inn, inn) + self.assertEqual(legal_case.status, "active") + self.assertEqual(response.data.total_claim_amount, 250000) + self.assertEqual(response.data.pagination.total_records, 1) + def test_api_error_handling(self): with TestHTTPServer() as server: server.add_json( @@ -583,6 +670,49 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): self.assertEqual(response.data.inspections[0].id, inspection_id) + def test_get_inspections_supports_checko_russian_response(self): + inn = _digits(10) + inspection_number = "77260041000101234567" + + with TestHTTPServer() as server: + server.add_json( + "/v2/inspections", + { + "data": { + "ЗапВсего": 1, + "СтрВсего": 1, + "СтрТекущ": 1, + "Записи": [ + { + "Номер": inspection_number, + "Статус": "Завершено", + "Наруш": True, + "ТипРасп": "Внеплановая", + "ТипПров": "Выездная", + "ДатаНач": "2026-03-01", + "ДатаОконч": "2026-03-05", + "Цель": "Контроль обязательных требований", + "ОргКонтр": { + "ОГРН": _digits(13), + "Наим": "Контрольный орган", + }, + "Объекты": [{"Результ": "Выявлены нарушения"}], + } + ], + }, + "meta": _meta_ok(), + }, + ) + client = _client_for(server) + response = client.get_inspections(InspectionsRequest(inn=inn)) + + inspection = response.data.inspections[0] + self.assertEqual(inspection.id, inspection_number) + self.assertEqual(inspection.authority_name, "Контрольный орган") + self.assertEqual(inspection.result, "Выявлены нарушения") + self.assertTrue(inspection.violations_found) + self.assertEqual(response.data.pagination.total_records, 1) + def test_iter_inspections_pagination(self): inn = _digits(10) -- 2.39.5 From c344bef777da9b15de40783b2d372b03e1fc80d5 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Sun, 19 Jul 2026 17:46:15 +0200 Subject: [PATCH 6/6] feat: restore registry procurement collection --- src/apps/parsers/clients/checko/client.py | 13 +- src/apps/parsers/clients/common/__init__.py | 2 + .../parsers/clients/common/eis_registry.py | 142 +++++++ .../0028_registry_procurement_claims.py | 43 ++ src/apps/parsers/models.py | 3 + src/apps/parsers/source_cards.py | 40 +- src/apps/parsers/tasks.py | 371 ++++++++++++++++-- tests/apps/parsers/test_checko_parsers.py | 52 +++ .../apps/parsers/test_eis_registry_client.py | 124 ++++++ .../test_registry_procurement_tasks.py | 252 ++++++++++++ .../apps/parsers/test_source_cards_service.py | 14 +- tests/apps/parsers/test_source_cards_views.py | 29 +- tests/apps/parsers/test_sources_api_e2e.py | 5 +- tests/apps/parsers/test_tasks.py | 16 +- 14 files changed, 1010 insertions(+), 96 deletions(-) create mode 100644 src/apps/parsers/clients/common/eis_registry.py create mode 100644 src/apps/parsers/migrations/0028_registry_procurement_claims.py create mode 100644 tests/apps/parsers/test_eis_registry_client.py create mode 100644 tests/apps/parsers/test_registry_procurement_tasks.py diff --git a/src/apps/parsers/clients/checko/client.py b/src/apps/parsers/clients/checko/client.py index 3f25cf5..d51833b 100644 --- a/src/apps/parsers/clients/checko/client.py +++ b/src/apps/parsers/clients/checko/client.py @@ -193,7 +193,18 @@ RU_FIELD_MAP = { "ЕФРСБ": "bankruptcy", "НомерДела": "case_number", # РНП - "НедобПост": "unfair_supplier", + "НедобПост": "is_unfair_supplier", + "НедобПостЗап": "unfair_supplier", + "РеестрНомер": "registry_number", + "ДатаПуб": "publish_date", + "ДатаУтв": "approval_date", + "ЗаказНаимСокр": "customer_short_name", + "ЗаказНаимПолн": "customer_full_name", + "ЗаказИНН": "customer_inn", + "ЗаказКПП": "customer_kpp", + "ЗакупНомер": "purchase_number", + "ЗакупОпис": "purchase_description", + "ЦенаКонтр": "contract_price", # Численность "СЧР": "employees_count", # Налоги diff --git a/src/apps/parsers/clients/common/__init__.py b/src/apps/parsers/clients/common/__init__.py index f64c3a6..ba255ae 100644 --- a/src/apps/parsers/clients/common/__init__.py +++ b/src/apps/parsers/clients/common/__init__.py @@ -1,5 +1,6 @@ """Общие клиенты и DTO для новых разнородных источников.""" +from apps.parsers.clients.common.eis_registry import EisRegistryProcurementClient from apps.parsers.clients.common.schemas import GenericParserItem from apps.parsers.clients.common.structured import ( StructuredDataClient, @@ -8,6 +9,7 @@ from apps.parsers.clients.common.structured import ( __all__ = [ "GenericParserItem", + "EisRegistryProcurementClient", "StructuredDataClient", "StructuredDataClientError", ] diff --git a/src/apps/parsers/clients/common/eis_registry.py b/src/apps/parsers/clients/common/eis_registry.py new file mode 100644 index 0000000..82746a6 --- /dev/null +++ b/src/apps/parsers/clients/common/eis_registry.py @@ -0,0 +1,142 @@ +"""Addressed EIS procurement lookup for organizations from the OПК registry.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field, replace +from datetime import date +from urllib.parse import urlencode + +from apps.parsers.clients.common.schemas import GenericParserItem +from apps.parsers.clients.common.structured import StructuredDataClient +from bs4 import BeautifulSoup +from dateutil.relativedelta import relativedelta + +EIS_PROCUREMENT_SEARCH_URL = ( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html" +) +EIS_SEARCH_PAGE_MAX_SIZE_BYTES = 4 * 1024 * 1024 +EIS_RECORDS_PER_PAGE = 50 + + +def _digits(value: str) -> str: + return re.sub(r"\D+", "", str(value or "")) + + +@dataclass +class EisRegistryProcurementClient: + """Fetch all recent EIS notices for one exact customer INN and law.""" + + source: str + law: str + proxies: list[str] | None = None + timeout: int = 120 + lookback_months: int = 12 + max_pages: int = 1000 + _structured_client: StructuredDataClient | None = field( + default=None, + repr=False, + ) + + @property + def structured_client(self) -> StructuredDataClient: + if self._structured_client is None: + self._structured_client = StructuredDataClient( + source=self.source, + proxies=self.proxies, + timeout=self.timeout, + ) + return self._structured_client + + def fetch_for_customer( + self, + *, + inn: str, + organization_name: str, + organization_id: str, + today: date | None = None, + ) -> list[GenericParserItem]: + """Fetch every result page and keep only cards with the exact customer INN.""" + customer_inn = _digits(inn) + if not customer_inn: + return [] + + period_end = today or date.today() + period_start = period_end - relativedelta(months=self.lookback_months) + records_by_external_id: dict[str, GenericParserItem] = {} + page_number = 1 + + while page_number <= self.max_pages: + url = self._build_search_url( + inn=customer_inn, + page_number=page_number, + period_start=period_start, + period_end=period_end, + ) + content = self.structured_client.http_client.download_file( + url, + max_size_bytes=EIS_SEARCH_PAGE_MAX_SIZE_BYTES, + ) + page_records = self.structured_client.fetch_records( + content=content, + file_name="results.html", + ) + for record in page_records: + if _digits(record.inn) != customer_inn: + continue + payload = dict(record.payload) + payload.update( + { + "provider": "eis", + "organization_id": organization_id, + "customer_inn": customer_inn, + "customer_name": organization_name, + "law": self.law, + "lookback_months": self.lookback_months, + } + ) + records_by_external_id[record.external_id] = replace( + record, + inn=customer_inn, + organisation_name=organization_name, + payload=payload, + ) + + last_page = self._last_page_number(content) + if page_number >= last_page: + break + page_number += 1 + + return list(records_by_external_id.values()) + + def _build_search_url( + self, + *, + inn: str, + page_number: int, + period_start: date, + period_end: date, + ) -> str: + law_param = "fz44" if self.law == "44" else "fz223" + params = { + law_param: "on", + "searchString": inn, + "strictEqual": "true", + "publishDateFrom": period_start.strftime("%d.%m.%Y"), + "publishDateTo": period_end.strftime("%d.%m.%Y"), + "sortBy": "UPDATE_DATE", + "pageNumber": str(page_number), + "sortDirection": "false", + "recordsPerPage": f"_{EIS_RECORDS_PER_PAGE}", + } + return f"{EIS_PROCUREMENT_SEARCH_URL}?{urlencode(params)}" + + @staticmethod + def _last_page_number(content: bytes) -> int: + soup = BeautifulSoup(content, "html.parser") + page_numbers = [1] + for node in soup.select("[data-pagenumber]"): + value = str(node.get("data-pagenumber") or "") + if value.isdigit(): + page_numbers.append(int(value)) + return max(page_numbers) diff --git a/src/apps/parsers/migrations/0028_registry_procurement_claims.py b/src/apps/parsers/migrations/0028_registry_procurement_claims.py new file mode 100644 index 0000000..683e582 --- /dev/null +++ b/src/apps/parsers/migrations/0028_registry_procurement_claims.py @@ -0,0 +1,43 @@ +from django.db import migrations, models + +LEGACY_WEEKLY_TASK_NAMES = [ + "parser:procurements_44fz:weekly-saturday-msk", + "parser:procurements_223fz:weekly-saturday-msk", + "parser:contracts:weekly-saturday-msk", + "parser:unfair_suppliers:weekly-saturday-msk", +] + + +def disable_legacy_weekly_tasks(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name__in=LEGACY_WEEKLY_TASK_NAMES).update(enabled=False) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0027_checkocollectionattempt"), + ] + + operations = [ + migrations.AlterField( + model_name="checkocollectionattempt", + name="source", + field=models.CharField( + choices=[ + ("arbitration", "Арбитражные дела"), + ("bankruptcy", "Банкротства"), + ("contracts", "Контракты"), + ("inspections", "Проверки"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ], + max_length=32, + verbose_name="источник", + ), + ), + migrations.RunPython( + disable_legacy_weekly_tasks, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 771f669..b2763d1 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -125,6 +125,9 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model): BANKRUPTCY = "bankruptcy", _("Банкротства") CONTRACTS = "contracts", _("Контракты") INSPECTIONS = "inspections", _("Проверки") + PROCUREMENTS_44FZ = "procurements_44fz", _("Закупки 44-ФЗ") + PROCUREMENTS_223FZ = "procurements_223fz", _("Закупки 223-ФЗ") + UNFAIR_SUPPLIERS = "unfair_suppliers", _("Недобросовестные поставщики") class Status(models.TextChoices): IN_PROGRESS = "in_progress", _("В процессе") diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index 93eabd0..29a7ac6 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -130,18 +130,16 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( description="Данные ЕИС закупок по тендерам и заказчикам.", order=20, task_names=( - "apps.parsers.tasks.parse_procurements", - "apps.parsers.tasks.sync_procurements", "apps.parsers.tasks.parse_procurements_44fz", "apps.parsers.tasks.parse_procurements_223fz", - "apps.parsers.tasks.parse_contracts", "apps.parsers.tasks.parse_registry_contracts", + "apps.parsers.tasks.parse_registry_enrichment_sources", ), source_items=( SourceItemDefinition( code="procurements", - title="Единая информационная система закупок", - description=("Закупки и связанные данные из ЕИС по 44-ФЗ и 223-ФЗ."), + title="Общие закупки", + description="Объединение закупок 44-ФЗ и 223-ФЗ без дублирования.", parser_source=ParserLoadLog.Source.PROCUREMENTS, ), SourceItemDefinition( @@ -168,7 +166,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( name="region_code", label="Код региона", description="Код региона ЕИС, например 77 для Москвы.", - required=True, + required=False, ), RefreshParamDefinition( name="law_type", @@ -368,6 +366,10 @@ SOURCE_RECORD_SOURCES_BY_ITEM_CODE = { for item in definition.source_items if item.parser_source } +SOURCE_RECORD_SOURCES_BY_ITEM_CODE["procurements"] = [ + ParserLoadLog.Source.PROCUREMENTS_44FZ, + ParserLoadLog.Source.PROCUREMENTS_223FZ, +] class SourceCardService: @@ -455,7 +457,14 @@ class SourceCardService: source_items = [ cls._build_source_item(item, context) for item in definition.source_items ] - records_count = sum(item["records_count"] for item in source_items) + if definition.slug == "public-procurements": + items_by_code = {item["code"]: item for item in source_items} + records_count = ( + items_by_code["procurements"]["records_count"] + + items_by_code["contracts"]["records_count"] + ) + else: + records_count = sum(item["records_count"] for item in source_items) organizations_count = cls._get_card_organizations_count( definition, source_items, context ) @@ -998,27 +1007,18 @@ class SourceCardService: return [task_info] if definition.slug == "public-procurements": - from apps.parsers.tasks import sync_procurements + from apps.parsers.tasks import parse_registry_enrichment_sources task_info = cls._enqueue_task( - task=sync_procurements, - task_name="apps.parsers.tasks.sync_procurements", + task=parse_registry_enrichment_sources, + task_name="apps.parsers.tasks.parse_registry_enrichment_sources", requested_by_id=requested_by_id, meta={ "source_card": definition.slug, - "source": ParserLoadLog.Source.PROCUREMENTS, - "region_code": params["region_code"], - "law_type": params.get("law_type", "44"), + "source": ParserLoadLog.Source.PROCUREMENTS_44FZ, }, kwargs={ "requested_by_id": requested_by_id, - "region_code": params["region_code"], - "law_type": params.get("law_type", "44"), - **{ - key: value - for key, value in params.items() - if key in {"current_year", "current_month"} - }, }, ) return [task_info] diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 4c1dda9..de9ee60 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -37,7 +37,11 @@ from apps.parsers.clients.checko import ( SearchType, ) from apps.parsers.clients.checko.exceptions import CheckoError -from apps.parsers.clients.common import GenericParserItem, StructuredDataClient +from apps.parsers.clients.common import ( + EisRegistryProcurementClient, + GenericParserItem, + StructuredDataClient, +) from apps.parsers.clients.fns import FNSApiClient, FNSApiReport from apps.parsers.clients.gisp import GispProductsClient from apps.parsers.clients.minpromtorg import ( @@ -82,6 +86,7 @@ FEDRESURS_CHECKO_FALLBACK_LIMIT = 100 ARBITRATION_CHECKO_LIMIT = 100 REGISTRY_INSPECTIONS_CHECKO_LIMIT = 1000 REGISTRY_CONTRACTS_CHECKO_LIMIT = 1000 +REGISTRY_ENRICHMENT_BATCH_SIZE = 250 FSTEC_CHECKO_IDENTITY_LOOKUP_LIMIT = 1000 PARSER_STALE_LOAD_MAX_AGE_MINUTES = 90 PARSER_SOFT_TIME_LIMIT_SECONDS = 15 * 60 @@ -177,9 +182,15 @@ def _active_registry_lookup_targets( *, limit: int | None = None, checko_source: str | None = None, + require_inn: bool = False, + organization_ids: list[str] | None = None, ) -> list[RegistryLookupTarget]: """Вернуть организации, которые сейчас состоят хотя бы в одном реестре.""" queryset = SourceOrganization.objects.filter(opk_registry_membership=True) + if organization_ids is not None: + queryset = queryset.filter(uid__in=organization_ids) + if require_inn: + queryset = queryset.exclude(inn="") if checko_source is not None: queryset = queryset.exclude( checko_collection_attempts__source=checko_source, @@ -215,6 +226,17 @@ def _active_registry_lookup_targets( return targets +def _resolve_registry_enrichment_limit(limit: int | None) -> int: + return _resolve_lookup_limit( + limit, + default=getattr( + settings, + "REGISTRY_ENRICHMENT_BATCH_SIZE", + REGISTRY_ENRICHMENT_BATCH_SIZE, + ), + ) + + def _resolve_proxies(proxies: list[str] | None) -> list[str] | None: """ Разрешить итоговый список прокси. @@ -1377,6 +1399,226 @@ def _fetch_checko_registry_inspections( return records +def _fetch_eis_registry_procurement_records( + *, + source: str, + law: str, + limit: int | None, + proxies: list[str] | None, + organization_ids: list[str] | None = None, +) -> list[GenericParserItem]: + """Fetch addressed EIS notices for the next monthly OПК registry batch.""" + resolved_limit = _resolve_registry_enrichment_limit(limit) + if resolved_limit <= 0: + return [] + + claim_source = { + "44": CheckoCollectionAttempt.Source.PROCUREMENTS_44FZ, + "223": CheckoCollectionAttempt.Source.PROCUREMENTS_223FZ, + }[law] + targets = _active_registry_lookup_targets( + limit=resolved_limit, + checko_source=claim_source, + require_inn=True, + organization_ids=organization_ids, + ) + if not targets: + raise ParserSourceSkipped( + f"no OПК registry organizations are due for monthly {law}-FZ lookup" + ) + + client = EisRegistryProcurementClient( + source=source, + law=law, + proxies=proxies, + ) + records: list[GenericParserItem] = [] + failed_lookups = 0 + for target in targets: + attempt = claim_monthly_collection( + organization_id=target.organization_id, + source=claim_source, + ) + if attempt is None: + continue + records_before = len(records) + try: + records.extend( + client.fetch_for_customer( + inn=target.inn, + organization_name=target.name, + organization_id=target.organization_id, + ) + ) + except HTTPClientError as exc: + failed_lookups += 1 + finish_collection(attempt, records_count=0, error=exc) + logger.info( + "EIS %s-FZ lookup failed for customer_inn=%s: %s", + law, + target.inn, + exc, + ) + else: + finish_collection( + attempt, + records_count=len(records) - records_before, + ) + + if failed_lookups == len(targets) and not records: + raise ParserSourceSkipped(f"EIS {law}-FZ lookups failed for all targets") + return records + + +def _checko_unfair_supplier_items( + *, + company, + target: RegistryLookupTarget, +) -> list[GenericParserItem]: + """Convert Checko НедобПостЗап values into supplier-bound source records.""" + company_inn = _normalize_identifier(getattr(company, "inn", "")) + company_ogrn = _normalize_identifier(getattr(company, "ogrn", "")) + if target.inn and company_inn != target.inn: + return [] + if not target.inn and target.ogrn and company_ogrn != target.ogrn: + return [] + + records = [] + for item in getattr(company, "unfair_supplier", ()): + registry_number = str(getattr(item, "registry_number", "") or "").strip() + purchase_number = str(getattr(item, "purchase_number", "") or "").strip() + stable_id = registry_number or purchase_number + if not stable_id: + stable_id = hashlib.sha256( + repr(item).encode("utf-8", errors="replace") + ).hexdigest() + customer = { + "short_name": getattr(item, "customer_short_name", None), + "full_name": getattr(item, "customer_full_name", None), + "inn": getattr(item, "customer_inn", None), + "kpp": getattr(item, "customer_kpp", None), + } + contract_price = getattr(item, "contract_price", None) + records.append( + GenericParserItem( + source=ParserLoadLog.Source.UNFAIR_SUPPLIERS, + external_id=f"checko-unfair-supplier:{stable_id}", + inn=target.inn, + ogrn=target.ogrn, + organisation_name=target.name, + title=str( + getattr(item, "purchase_description", None) + or "Запись реестра недобросовестных поставщиков" + ), + record_date=str( + getattr(item, "publish_date", None) + or getattr(item, "approval_date", None) + or "" + ), + amount=( + Decimal(str(contract_price)) if contract_price is not None else None + ), + status="included", + payload={ + "provider": "checko", + "organization_id": target.organization_id, + "registry_number": registry_number, + "publish_date": getattr(item, "publish_date", None), + "approval_date": getattr(item, "approval_date", None), + "purchase_number": purchase_number, + "purchase_description": getattr( + item, + "purchase_description", + None, + ), + "contract_price": contract_price, + "supplier": { + "inn": target.inn, + "ogrn": target.ogrn, + "name": target.name, + }, + "customer": customer, + }, + ) + ) + return records + + +def _fetch_checko_unfair_supplier_records( # noqa: C901 + *, + limit: int | None, + proxies: list[str] | None, + organization_ids: list[str] | None = None, +) -> list[GenericParserItem]: + """Fetch RNP entries through one Checko /company request per OПК organization.""" + api_key = getattr(settings, "CHECKO_API_KEY", "") + if not api_key: + raise ParserSourceSkipped("CHECKO_API_KEY is empty; RNP parser skipped") + + resolved_limit = _resolve_registry_enrichment_limit(limit) + if resolved_limit <= 0: + return [] + targets = _active_registry_lookup_targets( + limit=resolved_limit, + checko_source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS, + organization_ids=organization_ids, + ) + if not targets: + raise ParserSourceSkipped( + "no OПК registry organizations are due for monthly RNP lookup" + ) + + checko_proxies = ( + proxies if getattr(settings, "CHECKO_USE_RUNTIME_PROXIES", False) else None + ) + client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30) + records: list[GenericParserItem] = [] + failed_lookups = 0 + rate_limited = False + for target in targets: + attempt = claim_monthly_collection( + organization_id=target.organization_id, + source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS, + ) + if attempt is None: + continue + records_before = len(records) + try: + response = client.get_company( + CompanyRequest(inn=target.inn or None, ogrn=target.ogrn or None) + ) + except CheckoRateLimitError as exc: + finish_collection(attempt, records_count=0, error=exc) + failed_lookups += 1 + rate_limited = True + logger.warning("Checko RNP lookup stopped: quota/rate limit reached") + break + except CheckoError as exc: + finish_collection(attempt, records_count=0, error=exc) + failed_lookups += 1 + logger.info( + "Checko RNP lookup failed for target=%s: %s", + target.inn or target.ogrn, + exc, + ) + else: + if response.data is not None: + records.extend( + _checko_unfair_supplier_items( + company=response.data, + target=target, + ) + ) + finish_collection( + attempt, + records_count=len(records) - records_before, + ) + + if not records and (rate_limited or failed_lookups == len(targets)): + raise ParserSourceSkipped("Checko RNP lookups failed for all targets") + return records + + def _contract_party_payload(party) -> dict: if party is None: return {} @@ -2320,26 +2562,13 @@ def parse_all_sources( generic_results = {} if not getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False): generic_results = { - "procurements_44fz": parse_procurements_44fz.delay(proxies=proxies).id, - "procurements_223fz": parse_procurements_223fz.delay( - proxies=proxies + "registry_enrichment": parse_registry_enrichment_sources.delay( + limit=_resolve_registry_enrichment_limit(None), + proxies=proxies, ).id, - "contracts": parse_contracts.delay(proxies=proxies).id, - "registry_contracts": parse_registry_contracts.delay( - proxies=proxies - ).id, - "unfair_suppliers": parse_unfair_suppliers.delay(proxies=proxies).id, "fas_goz": parse_fas_goz_evasion.delay(proxies=proxies).id, "fns_financial": sync_fns_financial_reports.delay(proxies=proxies).id, - "arbitration": parse_arbitration_cases.delay(proxies=proxies).id, - "fedresurs_bankruptcy": parse_fedresurs_bankruptcy.delay( - proxies=proxies - ).id, - "registry_inspections": parse_registry_inspections.delay( - proxies=proxies - ).id, "fstec": parse_fstec_registers.delay(proxies=proxies).id, - "trudvsem": parse_trudvsem_vacancies.delay(proxies=proxies).id, } results = { @@ -2992,23 +3221,40 @@ def parse_procurements_44fz( *, file_url: str | None = None, file_path: str | None = None, + limit: int | None = None, proxies: list[str] | None = None, + organization_ids: list[str] | None = None, requested_by_id: int | None = None, ) -> dict: - """Парсинг официальной выдачи ЕИС 44-ФЗ в GenericParserRecord.""" + """Addressed OПК lookup by default; explicit files remain a manual tool.""" proxies = _resolve_proxies(proxies) + if file_url or file_path: + + def fetch_records(): + return _fetch_structured_records( + source_key="procurements_44fz", + file_url=file_url, + file_path=file_path, + proxies=proxies, + ) + else: + + def fetch_records(): + return _fetch_eis_registry_procurement_records( + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + law="44", + limit=limit, + proxies=proxies, + organization_ids=organization_ids, + ) + return _run_generic_parser( self, source_key="procurements_44fz", source=ParserLoadLog.Source.PROCUREMENTS_44FZ, task_name="apps.parsers.tasks.parse_procurements_44fz", requested_by_id=requested_by_id, - fetch_records=lambda: _fetch_structured_records( - source_key="procurements_44fz", - file_url=file_url, - file_path=file_path, - proxies=proxies, - ), + fetch_records=fetch_records, ) @@ -3018,23 +3264,40 @@ def parse_procurements_223fz( *, file_url: str | None = None, file_path: str | None = None, + limit: int | None = None, proxies: list[str] | None = None, + organization_ids: list[str] | None = None, requested_by_id: int | None = None, ) -> dict: - """Парсинг официальной выдачи ЕИС 223-ФЗ в GenericParserRecord.""" + """Addressed OПК lookup by default; explicit files remain a manual tool.""" proxies = _resolve_proxies(proxies) + if file_url or file_path: + + def fetch_records(): + return _fetch_structured_records( + source_key="procurements_223fz", + file_url=file_url, + file_path=file_path, + proxies=proxies, + ) + else: + + def fetch_records(): + return _fetch_eis_registry_procurement_records( + source=ParserLoadLog.Source.PROCUREMENTS_223FZ, + law="223", + limit=limit, + proxies=proxies, + organization_ids=organization_ids, + ) + return _run_generic_parser( self, source_key="procurements_223fz", source=ParserLoadLog.Source.PROCUREMENTS_223FZ, task_name="apps.parsers.tasks.parse_procurements_223fz", requested_by_id=requested_by_id, - fetch_records=lambda: _fetch_structured_records( - source_key="procurements_223fz", - file_url=file_url, - file_path=file_path, - proxies=proxies, - ), + fetch_records=fetch_records, ) @@ -3093,23 +3356,38 @@ def parse_unfair_suppliers( *, file_url: str | None = None, file_path: str | None = None, + limit: int | None = None, proxies: list[str] | None = None, + organization_ids: list[str] | None = None, requested_by_id: int | None = None, ) -> dict: - """Парсинг реестра недобросовестных поставщиков.""" + """Checko RNP lookup by default; explicit files remain a manual tool.""" proxies = _resolve_proxies(proxies) + if file_url or file_path: + + def fetch_records(): + return _fetch_structured_records( + source_key="unfair_suppliers", + file_url=file_url, + file_path=file_path, + proxies=proxies, + ) + else: + + def fetch_records(): + return _fetch_checko_unfair_supplier_records( + limit=limit, + proxies=proxies, + organization_ids=organization_ids, + ) + return _run_generic_parser( self, source_key="unfair_suppliers", source=ParserLoadLog.Source.UNFAIR_SUPPLIERS, task_name="apps.parsers.tasks.parse_unfair_suppliers", requested_by_id=requested_by_id, - fetch_records=lambda: _fetch_structured_records( - source_key="unfair_suppliers", - file_url=file_url, - file_path=file_path, - proxies=proxies, - ), + fetch_records=fetch_records, ) @@ -3241,11 +3519,13 @@ def parse_registry_enrichment_sources( """ Запустить daily-контур обогащения активных организаций из реестров. - Внутри остаются независимые задачи: одни забирают полный официальный реестр, - другие делают lookup по ИНН/ОГРН активных организаций. + Each addressed source advances through the next monthly registry batch. """ proxies = _resolve_proxies(proxies) + resolved_limit = _resolve_registry_enrichment_limit(limit) tasks_to_run = { + "procurements_44fz": parse_procurements_44fz, + "procurements_223fz": parse_procurements_223fz, "contracts": parse_registry_contracts, "unfair_suppliers": parse_unfair_suppliers, "arbitration": parse_arbitration_cases, @@ -3259,8 +3539,15 @@ def parse_registry_enrichment_sources( "proxies": proxies, "requested_by_id": requested_by_id, } - if key in {"contracts", "arbitration", "inspections"}: - kwargs["limit"] = limit + if key in { + "procurements_44fz", + "procurements_223fz", + "contracts", + "unfair_suppliers", + "arbitration", + "inspections", + }: + kwargs["limit"] = resolved_limit result = task.delay(**kwargs) results[key] = result.id return results diff --git a/tests/apps/parsers/test_checko_parsers.py b/tests/apps/parsers/test_checko_parsers.py index 3e215c3..bf01f94 100644 --- a/tests/apps/parsers/test_checko_parsers.py +++ b/tests/apps/parsers/test_checko_parsers.py @@ -37,6 +37,58 @@ class CheckoClientParsingTest(SimpleTestCase): data["\u0417\u0430\u043f\u0438\u0441\u0438"][0]["\u041e\u0413\u0420\u041d"], ) + def test_parse_company_maps_russian_unfair_supplier_records(self): + data = _map_ru_keys( + { + "ОГРН": "1027700000001", + "ИНН": "7701000001", + "НаимСокр": "ООО ОПК", + "НедобПост": True, + "НедобПостЗап": [ + { + "РеестрНомер": "RNP-1", + "ДатаПуб": "2026-07-01", + "ДатаУтв": "2026-06-30", + "ЗаказНаимСокр": "Заказчик", + "ЗаказНаимПолн": "Заказчик полный", + "ЗаказИНН": "7702000002", + "ЗаказКПП": "770201001", + "ЗакупНомер": "PURCHASE-1", + "ЗакупОпис": "Поставка оборудования", + "ЦенаКонтр": 1500000, + }, + { + "РеестрНомер": "RNP-2", + "ЗакупНомер": "PURCHASE-2", + }, + ], + } + ) + + company = self.client._parse_company_data(data) + + self.assertEqual(len(company.unfair_supplier), 2) + record = company.unfair_supplier[0] + self.assertEqual(record.registry_number, "RNP-1") + self.assertEqual(record.customer_inn, "7702000002") + self.assertEqual(record.purchase_description, "Поставка оборудования") + self.assertEqual(record.contract_price, 1500000) + self.assertEqual(company.unfair_supplier[1].registry_number, "RNP-2") + + def test_parse_company_handles_empty_unfair_supplier_records(self): + company = self.client._parse_company_data( + _map_ru_keys( + { + "ОГРН": "1027700000001", + "ИНН": "7701000001", + "НедобПост": False, + "НедобПостЗап": [], + } + ) + ) + + self.assertEqual(company.unfair_supplier, ()) + def test_parse_okved_info_with_additional(self): info = self.client._parse_okved_info( {"code": "62.01", "name": "Development", "version": "2001"}, diff --git a/tests/apps/parsers/test_eis_registry_client.py b/tests/apps/parsers/test_eis_registry_client.py new file mode 100644 index 0000000..a33d4d8 --- /dev/null +++ b/tests/apps/parsers/test_eis_registry_client.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from datetime import date +from decimal import Decimal +from urllib.parse import parse_qs, urlsplit + +from apps.parsers.clients.base import HTTPClientError +from apps.parsers.clients.common.eis_registry import EisRegistryProcurementClient +from apps.parsers.clients.common.schemas import GenericParserItem +from django.test import SimpleTestCase + + +def _record(*, external_id: str, inn: str, amount: str = "1") -> GenericParserItem: + return GenericParserItem( + source="procurements_44fz", + external_id=external_id, + inn=inn, + organisation_name="Наименование из ЕИС", + title="Закупка", + amount=Decimal(amount), + payload={"raw": external_id}, + ) + + +class _FakeHttpClient: + def __init__(self, pages: dict[int, bytes], *, error_page: int | None = None): + self.pages = pages + self.error_page = error_page + self.urls: list[str] = [] + + def download_file(self, url: str, **_kwargs) -> bytes: + self.urls.append(url) + page = int(parse_qs(urlsplit(url).query)["pageNumber"][0]) + if page == self.error_page: + raise HTTPClientError("page failed") + return self.pages[page] + + +class _FakeStructuredClient: + def __init__(self, pages: dict[int, bytes], records: dict[bytes, list]): + self.http_client = _FakeHttpClient(pages) + self.records = records + + def fetch_records(self, *, content: bytes, file_name: str): + assert file_name == "results.html" + return self.records[content] + + +class EisRegistryProcurementClientTest(SimpleTestCase): + def test_fetches_all_pages_filters_inn_and_deduplicates_external_id(self): + page_1 = b'' + page_2 = b'' + structured = _FakeStructuredClient( + {1: page_1, 2: page_2}, + { + page_1: [ + _record(external_id="same", inn="7701000001", amount="1"), + _record(external_id="foreign", inn="7701000099"), + ], + page_2: [ + _record(external_id="same", inn="7701000001", amount="2"), + _record(external_id="other", inn="7701000001"), + ], + }, + ) + client = EisRegistryProcurementClient( + source="procurements_44fz", + law="44", + _structured_client=structured, + ) + + records = client.fetch_for_customer( + inn="7701000001", + organization_name="ОПК Заказчик", + organization_id="org-1", + today=date(2026, 7, 19), + ) + + self.assertEqual([row.external_id for row in records], ["same", "other"]) + self.assertEqual(records[0].amount, Decimal("2")) + self.assertTrue(all(row.inn == "7701000001" for row in records)) + self.assertEqual(records[0].payload["organization_id"], "org-1") + query = parse_qs(urlsplit(structured.http_client.urls[0]).query) + self.assertEqual(query["fz44"], ["on"]) + self.assertEqual(query["strictEqual"], ["true"]) + self.assertEqual(query["publishDateFrom"], ["19.07.2025"]) + self.assertEqual(query["publishDateTo"], ["19.07.2026"]) + + def test_builds_223fz_query_and_accepts_empty_page(self): + empty_page = b"" + structured = _FakeStructuredClient({1: empty_page}, {empty_page: []}) + client = EisRegistryProcurementClient( + source="procurements_223fz", + law="223", + _structured_client=structured, + ) + + records = client.fetch_for_customer( + inn="7701000001", + organization_name="ОПК Заказчик", + organization_id="org-1", + ) + + self.assertEqual(records, []) + query = parse_qs(urlsplit(structured.http_client.urls[0]).query) + self.assertEqual(query["fz223"], ["on"]) + self.assertNotIn("fz44", query) + + def test_page_error_is_not_silenced(self): + page_1 = b'' + structured = _FakeStructuredClient({1: page_1}, {page_1: []}) + structured.http_client.error_page = 2 + client = EisRegistryProcurementClient( + source="procurements_44fz", + law="44", + _structured_client=structured, + ) + + with self.assertRaises(HTTPClientError): + client.fetch_for_customer( + inn="7701000001", + organization_name="ОПК Заказчик", + organization_id="org-1", + ) diff --git a/tests/apps/parsers/test_registry_procurement_tasks.py b/tests/apps/parsers/test_registry_procurement_tasks.py new file mode 100644 index 0000000..db6ab7a --- /dev/null +++ b/tests/apps/parsers/test_registry_procurement_tasks.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from contextlib import ExitStack +from types import SimpleNamespace +from unittest.mock import patch + +from apps.parsers import tasks as parser_tasks +from apps.parsers.clients.common.schemas import GenericParserItem +from apps.parsers.models import CheckoCollectionAttempt, ParserLoadLog +from django.test import TestCase, override_settings + +from tests.apps.parsers.organization_helpers import create_directory_organization + + +def _organization(index: int, *, membership: bool = True): + return create_directory_organization( + name=f"Организация {index}", + inn=f"7701000{index:03d}", + ogrn=f"1027700000{index:03d}", + opk_registry_membership=membership, + ) + + +class RegistryProcurementTasksTest(TestCase): + def test_daily_orchestrator_queues_addressed_sources_with_same_batch_limit(self): + task_names = ( + "parse_procurements_44fz", + "parse_procurements_223fz", + "parse_registry_contracts", + "parse_unfair_suppliers", + "parse_arbitration_cases", + "parse_fedresurs_bankruptcy", + "parse_registry_inspections", + "parse_trudvsem_vacancies", + ) + with ExitStack() as stack: + mocks = [ + stack.enter_context( + patch.object( + getattr(parser_tasks, task_name), + "delay", + return_value=SimpleNamespace(id=f"{task_name}-id"), + ) + ) + for task_name in task_names + ] + result = parser_tasks.parse_registry_enrichment_sources( + limit=20, + proxies=[], + ) + + self.assertEqual( + set(result), + { + "procurements_44fz", + "procurements_223fz", + "contracts", + "unfair_suppliers", + "arbitration", + "bankruptcy", + "inspections", + "vacancies", + }, + ) + for task_mock in mocks[:5]: + task_mock.assert_called_once_with( + proxies=[], + requested_by_id=None, + limit=20, + ) + mocks[5].assert_called_once_with(proxies=[], requested_by_id=None) + mocks[6].assert_called_once_with(proxies=[], requested_by_id=None, limit=20) + mocks[7].assert_called_once_with(proxies=[], requested_by_id=None) + + def test_eis_monthly_claims_advance_to_the_next_batch(self): + organizations = [_organization(index) for index in range(1, 4)] + outside_registry = _organization(9, membership=False) + requested_inns: list[str] = [] + + class _Client: + def __init__(self, **_kwargs): + return + + def fetch_for_customer(self, **kwargs): + requested_inns.append(kwargs["inn"]) + return [ + GenericParserItem( + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + external_id=f"notice-{kwargs['inn']}", + inn=kwargs["inn"], + organisation_name=kwargs["organization_name"], + ) + ] + + with patch.object(parser_tasks, "EisRegistryProcurementClient", _Client): + first = parser_tasks._fetch_eis_registry_procurement_records( + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + law="44", + limit=2, + proxies=[], + ) + second = parser_tasks._fetch_eis_registry_procurement_records( + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + law="44", + limit=2, + proxies=[], + ) + + self.assertEqual(len(first), 2) + self.assertEqual(len(second), 1) + self.assertEqual(requested_inns, [item.inn for item in organizations]) + self.assertNotIn(outside_registry.inn, requested_inns) + self.assertEqual( + CheckoCollectionAttempt.objects.filter( + source=CheckoCollectionAttempt.Source.PROCUREMENTS_44FZ, + status=CheckoCollectionAttempt.Status.SUCCESS, + ).count(), + 3, + ) + + def test_eis_request_error_creates_failed_monthly_claim(self): + organization = _organization(1) + + class _Client: + def __init__(self, **_kwargs): + return + + def fetch_for_customer(self, **_kwargs): + raise parser_tasks.HTTPClientError("EIS unavailable") + + with ( + patch.object(parser_tasks, "EisRegistryProcurementClient", _Client), + self.assertRaises(parser_tasks.ParserSourceSkipped), + ): + parser_tasks._fetch_eis_registry_procurement_records( + source=ParserLoadLog.Source.PROCUREMENTS_223FZ, + law="223", + limit=1, + proxies=[], + ) + + attempt = CheckoCollectionAttempt.objects.get( + organization=organization, + source=CheckoCollectionAttempt.Source.PROCUREMENTS_223FZ, + ) + self.assertEqual(attempt.status, CheckoCollectionAttempt.Status.FAILED) + + def test_explicit_organization_ids_do_not_fall_through_to_next_batch(self): + selected = _organization(1) + _organization(2) + calls: list[str] = [] + + class _Client: + def __init__(self, **_kwargs): + return + + def fetch_for_customer(self, **kwargs): + calls.append(kwargs["inn"]) + return [] + + kwargs = { + "source": ParserLoadLog.Source.PROCUREMENTS_44FZ, + "law": "44", + "limit": 20, + "proxies": [], + "organization_ids": [str(selected.uid)], + } + with patch.object(parser_tasks, "EisRegistryProcurementClient", _Client): + self.assertEqual( + parser_tasks._fetch_eis_registry_procurement_records(**kwargs), + [], + ) + with self.assertRaises(parser_tasks.ParserSourceSkipped): + parser_tasks._fetch_eis_registry_procurement_records(**kwargs) + + self.assertEqual(calls, [selected.inn]) + + @override_settings(CHECKO_API_KEY="test-key") + def test_checko_rnp_is_supplier_bound_and_not_requested_twice(self): + organization = _organization(1) + calls: list[str] = [] + + class _Client: + def __init__(self, **_kwargs): + return + + def get_company(self, request): + calls.append(request.inn) + return SimpleNamespace( + data=SimpleNamespace( + inn=organization.inn, + ogrn=organization.ogrn, + unfair_supplier=( + SimpleNamespace( + registry_number="RNP-1", + publish_date="2026-07-01", + approval_date="2026-06-30", + customer_short_name="Заказчик", + customer_full_name="Заказчик полный", + customer_inn="7702000002", + customer_kpp="770201001", + purchase_number="PURCHASE-1", + purchase_description="Поставка оборудования", + contract_price=1500000, + ), + ), + ) + ) + + with patch.object(parser_tasks, "CheckoClient", _Client): + records = parser_tasks._fetch_checko_unfair_supplier_records( + limit=1, + proxies=[], + ) + with self.assertRaises(parser_tasks.ParserSourceSkipped): + parser_tasks._fetch_checko_unfair_supplier_records( + limit=1, + proxies=[], + ) + + self.assertEqual(calls, [organization.inn]) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].inn, organization.inn) + self.assertEqual(records[0].payload["supplier"]["inn"], organization.inn) + self.assertEqual(records[0].payload["customer"]["inn"], "7702000002") + self.assertEqual(records[0].amount, 1500000) + + @override_settings(CHECKO_API_KEY="test-key") + def test_checko_rnp_api_error_creates_failed_monthly_claim(self): + organization = _organization(1) + + class _Client: + def __init__(self, **_kwargs): + return + + def get_company(self, _request): + raise parser_tasks.CheckoError("Checko unavailable") + + with ( + patch.object(parser_tasks, "CheckoClient", _Client), + self.assertRaises(parser_tasks.ParserSourceSkipped), + ): + parser_tasks._fetch_checko_unfair_supplier_records( + limit=1, + proxies=[], + ) + + attempt = CheckoCollectionAttempt.objects.get( + organization=organization, + source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS, + ) + self.assertEqual(attempt.status, CheckoCollectionAttempt.Status.FAILED) diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index a6049e8..68fb3d5 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -240,10 +240,12 @@ class SourceCardServiceUnitTest(SimpleTestCase): "apps.parsers.source_cards.SourceCardService._enqueue_task", return_value={ "task_id": "task-9", - "task_name": "apps.parsers.tasks.sync_procurements", + "task_name": "apps.parsers.tasks.parse_registry_enrichment_sources", }, ) - def test_refresh_card_for_procurements_uses_default_law_type(self, enqueue_mock): + def test_refresh_card_for_procurements_uses_addressed_registry_task( + self, enqueue_mock + ): result = SourceCardService.refresh_card( slug="public-procurements", requested_by_id=10, @@ -254,12 +256,7 @@ class SourceCardServiceUnitTest(SimpleTestCase): self.assertEqual(result["tasks"][0]["task_id"], "task-9") self.assertEqual( enqueue_mock.call_args.kwargs["kwargs"], - { - "requested_by_id": 10, - "region_code": "77", - "law_type": "44", - "current_year": 2026, - }, + {"requested_by_id": 10}, ) @patch( @@ -540,6 +537,7 @@ class SourceCardServiceDatabaseTest(TestCase): self.assertEqual(card["records_count"], 3) self.assertEqual(card["organizations_count"], 2) source_items = {item["code"]: item for item in card["source_items"]} + self.assertEqual(source_items["procurements"]["records_count"], 2) self.assertEqual(source_items["procurements_44fz"]["organizations_count"], 1) self.assertEqual(source_items["procurements_223fz"]["organizations_count"], 1) self.assertEqual(source_items["contracts"]["organizations_count"], 1) diff --git a/tests/apps/parsers/test_source_cards_views.py b/tests/apps/parsers/test_source_cards_views.py index 7b65485..0d2b98a 100644 --- a/tests/apps/parsers/test_source_cards_views.py +++ b/tests/apps/parsers/test_source_cards_views.py @@ -3,6 +3,8 @@ from __future__ import annotations import hashlib +from types import SimpleNamespace +from unittest.mock import patch from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import ParserLoadLog @@ -259,19 +261,24 @@ class SourceCardsApiTestCase(APITestCase): ).exists() ) - def test_refresh_procurements_requires_region_code(self): + @override_settings(CELERY_TASK_ALWAYS_EAGER=False) + def test_refresh_procurements_does_not_require_region_code(self): self.client.force_authenticate(self.admin) - response = self.client.post( - reverse( - "api_v1:sources:source-cards-refresh", - kwargs={"slug": "public-procurements"}, - ), - {}, - format="json", - ) + with patch( + "apps.parsers.tasks.parse_registry_enrichment_sources.apply_async", + return_value=SimpleNamespace(id="task-procurements"), + ): + response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "public-procurements"}, + ), + {}, + format="json", + ) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertFalse(response.data["success"]) + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["task_id"], "task-procurements") def test_refresh_forbidden_for_regular_user(self): response = self.client.post( diff --git a/tests/apps/parsers/test_sources_api_e2e.py b/tests/apps/parsers/test_sources_api_e2e.py index c873d30..cf74476 100644 --- a/tests/apps/parsers/test_sources_api_e2e.py +++ b/tests/apps/parsers/test_sources_api_e2e.py @@ -6,7 +6,6 @@ from unittest.mock import patch from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import FinancialReport, FinancialReportLine, ParserLoadLog from django.urls import reverse -from organizations.models import Organization from organizations.source_backfill import OrganizationSourceBackfillService from rest_framework import status from rest_framework.test import APITestCase @@ -206,7 +205,7 @@ class SourcesApiE2ETest(APITestCase): ) with patch( - "apps.parsers.tasks.sync_procurements.apply_async", + "apps.parsers.tasks.parse_registry_enrichment_sources.apply_async", return_value=SimpleNamespace(id="task-procurements"), ): procurements_response = self.client.post( @@ -241,7 +240,7 @@ class SourcesApiE2ETest(APITestCase): self.assertTrue( BackgroundJob.objects.filter( task_id="task-procurements", - task_name="apps.parsers.tasks.sync_procurements", + task_name="apps.parsers.tasks.parse_registry_enrichment_sources", user_id=self.admin.id, ).exists() ) diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 38b19d0..dcccb57 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -1855,18 +1855,10 @@ class MinpromtorgTasksTestCase(TestCase): "parse_industrial_products", "parse_manufactures", "sync_inspections", - "parse_procurements_44fz", - "parse_procurements_223fz", - "parse_contracts", - "parse_registry_contracts", - "parse_unfair_suppliers", + "parse_registry_enrichment_sources", "parse_fas_goz_evasion", "sync_fns_financial_reports", - "parse_arbitration_cases", - "parse_fedresurs_bankruptcy", - "parse_registry_inspections", "parse_fstec_registers", - "parse_trudvsem_vacancies", ) with ExitStack() as stack: delays = { @@ -1887,8 +1879,10 @@ class MinpromtorgTasksTestCase(TestCase): "sync_fns_financial_reports-id", ) delays["sync_fns_financial_reports"].assert_called_once_with(proxies=[]) - delays["parse_registry_contracts"].assert_called_once_with(proxies=[]) - delays["parse_registry_inspections"].assert_called_once_with(proxies=[]) + delays["parse_registry_enrichment_sources"].assert_called_once_with( + limit=250, + proxies=[], + ) def test_parse_all_minpromtorg_without_adapter(self): with TestHTTPServer() as server: -- 2.39.5