diff --git a/.env.prod.example b/.env.prod.example index 20622cb..0d1dedf 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -27,6 +27,9 @@ CELERY_WORKER_CONCURRENCY=2 CHECKO_API_KEY= ZAKUPKI_TOKEN= SUPERJOB_APP_ID= +HH_USER_AGENT= +# Must exceed the 200 MiB parser file limit to include multipart framing. +DATA_UPLOAD_MAX_MEMORY_SIZE=220200960 # Optional: comma-separated HTTP(S) proxies for parser tasks # Example: PARSER_PROXIES=http://user:pass@proxy1:8080,http://user:pass@proxy2:8080 PARSER_PROXIES= diff --git a/docker/Dockerfile b/docker/Dockerfile index 1837414..48db094 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -98,6 +98,7 @@ ENV PATH="/app/.venv/bin:${PATH}" \ CHECKO_API_KEY= \ ZAKUPKI_TOKEN= \ SUPERJOB_APP_ID= \ + HH_USER_AGENT= \ COLLECTSTATIC_ON_MIGRATE=0 \ BACKUP_ENCRYPTION_KEY= \ BACKUP_KEY_ID=default \ diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index 4d28f79..b02cdd4 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -34,7 +34,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.db.models import Q from django.utils import timezone -from organizations.models import Organization +from organizations.models import Organization, OrganizationSourceRecord class StateCorpExchangeError(ValueError): @@ -395,6 +395,42 @@ class StateCorpExchangeService: "ogrn": cls._digits(record.ogrn), } ) + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.INDUSTRIAL], + ): + payload = cls._record_payload(record) + certificate_number = cls._payload_lookup( + payload, ["certificate_number", "registry_number"] + ) + if not certificate_number: + continue + issue_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["issue_date_normalized", "issue_date"]) + or record.record_date, + ) + expiry_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["expiry_date_normalized", "expiry_date"]), + ) + items.append( + { + "organization_inn": cls._digits(record.inn), + "certificate_number": certificate_number, + "issue_date": issue_date.isoformat() if issue_date else None, + "expiry_date": expiry_date.isoformat() if expiry_date else None, + "certificate_file_url": cls._payload_lookup( + payload, ["certificate_file_url"] + ) + or record.url, + "organisation_name": cls._payload_lookup( + payload, ["organisation_name"] + ) + or record.registry_organization.name, + "ogrn": cls._digits(record.ogrn), + } + ) return items @classmethod @@ -405,7 +441,7 @@ class StateCorpExchangeService: queryset = ManufacturerRecord.objects.filter(inn__in=allowed_inns).order_by( "id" ) - return [ + items = [ { "organization_inn": cls._digits(record.inn), "full_legal_name": record.full_legal_name, @@ -415,6 +451,26 @@ class StateCorpExchangeService: } for record in queryset ] + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.MANUFACTURES], + ): + payload = cls._record_payload(record) + items.append( + { + "organization_inn": cls._digits(record.inn), + "full_legal_name": cls._payload_lookup( + payload, ["full_legal_name", "organisation_name"] + ) + or record.registry_organization.full_name + or record.registry_organization.name, + "inn": cls._digits(record.inn), + "ogrn": cls._digits(record.ogrn), + "address": cls._payload_lookup(payload, ["address"]) + or record.registry_organization.legal_address, + } + ) + return items @classmethod def _serialize_industrial_products( @@ -424,7 +480,7 @@ class StateCorpExchangeService: queryset = IndustrialProductRecord.objects.filter( inn__in=allowed_inns ).order_by("id") - return [ + items = [ { "organization_inn": cls._digits(record.inn), "product_name": record.product_name, @@ -437,6 +493,27 @@ class StateCorpExchangeService: } for record in queryset ] + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.INDUSTRIAL_PRODUCTS], + ): + payload = cls._record_payload(record) + product_name = cls._payload_lookup(payload, ["product_name"]) + items.append( + { + "organization_inn": cls._digits(record.inn), + "product_name": product_name or record.title, + "product_class": cls._payload_lookup( + payload, ["product_model", "regulatory_document"] + ) + or "Промышленная продукция", + "okpd2_code": cls._payload_lookup(payload, ["okpd2_code"]), + "tnved_code": cls._payload_lookup(payload, ["tnved_code"]), + "registry_number": cls._payload_lookup(payload, ["registry_number"]) + or record.external_id, + } + ) + return items @classmethod def _serialize_prosecutor_checks( @@ -463,6 +540,36 @@ class StateCorpExchangeService: "status": record.status, } ) + for record in cls._canonical_records( + allowed_inns, + sources=[ParserLoadLog.Source.INSPECTIONS], + ): + payload = cls._record_payload(record) + start_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["start_date_normalized", "start_date"]) + or record.record_date, + ) + if start_date is None: + continue + items.append( + { + "organization_inn": cls._digits(record.inn), + "registration_number": cls._payload_lookup( + payload, ["registration_number"] + ) + or record.external_id, + "law_type": cls._payload_lookup(payload, ["legal_basis"]), + "control_authority": cls._payload_lookup( + payload, ["control_authority"] + ), + "prosecutor_office": cls._payload_lookup( + payload, ["prosecutor_office"] + ), + "start_date": start_date.isoformat(), + "status": record.status or cls._payload_lookup(payload, ["status"]), + } + ) return items @classmethod @@ -504,6 +611,7 @@ class StateCorpExchangeService: for record in cls._generic_records( allowed_inns, sources=[ + ParserLoadLog.Source.PROCUREMENTS, ParserLoadLog.Source.PROCUREMENTS_44FZ, ParserLoadLog.Source.PROCUREMENTS_223FZ, ParserLoadLog.Source.CONTRACTS, @@ -585,7 +693,7 @@ class StateCorpExchangeService: .prefetch_related("lines") .order_by("id") ) - return [ + items = [ { "organization_inn": allowed_ogrn_to_inn[report.ogrn], "external_id": report.external_id, @@ -611,6 +719,33 @@ class StateCorpExchangeService: for report in queryset if report.external_id ] + for record in cls._canonical_records( + set(allowed_ogrn_to_inn.values()), + sources=[ParserLoadLog.Source.FNS_REPORTS], + ): + if not record.external_id: + continue + payload = cls._record_payload(record) + items.append( + { + "organization_inn": cls._digits(record.inn), + "external_id": record.external_id, + "ogrn": cls._digits(record.ogrn), + "file_name": cls._payload_lookup(payload, ["file_name"]), + "file_hash": cls._payload_lookup(payload, ["file_hash"]), + "load_batch": record.load_batch, + "status": record.status, + "source": cls._payload_lookup(payload, ["source"]) or record.source, + "error_message": cls._payload_lookup(payload, ["error_message"]), + "lines": [ + cls._serialize_financial_report_line(line) + for line in record.financial_lines.all().order_by( + "year", "form_code", "line_code" + ) + ], + } + ) + return items @staticmethod def _serialize_financial_report_line( @@ -631,11 +766,10 @@ class StateCorpExchangeService: allowed_inns: set[str], ) -> list[dict[str, str]]: items: list[dict[str, str]] = [] - queryset = GenericParserRecord.objects.filter( - source=ParserLoadLog.Source.ARBITRATION, - inn__in=allowed_inns, - ).order_by("id") - for record in queryset: + for record in cls._generic_records( + allowed_inns, + sources=[ParserLoadLog.Source.ARBITRATION], + ): payload = record.payload if isinstance(record.payload, dict) else {} target = payload.get("target") if not isinstance(target, dict): @@ -843,7 +977,7 @@ class StateCorpExchangeService: return items @staticmethod - def _record_payload(record: GenericParserRecord) -> dict[str, Any]: + def _record_payload(record: Any) -> dict[str, Any]: return record.payload if isinstance(record.payload, dict) else {} @classmethod @@ -854,11 +988,35 @@ class StateCorpExchangeService: sources: list[str], ): if not allowed_inns: - return GenericParserRecord.objects.none() - return GenericParserRecord.objects.filter( - source__in=sources, - inn__in=allowed_inns, - ).order_by("id") + return [] + legacy_records = list( + GenericParserRecord.objects.filter( + source__in=sources, + inn__in=allowed_inns, + ).order_by("id") + ) + return legacy_records + list( + cls._canonical_records(allowed_inns, sources=sources) + ) + + @staticmethod + def _canonical_records( + allowed_inns: set[str], + *, + sources: list[str], + ): + """Return canonical-only records, excluding legacy-backed mirrors.""" + if not allowed_inns: + return OrganizationSourceRecord.objects.none() + return ( + OrganizationSourceRecord.objects.filter( + source__in=sources, + extension__organization__inn__in=allowed_inns, + legacy_model="", + ) + .select_related("extension__organization") + .order_by("created_at", "uid") + ) @staticmethod def _payload_lookup(payload: dict[str, Any], candidates: list[str]) -> str: diff --git a/src/apps/parsers/clients/base.py b/src/apps/parsers/clients/base.py index af399c5..d308ac1 100644 --- a/src/apps/parsers/clients/base.py +++ b/src/apps/parsers/clients/base.py @@ -39,7 +39,15 @@ class ConnectionError(HTTPClientError): class HTTPError(HTTPClientError): """HTTP ошибка (4xx, 5xx).""" - pass + def __init__( + self, + message: str, + status_code: int | None = None, + url: str | None = None, + response_data: Any | None = None, + ): + self.response_data = response_data + super().__init__(message, status_code=status_code, url=url) @dataclass @@ -204,10 +212,15 @@ class BaseHTTPClient: if not response.ok: logger.error("HTTP error %d: %s", response.status_code, url) + response_data = None + if "json" in response.headers.get("Content-Type", "").lower(): + with suppress(ValueError): + response_data = response.json() raise HTTPError( f"HTTP {response.status_code} for {url}", status_code=response.status_code, url=url, + response_data=response_data, ) logger.debug("Response %d from %s", response.status_code, url) diff --git a/src/apps/parsers/clients/checko/client.py b/src/apps/parsers/clients/checko/client.py index 05a0397..fb8710e 100644 --- a/src/apps/parsers/clients/checko/client.py +++ b/src/apps/parsers/clients/checko/client.py @@ -10,7 +10,7 @@ from collections.abc import Iterator from dataclasses import dataclass, field from typing import Any -from apps.parsers.clients.base import BaseHTTPClient +from apps.parsers.clients.base import BaseHTTPClient, HTTPError from apps.parsers.clients.checko.exceptions import ( CheckoAPIError, CheckoConnectionError, @@ -317,33 +317,26 @@ class CheckoClient: try: data = self._http_client.get_json(endpoint, params=params) + except HTTPError as e: + if isinstance(e.response_data, dict): + try: + self._raise_api_error(e.response_data, status_code=e.status_code) + except CheckoRateLimitError: + raise + except CheckoAPIError: + # Preserve the existing classification for non-quota HTTP + # errors while still recovering Checko quota metadata. + pass + logger.error("Checko HTTP request failed with status=%s", e.status_code) + raise CheckoConnectionError( + "Checko API request failed", + url=e.url, + ) from e except Exception as e: logger.error("Connection error: %s", e) raise CheckoConnectionError(f"Failed to connect to Checko API: {e}") from e - # Проверяем статус ответа - meta = data.get("meta", {}) - status = meta.get("status", "error") - - if status != "ok": - message = meta.get("message", "Unknown error") - balance = meta.get("balance") - - # Определяем тип ошибки - if "лимит" in message.lower() or "limit" in message.lower(): - raise CheckoRateLimitError( - message=message, - balance=balance, - ) - if "не найден" in message.lower() or "not found" in message.lower(): - raise CheckoNotFoundError( - message=message, - balance=balance, - ) - raise CheckoAPIError( - message=message, - balance=balance, - ) + self._raise_api_error(data) # Map Russian field names to English if "data" in data: @@ -351,6 +344,44 @@ class CheckoClient: return data + @staticmethod + def _raise_api_error( + data: dict[str, Any], + *, + status_code: int | None = None, + ) -> None: + """Классифицировать ошибку Checko из JSON независимо от HTTP статуса.""" + meta = data.get("meta", {}) + status = meta.get("status", "error") + + if status == "ok": + return + + message = str(meta.get("message", "Unknown error")) + balance = meta.get("balance") + request_count = meta.get("today_request_count") + error_kwargs = { + "message": message, + "status_code": status_code, + "balance": balance, + "request_count": request_count, + } + normalized_message = message.lower() + + if any( + marker in normalized_message + for marker in ("лимит", "limit", "квот", "quota") + ): + raise CheckoRateLimitError(**error_kwargs) + if "не найден" in normalized_message or "not found" in normalized_message: + raise CheckoNotFoundError(**error_kwargs) + raise CheckoAPIError( + message=message, + status_code=status_code, + balance=balance, + request_count=request_count, + ) + # ========================================================================= # Парсинг общих моделей # ========================================================================= diff --git a/src/apps/parsers/clients/common/eis_russian_trusted_ca.pem b/src/apps/parsers/clients/common/eis_russian_trusted_ca.pem new file mode 100644 index 0000000..bf2300c --- /dev/null +++ b/src/apps/parsers/clients/common/eis_russian_trusted_ca.pem @@ -0,0 +1,79 @@ +# Project-scoped CA bundle for HTTPS endpoints at zakupki.gov.ru. +# Sources (official AIA endpoints published by the certificate chain): +# - http://nuc-cdp.digital.gov.ru/cdp/rootca_ssl_rsa2022.crt +# - http://nuc-cdp.digital.gov.ru/cdp/subca_ssl_rsa2024.crt +# SHA-256 fingerprints checked on 2026-07-14: +# - root: D2:6D:2D:02:31:B7:C3:9F:92:CC:73:85:12:BA:54:10:35:19:E4:40:5D:68:B5:BD:70:3E:97:88:CA:8E:CF:31 +# - sub: 21:55:78:50:36:C9:00:DB:B5:F1:BB:2A:15:69:C8:0C:55:59:5B:D6:BF:94:86:7A:29:BB:DD:BC:7D:88:A3:F2 +-----BEGIN CERTIFICATE----- +MIIFwjCCA6qgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx +PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu +ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg +Q0EwHhcNMjIwMzAxMjEwNDE1WhcNMzIwMjI3MjEwNDE1WjBwMQswCQYDVQQGEwJS +VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg +YW5kIENvbW11bmljYXRpb25zMSAwHgYDVQQDDBdSdXNzaWFuIFRydXN0ZWQgUm9v +dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMfFOZ8pUAL3+r2n +qqE0Zp52selXsKGFYoG0GM5bwz1bSFtCt+AZQMhkWQheI3poZAToYJu69pHLKS6Q +XBiwBC1cvzYmUYKMYZC7jE5YhEU2bSL0mX7NaMxMDmH2/NwuOVRj8OImVa5s1F4U +zn4Kv3PFlDBjjSjXKVY9kmjUBsXQrIHeaqmUIsPIlNWUnimXS0I0abExqkbdrXbX +YwCOXhOO2pDUx3ckmJlCMUGacUTnylyQW2VsJIyIGA8V0xzdaeUXg0VZ6ZmNUr5Y +Ber/EAOLPb8NYpsAhJe2mXjMB/J9HNsoFMBFJ0lLOT/+dQvjbdRZoOT8eqJpWnVD +U+QL/qEZnz57N88OWM3rabJkRNdU/Z7x5SFIM9FrqtN8xewsiBWBI0K6XFuOBOTD +4V08o4TzJ8+Ccq5XlCUW2L48pZNCYuBDfBh7FxkB7qDgGDiaftEkZZfApRg2E+M9 +G8wkNKTPLDc4wH0FDTijhgxR3Y4PiS1HL2Zhw7bD3CbslmEGgfnnZojNkJtcLeBH +BLa52/dSwNU4WWLubaYSiAmA9IUMX1/RpfpxOxd4Ykmhz97oFbUaDJFipIggx5sX +ePAlkTdWnv+RWBxlJwMQ25oEHmRguNYf4Zr/Rxr9cS93Y+mdXIZaBEE0KS2iLRqa +OiWBki9IMQU4phqPOBAaG7A+eP8PAgMBAAGjZjBkMB0GA1UdDgQWBBTh0YHlzlpf +BKrS6badZrHF+qwshzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qwshzAS +BgNVHRMBAf8ECDAGAQH/AgEEMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsF +AAOCAgEAALIY1wkilt/urfEVM5vKzr6utOeDWCUczmWX/RX4ljpRdgF+5fAIS4vH +tmXkqpSCOVeWUrJV9QvZn6L227ZwuE15cWi8DCDal3Ue90WgAJJZMfTshN4OI8cq +W9E4EG9wglbEtMnObHlms8F3CHmrw3k6KmUkWGoa+/ENmcVl68u/cMRl1JbW2bM+ +/3A+SAg2c6iPDlehczKx2oa95QW0SkPPWGuNA/CE8CpyANIhu9XFrj3RQ3EqeRcS +AQQod1RNuHpfETLU/A2gMmvn/w/sx7TB3W5BPs6rprOA37tutPq9u6FTZOcG1Oqj +C/B7yTqgI7rbyvox7DEXoX7rIiEqyNNUguTk/u3SZ4VXE2kmxdmSh3TQvybfbnXV +4JbCZVaqiZraqc7oZMnRoWrXRG3ztbnbes/9qhRGI7PqXqeKJBztxRTEVj8ONs1d +WN5szTwaPIvhkhO3CO5ErU2rVdUr89wKpNXbBODFKRtgxUT70YpmJ46VVaqdAhOZ +D9EUUn4YaeLaS8AjSF/h7UkjOibNc4qVDiPP+rkehFWM66PVnP1Msh93tc+taIfC +EYVMxjh8zNbFuoc7fzvvrFILLe7ifvEIUqSVIC/AzplM/Jxw7buXFeGP1qVCBEHq +391d/9RAfaZ12zkwFsl+IKwE/OZxW8AHa9i1p4GO0YSNuczzEm4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIG6DCCBNCgAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwcDELMAkGA1UEBhMCUlUx +PzA9BgNVBAoMNlRoZSBNaW5pc3RyeSBvZiBEaWdpdGFsIERldmVsb3BtZW50IGFu +ZCBDb21tdW5pY2F0aW9uczEgMB4GA1UEAwwXUnVzc2lhbiBUcnVzdGVkIFJvb3Qg +Q0EwHhcNMjQwNzE1MTI1MDQxWhcNMjkwNzE5MTI1MDQxWjBvMQswCQYDVQQGEwJS +VTE/MD0GA1UECgw2VGhlIE1pbmlzdHJ5IG9mIERpZ2l0YWwgRGV2ZWxvcG1lbnQg +YW5kIENvbW11bmljYXRpb25zMR8wHQYDVQQDDBZSdXNzaWFuIFRydXN0ZWQgU3Vi +IENBMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1j0rkZECOt1S8o7I +JY+4YKAxuEa5xaHKHXT2EpkuC/0krqMOjUy2oPIRNgR5g8X0Jl6jamxeGLc4Q1tf +ju6or9oSRYThIUhRsFDQNBiBBEXoBgWxTfiKB2eyT97+pz5TBtBiRCPaLGRHYLRb +9Jz2HkJlxbtNPjtDrF5DPHym+mZ1M1z3hIQYAqJwLpsEBnsw/VxWMlxqHoeewd0h +uJMd71KQ5vOKlz7KrIZ6EobNNa6wItuvsfj3kYCK7O78uLHGXXFxdr8Hae9lMUmC +8F7AFwa+bO1LRlTlqW7rE3rLf+jj70N01N8T3o22v14YBaFBWQWncAVYD2JuL3tH +252+kdNOERf1fLbLRigJAbd+hOhWYlNf963TFDgnNPliHNIW72SygVBnI2V3JwO1 +dp1hVKpK/zt8ziGdHW4gmOLTsH50YKdR4jNqUgQv4wASlKn9OpN6zHYc5G8h86fY +BM+zxE5ikGI+I/vIqBuI0eaDU92AWN/YjFLpu8tMu9kLRSCf1vug6FIfDPWVo7iP +ac/SI2v8jnnpaW7ph/Pz3WkzaG7ZZJsfFs+8dploWc6LOoDtbFBhMdGMxu024msC +0PSjZb5ODXPIaO2NsA7fMiAtZcoK6anTUJh4zOP/stA9qsJGNxdrEmiPXSmBZY/N +Y0wkZgZ6JTDhw7038bPvctkblJkCAwEAAaOCAYswggGHMB0GA1UdDgQWBBR3Pdk5 +r0K93FvKduru/c4+YSkwXzAfBgNVHSMEGDAWgBTh0YHlzlpfBKrS6badZrHF+qws +hzAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADCBmAYIKwYBBQUH +AQEEgYswgYgwQAYIKwYBBQUHMAKGNGh0dHA6Ly9udWMtY2RwLnZvc2tob2QucnUv +Y2RwL3Jvb3RjYV9zc2xfcnNhMjAyMi5jcnQwRAYIKwYBBQUHMAKGOGh0dHA6Ly9u +dWMtY2RwLmRpZ2l0YWwuZ292LnJ1L2NkcC9yb290Y2Ffc3NsX3JzYTIwMjIuY3J0 +MIGFBgNVHR8EfjB8MDqgOKA2hjRodHRwOi8vbnVjLWNkcC52b3NraG9kLnJ1L2Nk +cC9yb290Y2Ffc3NsX3JzYTIwMjIuY3JsMD6gPKA6hjhodHRwOi8vbnVjLWNkcC5k +aWdpdGFsLmdvdi5ydS9jZHAvcm9vdGNhX3NzbF9yc2EyMDIyLmNybDANBgkqhkiG +9w0BAQsFAAOCAgEAmsINXtQ7wwUWvIeOr80MdJS/5G4xhyZOVEmeUorThquT672y +cCg3XCxc4fwbiZqSSbBqntQ7RtiTAKMYMvBageKoVHbzz+R4jX01tKcTx8cDePrz +dJ73bLNUorE7RU9QsW4KyiUeRmjMDV23AUlEvuQFTwgkHXvbac1BBdPn9CrssQuF +5EGohZKcQPFiAAc4SHbRNhlr7uAwgpc/erzI9EAcvA6BVAXcVKoeGpV01uexUgZ6 +St5RP9UmDWNA7T4yVXWJ233N0Q8bl+6AswINQ3PosPu6yQQHQjr65YS06epK+AeI +6j+oGR4xI7EhTQhQvaobnGmX/8QQ7XDRYCP2HXYxiffnn/CfZ/BVyKLYeY1ZipjE +nzqdQIC2+Q3WtY8jsVRQMP38WFRmtsIt5snehnPTs5bKGVIcYzj3o3Ex/K7agEz0 +zAJ0JR5ivXZOvNkT0g9x1v+S1IkU3e/nX1a+tpRquMtnHX0L2lXArNHUbaOO9EJt +d57WaIpofV5cVhhwShOgAuBc9UMJF3/n4t4RKiPxtsK8P67gcmphMhslj7AMYrYM +ej2NvQZY4m3ub3CPC/PrTjDONvb+8g5xrKtxBjYqC74HSB4dg9G3WimSDUuP2Su6 +G2y2TUeyJuCvCLz289VoO0vg7cNdMobE3KCqAiiNhN2VBFxHAUKmUoRcRdw= +-----END CERTIFICATE----- diff --git a/src/apps/parsers/clients/common/structured.py b/src/apps/parsers/clients/common/structured.py index f53f4ea..d0486e4 100644 --- a/src/apps/parsers/clients/common/structured.py +++ b/src/apps/parsers/clients/common/structured.py @@ -10,8 +10,9 @@ import zipfile from collections import Counter from dataclasses import dataclass, field from decimal import Decimal, InvalidOperation +from pathlib import Path from typing import Any -from urllib.parse import urljoin +from urllib.parse import parse_qs, urljoin, urlsplit from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError from apps.parsers.clients.common.schemas import GenericParserItem @@ -55,15 +56,19 @@ EIS_CARD_SOURCES = { "contracts", "unfair_suppliers", } +EIS_CA_BUNDLE_PATH = Path(__file__).with_name("eis_russian_trusted_ca.pem") ZAKUPKI_BASE_URL = "https://zakupki.gov.ru" +ZAKUPKI_HOST = "zakupki.gov.ru" ZAKUPKI_DETAIL_MAX_FILE_SIZE_BYTES = 2 * 1024 * 1024 ZAKUPKI_DETAIL_URL_MARKERS = ( + "/contract/contractCard/common-info", + "/223/purchase/public/purchase/info/common-info", "/view/common-info", "/card/common-info", "/contractCard/common-info", - "/notice/", "/orderclause/card/", "/contract/contractCard/", + "/notice/", ) INN_RE = re.compile(r"(? BaseHTTPClient: """Ленивая инициализация HTTP клиента.""" if self._http_client is None: + verify_ssl = self.verify_ssl + if verify_ssl is True and self.source in EIS_CARD_SOURCES: + verify_ssl = str(EIS_CA_BUNDLE_PATH) self._http_client = BaseHTTPClient( base_url="", proxies=self.proxies, timeout=self.timeout, - verify_ssl=self.verify_ssl, + verify_ssl=verify_ssl, ) return self._http_client @@ -179,7 +187,6 @@ class StructuredDataClient: "skip": 0, "take": GISP_PRODUCTS_PAGE_SIZE, "requireTotalCount": True, - "sort": [{"selector": "res_date", "desc": True}], } }, ) @@ -481,6 +488,9 @@ class StructuredDataClient: "Номер реестровой записи в ЕРУЗ", "Начальная цена", "Цена контракта", + "Заключение контракта", + "Размещен контракт в реестре контрактов", + "Обновлен контракт в реестре контрактов", "Размещено", "Обновлено", "Окончание подачи заявок", @@ -504,6 +514,9 @@ class StructuredDataClient: if detail_url: row["url"] = detail_url row["detail_url"] = detail_url + identity_url = self._select_zakupki_identity_url(card) + if identity_url: + row["identity_url"] = identity_url if lines[0].endswith("-ФЗ"): row["law"] = lines[0] if card_index < self.max_eis_detail_pages: @@ -520,7 +533,9 @@ class StructuredDataClient: href = str(link.get("href") or "").strip() if not href or href.startswith(("#", "javascript:")): continue - urls.append(urljoin(ZAKUPKI_BASE_URL, href)) + url = self._resolve_zakupki_url(href) + if url: + urls.append(url) for marker in ZAKUPKI_DETAIL_URL_MARKERS: for url in urls: @@ -528,6 +543,27 @@ class StructuredDataClient: return url return urls[0] if urls else "" + def _select_zakupki_identity_url(self, card: Any) -> str: + """Выбрать страницу организации с непустыми реквизитами заказчика.""" + for link in card.find_all("a", href=True): + href = str(link.get("href") or "").strip() + url = self._resolve_zakupki_url(href) + if not url or "/epz/organization/" not in urlsplit(url).path: + continue + query = parse_qs(urlsplit(url).query) + if any(query.get(key) for key in ("organizationCode", "inn", "ogrn")): + return url + return "" + + @staticmethod + def _resolve_zakupki_url(href: str) -> str: + """Разрешить только HTTPS-ссылки официального хоста ЕИС.""" + url = urljoin(ZAKUPKI_BASE_URL, href) + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.hostname != ZAKUPKI_HOST: + return "" + return url + def _enrich_zakupki_card_from_detail(self, row: dict[str, Any]) -> None: """Дозагрузить ИНН/КПП/ОГРН заказчика со страницы карточки ЕИС.""" if not self.enrich_eis_detail_pages: @@ -535,7 +571,9 @@ class StructuredDataClient: if row.get("inn") and row.get("ogrn"): return - detail_url = str(row.get("detail_url") or row.get("url") or "").strip() + detail_url = str( + row.get("identity_url") or row.get("detail_url") or row.get("url") or "" + ).strip() if not detail_url: return @@ -608,12 +646,14 @@ class StructuredDataClient: labels: tuple[str, ...], ) -> str: """Найти текстовое значение рядом с подписью на detail page.""" - for index, line in enumerate(lines[:-1]): - if not self._line_contains_label(line, labels): - continue - value = lines[index + 1].strip() - if value and not self._looks_like_detail_label(value): - return value + for label in labels: + normalized_label = self._normalize_detail_label(label) + for index, line in enumerate(lines[:-1]): + if self._normalize_detail_label(line) != normalized_label: + continue + value = lines[index + 1].strip() + if value and not self._looks_like_detail_label(value): + return value return "" @staticmethod @@ -763,7 +803,7 @@ class StructuredDataClient: ) -> None: """Заполнить пары label -> следующая строка из карточки.""" for index, line in enumerate(lines[:-1]): - if line in labels: + if line in labels and lines[index + 1] not in labels: row[line] = lines[index + 1] def _fill_zakupki_status( @@ -880,6 +920,9 @@ class StructuredDataClient: "объекты закупки", "наименование документа", "наименование закупки", + "наименование фио недобросовестного поставщика", + "полное наименование лица", + "фирменное наименование лица", "предмет", "описание", "должность", @@ -901,6 +944,8 @@ class StructuredDataClient: "дата внесения в реестр", "дата вступления постановления", "размещено", + "размещен контракт в реестре контрактов", + "заключение контракта", "обновлено", "включено", "утверждение", diff --git a/src/apps/parsers/clients/fns/__init__.py b/src/apps/parsers/clients/fns/__init__.py index 7383ac3..af952d0 100644 --- a/src/apps/parsers/clients/fns/__init__.py +++ b/src/apps/parsers/clients/fns/__init__.py @@ -1,10 +1,14 @@ -""" -Парсер бухгалтерской отчетности ФНС. - -Обрабатывает Excel файлы формата fin_{external_id}_{ogrn}.xlsx. -""" +"""Клиенты бухгалтерской отчётности ФНС: ГИР БО API и Excel.""" +from apps.parsers.clients.fns.api import FNSApiClient, FNSApiClientError, FNSApiReport from apps.parsers.clients.fns.parser import FNSExcelParser from apps.parsers.clients.fns.schemas import ParsedReport, ReportLine -__all__ = ["FNSExcelParser", "ParsedReport", "ReportLine"] +__all__ = [ + "FNSApiClient", + "FNSApiClientError", + "FNSApiReport", + "FNSExcelParser", + "ParsedReport", + "ReportLine", +] diff --git a/src/apps/parsers/clients/fns/api.py b/src/apps/parsers/clients/fns/api.py new file mode 100644 index 0000000..3779944 --- /dev/null +++ b/src/apps/parsers/clients/fns/api.py @@ -0,0 +1,303 @@ +"""Клиент публичного API ГИР БО ФНС для точечного получения отчётности.""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import asdict, dataclass, field +from decimal import Decimal, InvalidOperation +from typing import Any + +from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError +from apps.parsers.clients.fns.schemas import ReportLine +from requests.adapters import BaseAdapter + +DEFAULT_BASE_URL = "https://bo.nalog.gov.ru" +DEFAULT_MAX_PERIODS = 5 +SEARCH_ENDPOINT = "/advanced-search/organizations/search" +FORM_CODES = { + "balance": "1", + "financialResult": "2", + "capitalChange": "3", + "fundsMovement": "4", + "targetedFundsUsing": "6", +} +FORM_NAMES = { + "1": "Бухгалтерский баланс", + "2": "Отчёт о финансовых результатах", + "3": "Отчёт об изменениях капитала", + "4": "Отчёт о движении денежных средств", + "6": "Отчёт о целевом использовании средств", +} +VALUE_KEY_RE = re.compile(r"^(current|previous|beforePrevious)(\d+)$") +INN_RE = re.compile(r"^(?:\d{10}|\d{12})$") + + +class FNSApiClientError(HTTPClientError): + """Ошибка клиента публичного API ГИР БО.""" + + +@dataclass(frozen=True) +class FNSApiReport: + """Один опубликованный вариант бухгалтерской отчётности ГИР БО.""" + + organization_id: int + external_id: str + inn: str + ogrn: str + organization_name: str + year: int + period_type: int | None + actual_date: str + lines: list[ReportLine] + raw_payload: dict[str, Any] + + @property + def file_name(self) -> str: + """Стабильное логическое имя API payload для существующей модели.""" + safe_external_id = self.external_id.replace(":", "_") + return f"girbo_{self.organization_id}_{self.year}_{safe_external_id}.json" + + @property + def file_hash(self) -> str: + """Детерминированный хеш API payload для идемпотентной загрузки.""" + encoded = json.dumps( + self.raw_payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def to_save_report_kwargs(self, *, batch_id: int) -> dict[str, Any]: + """Подготовить аргументы для ``FNSReportService.save_report``.""" + return { + "external_id": self.external_id, + "ogrn": self.ogrn, + "file_name": self.file_name, + "file_hash": self.file_hash, + "source": "api", + "batch_id": batch_id, + "lines_data": [asdict(line) for line in self.lines], + } + + +@dataclass +class FNSApiClient: + """Получает ГИР БО только для явно заданной организации или ИНН.""" + + base_url: str = DEFAULT_BASE_URL + max_periods: int = DEFAULT_MAX_PERIODS + proxies: list[str] | None = None + timeout: int = 60 + http_adapter: BaseAdapter | None = None + _http_client: BaseHTTPClient | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + if self.max_periods < 1: + raise ValueError("max_periods must be positive") + + @property + def http_client(self) -> BaseHTTPClient: + """Ленивая browser-like HTTP-сессия, обязательная для публичного API.""" + if self._http_client is None: + self._http_client = BaseHTTPClient( + base_url=self.base_url, + proxies=self.proxies, + timeout=self.timeout, + adapter=self.http_adapter, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + "Accept": "application/json, text/plain, */*", + "Referer": f"{self.base_url.rstrip('/')}/", + }, + ) + return self._http_client + + def fetch_reports_by_inn(self, inn: str) -> list[FNSApiReport]: + """Найти точное совпадение по ИНН и получить его отчётность.""" + organization = self.find_organization_by_inn(inn) + if organization is None: + raise FNSApiClientError(f"FNS organization not found for INN {inn}") + return self.fetch_reports_by_organization_id(organization["id"]) + + def find_organization_by_inn(self, inn: str) -> dict[str, Any] | None: + """Вернуть точное совпадение из поиска, не сканируя общую выдачу.""" + normalized_inn = self._normalize_inn(inn) + data = self.http_client.get_json( + SEARCH_ENDPOINT, + params={"query": normalized_inn, "page": 0, "size": 10}, + ) + if not isinstance(data, dict) or not isinstance(data.get("content"), list): + raise FNSApiClientError("FNS search API returned invalid content") + for item in data["content"]: + if not isinstance(item, dict): + continue + candidate_inn = re.sub(r"\D", "", str(item.get("inn") or "")) + if candidate_inn == normalized_inn and self._organization_id(item): + return item + return None + + def fetch_reports_by_organization_id( + self, organization_id: int | str + ) -> list[FNSApiReport]: + """Получить опубликованные периоды для конкретного ID организации.""" + parsed_id = self._normalize_organization_id(organization_id) + organization = self.http_client.get_json(f"/nbo/organizations/{parsed_id}") + bfo = self.http_client.get_json(f"/nbo/organizations/{parsed_id}/bfo/") + if not isinstance(organization, dict): + raise FNSApiClientError("FNS organization API returned invalid payload") + if not isinstance(bfo, list): + raise FNSApiClientError("FNS BFO API returned invalid payload") + return self._map_reports( + organization_id=parsed_id, + organization=organization, + bfo=bfo[: self.max_periods], + ) + + def _map_reports( + self, + *, + organization_id: int, + organization: dict[str, Any], + bfo: list[Any], + ) -> list[FNSApiReport]: + reports: list[FNSApiReport] = [] + inn = re.sub(r"\D", "", str(organization.get("inn") or "")) + ogrn = re.sub(r"\D", "", str(organization.get("ogrn") or "")) + organization_name = str( + organization.get("fullName") or organization.get("shortName") or "" + ).strip() + + for period_payload in bfo: + if not isinstance(period_payload, dict): + continue + year = self._read_year(period_payload.get("period")) + type_corrections = period_payload.get("typeCorrections") + if year is None or not isinstance(type_corrections, list): + continue + for typed_correction in type_corrections: + if not isinstance(typed_correction, dict): + continue + correction = typed_correction.get("correction") + if not isinstance(correction, dict): + continue + correction_id = correction.get("id") + if correction_id in (None, ""): + continue + reports.append( + FNSApiReport( + organization_id=organization_id, + external_id=f"girbo:{correction_id}", + inn=inn, + ogrn=ogrn, + organization_name=organization_name, + year=year, + period_type=self._optional_int(typed_correction.get("type")), + actual_date=str( + period_payload.get("actualBfoDate") + or correction.get("datePresent") + or "" + ), + lines=self._map_lines(correction, year=year), + raw_payload=correction, + ) + ) + return reports + + @classmethod + def _map_lines(cls, correction: dict[str, Any], *, year: int) -> list[ReportLine]: + lines: list[ReportLine] = [] + for report_key, form_code in FORM_CODES.items(): + form = correction.get(report_key) + if not isinstance(form, dict): + continue + values_by_code: dict[str, dict[str, int | None]] = {} + for key, value in form.items(): + match = VALUE_KEY_RE.fullmatch(str(key)) + if match is None: + continue + value_kind, line_code = match.groups() + values_by_code.setdefault(line_code, {})[ + value_kind + ] = cls._optional_int(value) + + for line_code in sorted(values_by_code): + values = values_by_code[line_code] + period_start = values.get("previous") + period_end = values.get("current") + if period_start is None and period_end is None: + continue + lines.append( + ReportLine( + form_code=form_code, + line_code=line_code, + line_name=f"{FORM_NAMES[form_code]}, строка {line_code}", + year=year, + period_start=period_start, + period_end=period_end, + ) + ) + return lines + + @staticmethod + def _normalize_inn(value: str) -> str: + inn = re.sub(r"\D", "", str(value)) + if not INN_RE.fullmatch(inn): + raise ValueError("INN must contain 10 or 12 digits") + return inn + + @staticmethod + def _normalize_organization_id(value: int | str) -> int: + try: + organization_id = int(value) + except (TypeError, ValueError) as exc: + raise ValueError("organization_id must be a positive integer") from exc + if organization_id <= 0: + raise ValueError("organization_id must be a positive integer") + return organization_id + + @staticmethod + def _organization_id(item: dict[str, Any]) -> int | None: + raw_value = item.get("id") + if raw_value is None: + return None + try: + value = int(raw_value) + except (TypeError, ValueError): + return None + return value if value > 0 else None + + @staticmethod + def _read_year(value: Any) -> int | None: + try: + year = int(value) + except (TypeError, ValueError): + return None + return year if 1990 <= year <= 2100 else None + + @staticmethod + def _optional_int(value: Any) -> int | None: + if value is None or value == "" or isinstance(value, bool): + return None + try: + return int(Decimal(str(value))) + except (InvalidOperation, TypeError, ValueError): + return None + + def close(self) -> None: + """Закрыть HTTP-сессию.""" + if self._http_client is not None: + self._http_client.close() + self._http_client = None + + def __enter__(self) -> FNSApiClient: + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/src/apps/parsers/clients/gisp/__init__.py b/src/apps/parsers/clients/gisp/__init__.py new file mode 100644 index 0000000..5490eb6 --- /dev/null +++ b/src/apps/parsers/clients/gisp/__init__.py @@ -0,0 +1,13 @@ +"""Клиент официального реестра промышленной продукции ГИСП.""" + +from apps.parsers.clients.gisp.client import ( + GispProductsClient, + GispProductsClientError, + GispProductsResult, +) + +__all__ = [ + "GispProductsClient", + "GispProductsClientError", + "GispProductsResult", +] diff --git a/src/apps/parsers/clients/gisp/client.py b/src/apps/parsers/clients/gisp/client.py new file mode 100644 index 0000000..3b5da88 --- /dev/null +++ b/src/apps/parsers/clients/gisp/client.py @@ -0,0 +1,226 @@ +"""Клиент публичного UI API реестра промышленной продукции ГИСП.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError +from apps.parsers.clients.minpromtorg.schemas import IndustrialProduct +from requests.adapters import BaseAdapter + +DEFAULT_BASE_URL = "https://gisp.gov.ru" +PRODUCTS_ENDPOINT = "/pp719v2/pub/prod/b/" +DEFAULT_PAGE_SIZE = 100 +DEFAULT_MAX_PAGES = 10 +DEFAULT_MAX_RECORDS = 1_000 +MAX_PAGE_SIZE = 100 +MAX_PAGES = 100 +MAX_RECORDS = 10_000 + + +class GispProductsClientError(HTTPClientError): + """Ошибка клиента реестра продукции ГИСП.""" + + +@dataclass(frozen=True) +class GispProductsResult: + """Ограниченный результат последовательной загрузки страниц ГИСП.""" + + products: list[IndustrialProduct] + total_count: int | None + pages_fetched: int + items_fetched: int + truncated: bool + + +@dataclass +class GispProductsClient: + """Загружает продукцию ГИСП с явными ограничителями объёма.""" + + base_url: str = DEFAULT_BASE_URL + page_size: int = DEFAULT_PAGE_SIZE + max_pages: int = DEFAULT_MAX_PAGES + max_records: int = DEFAULT_MAX_RECORDS + proxies: list[str] | None = None + timeout: int = 120 + http_adapter: BaseAdapter | None = None + _http_client: BaseHTTPClient | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + if not 1 <= self.page_size <= MAX_PAGE_SIZE: + raise ValueError(f"page_size must be between 1 and {MAX_PAGE_SIZE}") + if not 1 <= self.max_pages <= MAX_PAGES: + raise ValueError(f"max_pages must be between 1 and {MAX_PAGES}") + if not 1 <= self.max_records <= MAX_RECORDS: + raise ValueError(f"max_records must be between 1 and {MAX_RECORDS}") + + @property + def http_client(self) -> BaseHTTPClient: + """Ленивая инициализация HTTP клиента.""" + if self._http_client is None: + self._http_client = BaseHTTPClient( + base_url=self.base_url, + proxies=self.proxies, + timeout=self.timeout, + adapter=self.http_adapter, + headers={ + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json", + "Referer": f"{self.base_url.rstrip('/')}/pp719v2/pub/prod/", + }, + ) + return self._http_client + + def fetch_products(self) -> GispProductsResult: + """Последовательно загрузить страницы, не превышая заданные лимиты.""" + products: list[IndustrialProduct] = [] + total_count: int | None = None + pages_fetched = 0 + items_fetched = 0 + offset = 0 + source_exhausted = False + + for _page in range(self.max_pages): + remaining = self.max_records - items_fetched + if remaining <= 0: + break + take = min(self.page_size, remaining) + items, total_count = self._fetch_page( + offset=offset, + take=take, + previous_total_count=total_count, + ) + pages_fetched += 1 + items_fetched += len(items) + offset += len(items) + + for item in items: + if not isinstance(item, dict): + continue + product = self._map_product(item) + if product is not None: + products.append(product) + + if self._source_exhausted( + items_count=len(items), + requested_count=take, + offset=offset, + total_count=total_count, + ): + source_exhausted = True + break + + truncated = not source_exhausted and ( + total_count is None or items_fetched < total_count + ) + return GispProductsResult( + products=products, + total_count=total_count, + pages_fetched=pages_fetched, + items_fetched=items_fetched, + truncated=truncated, + ) + + def _fetch_page( + self, + *, + offset: int, + take: int, + previous_total_count: int | None, + ) -> tuple[list[Any], int | None]: + data = self.http_client.post_json( + PRODUCTS_ENDPOINT, + payload={ + "opt": { + "skip": offset, + "take": take, + "requireTotalCount": True, + } + }, + ) + if data.get("ok") is False: + raise GispProductsClientError("GISP products API returned ok=false") + items = data.get("items") + if not isinstance(items, list): + raise GispProductsClientError("GISP products API returned no items list") + total_count = self._read_total_count(data, previous=previous_total_count) + return items, total_count + + @staticmethod + def _source_exhausted( + *, + items_count: int, + requested_count: int, + offset: int, + total_count: int | None, + ) -> bool: + if total_count is not None: + if offset >= total_count: + return True + if items_count == 0: + raise GispProductsClientError( + "GISP products API returned an empty page before total_count" + ) + return False + return items_count < requested_count + + @staticmethod + def _read_total_count(data: dict[str, Any], *, previous: int | None) -> int | None: + value = data.get("total_count", data.get("totalCount")) + if value is None: + return previous + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise GispProductsClientError( + "GISP products API returned invalid total_count" + ) from exc + if parsed < 0: + raise GispProductsClientError( + "GISP products API returned negative total_count" + ) + return parsed + + @classmethod + def _map_product(cls, item: dict[str, Any]) -> IndustrialProduct | None: + registry_number = cls._text( + item.get("_product_reg_number_2023") or item.get("_product_reg_number_2022") + ) + product_name = cls._text(item.get("_product_name")) + if not registry_number or not product_name: + return None + + document_parts = [ + cls._text(item.get("_basedondoc_name")), + cls._text(item.get("_basedondoc_num")), + cls._text(item.get("_basedondoc_date")), + ] + regulatory_document = " | ".join(part for part in document_parts if part) + return IndustrialProduct( + full_organisation_name=cls._text(item.get("_org_name")), + inn=cls._text(item.get("_org_inn")), + ogrn=cls._text(item.get("_org_ogrn")), + registry_number=registry_number, + product_name=product_name, + product_model=cls._text(item.get("_product_spec")), + okpd2_code=cls._text(item.get("_product_okpd2")), + tnved_code=cls._text(item.get("_product_tnved")), + regulatory_document=regulatory_document, + ) + + @staticmethod + def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + def close(self) -> None: + """Закрыть HTTP-сессию.""" + if self._http_client is not None: + self._http_client.close() + self._http_client = None + + def __enter__(self) -> GispProductsClient: + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/src/apps/parsers/clients/proverki/client.py b/src/apps/parsers/clients/proverki/client.py index 195e3f2..bfc0df7 100644 --- a/src/apps/parsers/clients/proverki/client.py +++ b/src/apps/parsers/clients/proverki/client.py @@ -301,8 +301,10 @@ class ProverkiClient: if self.use_playwright: if progress_callback: progress_callback(20, "Навигация по порталу...") - content = self._download_from_portal(file_url, progress_callback) - self._close_playwright() + try: + content = self._download_from_portal(file_url, progress_callback) + finally: + self._close_playwright() else: if progress_callback: progress_callback(20, f"Скачивание {file_url}...") @@ -330,10 +332,12 @@ class ProverkiClient: progress_callback( 22, "Сервер требует JavaScript, запускаем браузер..." ) - content = self._download_with_playwright( - file_url, progress_callback - ) - self._close_playwright() + try: + content = self._download_with_playwright( + file_url, progress_callback + ) + finally: + self._close_playwright() else: raise ProverkiClientError( "Сервер вернул HTML вместо данных. " @@ -789,7 +793,7 @@ class ProverkiClient: Адаптируется к различным форматам XML proverki.gov.ru: - Данные могут быть в атрибутах элементов - - Данные могут быть во вложенных элементах (I_SUBJECT, I_AUTHORITY) + - Данные могут быть во вложенных элементах старой и текущей схемы """ try: # Определяем namespace элемента @@ -810,6 +814,10 @@ class ProverkiClient: i_authority = find_child("I_AUTHORITY") # Контролирующий орган i_approve = find_child("I_APPROVE") # Информация об утверждении i_classification = find_child("I_CLASSIFICATION") # Классификация + subject = find_child("SUBJECT") + kno_organization = find_child("KNO_ORGANIZATION") + kind_control = find_child("KIND_CONTROL") + kind_knm = find_child("KIND_KNM") def get_attr_value(attr_names: list[str]) -> str: # noqa: C901 """Найти значение атрибута в элементе или вложенных элементах.""" @@ -824,6 +832,12 @@ class ProverkiClient: if name in i_subject.attrib: return i_subject.attrib[name].strip() + # Ищем в SUBJECT текущей схемы ФЗ-248 (NAME, INN, OGRN) + if subject is not None: + for name in attr_names: + if name in subject.attrib: + return subject.attrib[name].strip() + # Ищем в I_AUTHORITY (FRGU_ORG_NAME) if i_authority is not None: for name in attr_names: @@ -853,6 +867,12 @@ class ProverkiClient: return "" + def get_value_attr(child: ET.Element | None) -> str: + """Прочитать VALUE у справочного элемента текущей схемы.""" + if child is None: + return "" + return child.attrib.get("VALUE", "").strip() + # Маппинг атрибутов на поля Inspection # Используем названия из реального XML proverki.gov.ru # Включая русские теги (КНМ формат) @@ -876,6 +896,7 @@ class ProverkiClient: "ORG_NAME", "FULL_NAME", "SHORT_NAME", + "NAME", "I_NAME", "organisation_name", "org_name", @@ -893,7 +914,7 @@ class ProverkiClient: "authority", "КонтрольныйОрган", ] - ) + ) or get_value_attr(kno_organization) inspection_type = get_attr_value( [ "ITYPE_NAME", @@ -903,7 +924,7 @@ class ProverkiClient: "type", "ТипПроверки", ] - ) + ) or get_value_attr(kind_control) inspection_form = get_attr_value( [ "ICARRYOUT_TYPE_NAME", @@ -913,7 +934,7 @@ class ProverkiClient: "form", "ФормаПроверки", ] - ) + ) or get_value_attr(kind_knm) start_date = get_attr_value( [ "START_DATE", @@ -928,6 +949,7 @@ class ProverkiClient: end_date = get_attr_value( [ "END_DATE", + "STOP_DATE", "I_DATE_END", "DATE_END", "end_date", @@ -1032,6 +1054,10 @@ class ProverkiClient: "playwright install chromium" ) from e except Exception as e: + # sync_playwright().start() owns an asyncio runtime. If the + # browser launch fails, leaving it active makes subsequent + # synchronous Django ORM calls look like async-context calls. + self._close_playwright() raise ProverkiClientError( f"Не удалось запустить Playwright browser: {e}" ) from e diff --git a/src/apps/parsers/clients/trudvsem/client.py b/src/apps/parsers/clients/trudvsem/client.py index ae940e4..3dc6951 100644 --- a/src/apps/parsers/clients/trudvsem/client.py +++ b/src/apps/parsers/clients/trudvsem/client.py @@ -14,7 +14,7 @@ from requests.adapters import BaseAdapter logger = logging.getLogger(__name__) -DEFAULT_BASE_URL = "http://opendata.trudvsem.ru/api/v1" +DEFAULT_BASE_URL = "https://opendata.trudvsem.ru/api/v1" VACANCIES_ENDPOINT = "/vacancies" COMPANY_INN_VACANCIES_ENDPOINT = "/vacancies/company/inn/{inn}" diff --git a/src/apps/parsers/clients/vacancies.py b/src/apps/parsers/clients/vacancies.py index 1f5f330..c9da5b8 100644 --- a/src/apps/parsers/clients/vacancies.py +++ b/src/apps/parsers/clients/vacancies.py @@ -87,6 +87,7 @@ def _stable_external_id(source: str, payload: Mapping[str, Any]) -> str: class HHVacanciesClient: """Клиент публичного API HeadHunter.""" + user_agent: str = "" proxies: list[str] | None = None base_url: str = HH_BASE_URL timeout: int = 120 @@ -97,12 +98,20 @@ class HHVacanciesClient: @property def http_client(self) -> BaseHTTPClient: if self._http_client is None: + user_agent = self.user_agent.strip() + if not user_agent: + raise VacanciesClientError( + "HH_USER_AGENT is required; use 'App/Version (contact email)'" + ) self._http_client = BaseHTTPClient( base_url=self.base_url, proxies=self.proxies, timeout=self.timeout, adapter=self.http_adapter, - headers={"Accept": "application/json"}, + headers={ + "Accept": "application/json", + "HH-User-Agent": user_agent, + }, ) return self._http_client @@ -270,6 +279,7 @@ class VacanciesClient: proxies: list[str] | None = None superjob_app_id: str = "" + hh_user_agent: str = "" sources: list[str] | None = None source_clients: dict[str, VacancyProvider] | None = None _source_clients_cache: dict[str, VacancyProvider] | None = field( @@ -287,7 +297,10 @@ class VacanciesClient: clients: dict[str, VacancyProvider] = { TRUDVSEM_SOURCE: TrudvsemClient(proxies=self.proxies), - HH_SOURCE: HHVacanciesClient(proxies=self.proxies), + HH_SOURCE: HHVacanciesClient( + user_agent=self.hh_user_agent, + proxies=self.proxies, + ), } if self.superjob_app_id: clients[SUPERJOB_SOURCE] = SuperJobVacanciesClient( diff --git a/src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py b/src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py new file mode 100644 index 0000000..5975a8f --- /dev/null +++ b/src/apps/parsers/migrations/0025_consolidate_inspections_schedule.py @@ -0,0 +1,63 @@ +import json + +from django.db import migrations + +CANONICAL_SCHEDULE_NAME = "parser:sync_inspections:weekly-saturday-msk" +CANONICAL_TASK_NAME = "apps.parsers.tasks.sync_inspections" +LEGACY_SCHEDULE_NAMES = ( + "parser:inspections:weekly-saturday-msk", + "parse-inspections-weekly", +) +WEEKLY_MSK_CRON = { + "minute": "0", + "hour": "0", + "day_of_week": "6", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "Europe/Moscow", +} + + +def consolidate_inspections_schedule(apps, schema_editor): + CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + PeriodicTask.objects.filter(name__in=LEGACY_SCHEDULE_NAMES).delete() + + canonical = PeriodicTask.objects.filter(name=CANONICAL_SCHEDULE_NAME).first() + if canonical is not None: + canonical.task = CANONICAL_TASK_NAME + canonical.args = json.dumps([]) + canonical.kwargs = json.dumps({}) + canonical.save(update_fields=["task", "args", "kwargs"]) + return + + crontab, _ = CrontabSchedule.objects.get_or_create(**WEEKLY_MSK_CRON) + field_names = {field.name for field in PeriodicTask._meta.fields} + schedule_fields = {"crontab": crontab} + for field_name in ("interval", "solar", "clocked"): + if field_name in field_names: + schedule_fields[field_name] = None + + PeriodicTask.objects.create( + name=CANONICAL_SCHEDULE_NAME, + task=CANONICAL_TASK_NAME, + args=json.dumps([]), + kwargs=json.dumps({}), + enabled=True, + description="Default inspections sync: weekly on Saturday 00:00 MSK.", + **schedule_fields, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0024_auto_20260607_1017"), + ] + + operations = [ + migrations.RunPython( + consolidate_inspections_schedule, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/src/apps/parsers/migrations/0026_update_fns_financial_schedule.py b/src/apps/parsers/migrations/0026_update_fns_financial_schedule.py new file mode 100644 index 0000000..f74cd6a --- /dev/null +++ b/src/apps/parsers/migrations/0026_update_fns_financial_schedule.py @@ -0,0 +1,51 @@ +import json + +from django.db import migrations + +SCHEDULE_NAME = "parser:fns_financial:weekly-saturday-msk" +TASK_NAME = "apps.parsers.tasks.sync_fns_financial_reports" +WEEKLY_MSK_CRON = { + "minute": "0", + "hour": "0", + "day_of_week": "6", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "Europe/Moscow", +} + + +def update_fns_financial_schedule(apps, schema_editor): + CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + crontab, _ = CrontabSchedule.objects.get_or_create(**WEEKLY_MSK_CRON) + field_names = {field.name for field in PeriodicTask._meta.fields} + schedule_fields = {"crontab": crontab} + for field_name in ("interval", "solar", "clocked"): + if field_name in field_names: + schedule_fields[field_name] = None + + PeriodicTask.objects.update_or_create( + name=SCHEDULE_NAME, + defaults={ + "task": TASK_NAME, + "args": json.dumps([]), + "kwargs": json.dumps({}), + "enabled": True, + "description": "Default FNS API sync: weekly on Saturday 00:00 MSK.", + **schedule_fields, + }, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0025_consolidate_inspections_schedule"), + ] + + operations = [ + migrations.RunPython( + update_fns_financial_schedule, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index 219d45d..93eabd0 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -921,11 +921,11 @@ class SourceCardService: params: dict[str, Any], ) -> list[dict[str, str]]: if definition.slug == "financial-indicators": - from apps.parsers.tasks import scan_fns_directory + from apps.parsers.tasks import sync_fns_financial_reports task_info = cls._enqueue_task( - task=scan_fns_directory, - task_name="apps.parsers.tasks.scan_fns_directory", + task=sync_fns_financial_reports, + task_name="apps.parsers.tasks.sync_fns_financial_reports", requested_by_id=requested_by_id, meta={ "source_card": definition.slug, diff --git a/src/apps/parsers/source_registry.py b/src/apps/parsers/source_registry.py index d7a665c..d9b14e3 100644 --- a/src/apps/parsers/source_registry.py +++ b/src/apps/parsers/source_registry.py @@ -136,7 +136,9 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { task_name="apps.parsers.tasks.parse_procurements_44fz", mode="official_api", status="implemented", - upstream_url="https://zakupki.gov.ru/epz/order/extendedsearch/results.html", + upstream_url=( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html?fz44=on" + ), access_method="eis_official_api", parser_strategy="eis_44fz_search", source_notes="Официальный поиск ЕИС; XML-выгрузки ЕИС требуют отдельного discovery по реестрам.", @@ -151,10 +153,12 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { task_name="apps.parsers.tasks.parse_procurements_223fz", mode="official_api", status="implemented", - upstream_url="https://zakupki.gov.ru/epz/orderclause/search/results.html", + upstream_url=( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html?fz223=on" + ), access_method="eis_official_api", parser_strategy="eis_223fz_search", - source_notes="Официальный реестр положений о закупках 223-ФЗ в ЕИС.", + source_notes="Официальный поиск закупочных процедур 223-ФЗ в ЕИС.", api_route="eis/procurements-223fz", ), "contracts": ParserSourceDescriptor( @@ -205,19 +209,17 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { title="Финансово-экономические показатели", agency="ФНС России", data_scope="Финансово-экономическая выгрузка", - task_name="apps.parsers.tasks.scan_fns_directory", + task_name="apps.parsers.tasks.sync_fns_financial_reports", mode="official_api", status="implemented", owner="Сергей", - upstream_url=( - "https://bo.nalog.gov.ru/advanced-search/organizations/search" - "?query=%D0%9E%D0%9E%D0%9E&page=0&size=100" - ), + upstream_url="https://bo.nalog.gov.ru/advanced-search/organizations/search", access_method="public_web_api", - parser_strategy="fns_bfo_search_and_download", + parser_strategy="fns_bfo_bounded_registry_sync", source_notes=( - "ГИР БО: поиск организаций и скачивание отчетности с ЭП ФНС. " - "Ручная загрузка разрешена для финансовых выгрузок от Сергея." + "ГИР БО: точечная загрузка по ИНН/ID ФНС или ограниченному набору " + "активных организаций реестров; общий каталог не сканируется. " + "Ручная загрузка файлов и файловый scanner сохранены." ), supports_file_upload=True, api_route="fns/reports", diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 0934eb7..ff33f75 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -21,6 +21,7 @@ from apps.core.tasks import PeriodicTask as CorePeriodicTask from apps.parsers.clients.base import HTTPClientError from apps.parsers.clients.checko import ( CheckoClient, + CheckoRateLimitError, CompanyRequest, ContractLaw, ContractsRequest, @@ -32,9 +33,10 @@ from apps.parsers.clients.checko import ( ) from apps.parsers.clients.checko.exceptions import CheckoError from apps.parsers.clients.common import GenericParserItem, StructuredDataClient +from apps.parsers.clients.fns import FNSApiClient, FNSApiReport +from apps.parsers.clients.gisp import GispProductsClient from apps.parsers.clients.minpromtorg import ( IndustrialProductionClient, - IndustrialProductsClient, ManufacturesClient, ) from apps.parsers.clients.proverki import ProverkiClient @@ -58,6 +60,7 @@ from apps.parsers.services import ( from apps.parsers.source_registry import PARSER_SOURCES from celery import shared_task from django.conf import settings +from django.db.models import Q from organizations.models import Organization as SourceOrganization from organizations.services import ( normalize_organization_name as normalize_identity_name, @@ -81,6 +84,10 @@ PARSER_TIME_LIMIT_SECONDS = 20 * 60 INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS = 45 * 60 INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS = 60 * 60 CHECKO_SEARCH_RESULT_LIMIT = 100 +FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT = 25 +FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT = 100 +FNS_API_DEFAULT_MAX_PERIODS = 5 +FNS_API_MAX_PERIODS = 10 class ParserSourceSkipped(Exception): @@ -109,6 +116,16 @@ class FstecIdentityCandidate: provider: str +@dataclass(frozen=True) +class FNSApiFetchResult: + """Итог ограниченной загрузки отчётов из ГИР БО.""" + + reports: list[FNSApiReport] + target_count: int + successful_targets: int + failed_targets: int + + VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION = 100 VACANCY_REGISTRY_TEXT_SEARCH_MAX_PAGES_PER_ORGANIZATION = 1 VACANCY_EMPLOYER_WORD_RE = re.compile(r"[0-9A-Za-zА-Яа-яЁё]+") @@ -410,12 +427,17 @@ def _fetch_fedresurs_bankruptcy_records( """Загрузить банкротства: официальный портал, затем fallback через Checko.""" official_error: Exception | None = None try: - return _fetch_structured_records( + official_records = _fetch_structured_records( source_key="fedresurs_bankruptcy", file_url=file_url, file_path=file_path, proxies=proxies, ) + if official_records or file_url or file_path: + return official_records + logger.warning( + "Fedresurs official source returned no records, falling back to Checko" + ) except Exception as exc: if file_url or file_path: raise @@ -424,15 +446,20 @@ def _fetch_fedresurs_bankruptcy_records( "Fedresurs official source failed, falling back to Checko: %s", exc, ) - records = _fetch_checko_bankruptcy_records(proxies=proxies) - if records: - return records - if isinstance(official_error, HTTPClientError): - raise ParserSourceSkipped( - "fedresurs upstream is unavailable or blocked; " - "Checko fallback returned no bankruptcy records" - ) from official_error + records = _fetch_checko_bankruptcy_records(proxies=proxies) + if records: return records + if official_error is None: + raise ParserSourceSkipped( + "fedresurs official source returned no bankruptcy records; " + "Checko fallback returned no bankruptcy records" + ) + if isinstance(official_error, HTTPClientError): + raise ParserSourceSkipped( + "fedresurs upstream is unavailable or blocked; " + "Checko fallback returned no bankruptcy records" + ) from official_error + raise official_error def _fetch_fstec_records( @@ -805,6 +832,13 @@ def _fetch_checko_bankruptcy_records( response = client.get_company( CompanyRequest(inn=target.inn or None, ogrn=target.ogrn or None) ) + except CheckoRateLimitError as exc: + logger.warning( + "Checko bankruptcy fallback stopped: quota/rate limit reached " + "(status_code=%s)", + exc.status_code, + ) + break except CheckoError as exc: logger.info( "Checko bankruptcy lookup skipped for target=%s: %s", @@ -1878,6 +1912,9 @@ def parse_industrial_products( proxies: list[str] | None = None, client_adapter: BaseAdapter | None = None, requested_by_id: int | None = None, + page_size: int = 100, + max_pages: int = 10, + max_records: int = 1_000, ) -> dict: """ Задача парсинга реестра промышленной продукции. @@ -1885,6 +1922,9 @@ def parse_industrial_products( Args: proxies: Список прокси-серверов (опционально). client_adapter: HTTP-адаптер (опционально). + page_size: Размер страницы ГИСП (не более 100). + max_pages: Максимальное число страниц ГИСП (не более 100). + max_records: Максимальное число записей за запуск (не более 10 000). Returns: Результат: batch_id, saved, status @@ -1916,12 +1956,18 @@ def parse_industrial_products( job.update_progress(0, "Инициализация парсера...") try: - job.update_progress(10, "Загрузка данных с API Минпромторга...") - client_kwargs = {"proxies": proxies} + job.update_progress(10, "Загрузка данных из реестра ГИСП...") + client_kwargs = { + "proxies": proxies, + "page_size": page_size, + "max_pages": max_pages, + "max_records": max_records, + } if client_adapter: client_kwargs["http_adapter"] = client_adapter - with IndustrialProductsClient(**client_kwargs) as client: - products = client.fetch_products() + with GispProductsClient(**client_kwargs) as client: + fetch_result = client.fetch_products() + products = fetch_result.products job.update_progress(50, f"Сохранение {len(products)} записей продукции...") saved_count = IndustrialProductService.save_products( @@ -1935,19 +1981,27 @@ def parse_industrial_products( records_count=saved_count, ) - job.complete(result={"batch_id": batch_id, "saved": saved_count}) - - logger.info( - "Industrial products parsing completed (batch_id=%d, saved=%d)", - batch_id, - saved_count, - ) - - return { + result = { "batch_id": batch_id, "saved": saved_count, "status": "success", + "fetched": fetch_result.items_fetched, + "total": fetch_result.total_count, + "truncated": fetch_result.truncated, } + job.complete(result=result) + + logger.info( + "GISP industrial products parsing completed " + "(batch_id=%d, fetched=%d, total=%s, truncated=%s, saved=%d)", + batch_id, + fetch_result.items_fetched, + fetch_result.total_count, + fetch_result.truncated, + saved_count, + ) + + return result except Exception as e: logger.error("Industrial products parsing failed: %s", e, exc_info=True) @@ -2154,7 +2208,7 @@ def parse_all_sources( manufactures_result = parse_manufactures.apply( kwargs={"proxies": proxies, "client_adapter": client_adapter} ) - inspections_result = parse_inspections.apply( + inspections_result = sync_inspections.apply( kwargs={ "proxies": proxies, "client_adapter": client_adapter, @@ -2175,7 +2229,7 @@ def parse_all_sources( proxies=proxies, client_adapter=client_adapter, ) - inspections_result = parse_inspections.delay( + inspections_result = sync_inspections.delay( proxies=proxies, client_adapter=client_adapter, use_playwright=inspections_use_playwright, @@ -2190,6 +2244,7 @@ def parse_all_sources( "contracts": parse_contracts.delay(proxies=proxies).id, "unfair_suppliers": parse_unfair_suppliers.delay(proxies=proxies).id, "fas_goz": parse_fas_goz_evasion.delay(proxies=proxies).id, + "fns_financial": sync_fns_financial_reports.delay(proxies=proxies).id, "arbitration": parse_arbitration_cases.delay(proxies=proxies).id, "fedresurs_bankruptcy": parse_fedresurs_bankruptcy.delay( proxies=proxies @@ -3162,6 +3217,7 @@ def cleanup_stale_parser_loads(max_age_minutes: int | None = None) -> dict: ) source_values = {descriptor.source for descriptor in PARSER_SOURCES.values()} task_names = {descriptor.task_name for descriptor in PARSER_SOURCES.values()} + task_names.add("apps.parsers.tasks.scan_fns_directory") marked_failed = ParserLoadLogService.mark_stale_in_progress_failed( max_age_minutes=int(max_age_minutes) ) @@ -3243,6 +3299,7 @@ def _fetch_vacancy_records( with VacanciesClient( proxies=proxies, superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""), + hh_user_agent=getattr(settings, "HH_USER_AGENT", ""), sources=vacancy_sources, ) as client: return client.fetch_vacancies( @@ -3468,6 +3525,7 @@ def _fetch_registry_organization_vacancy_records( with VacanciesClient( proxies=proxies, superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""), + hh_user_agent=getattr(settings, "HH_USER_AGENT", ""), sources=vacancy_sources, ) as client: for target in targets: @@ -3518,6 +3576,231 @@ def _fetch_registry_organization_vacancy_records( # ============================================================================= +def _resolve_fns_api_registry_limit(value: int | None) -> int: + """Ограничить точечную синхронизацию безопасным числом организаций.""" + resolved = _resolve_lookup_limit( + value, + default=FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT, + ) + return min(resolved, FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT) + + +def _resolve_fns_api_max_periods(value: int | None) -> int: + """Ограничить число запрашиваемых отчётных периодов одной организации.""" + resolved = _resolve_lookup_limit(value, default=FNS_API_DEFAULT_MAX_PERIODS) + return min(max(resolved, 1), FNS_API_MAX_PERIODS) + + +def _active_fns_registry_inns(*, limit: int) -> list[str]: + """Вернуть ограниченный набор ИНН активных организаций реестров.""" + if limit <= 0: + return [] + + queryset = ( + SourceOrganization.objects.filter( + Q(opk_registry_membership=True) + | Q(goz_participation=True) + | ~Q(ropk_num="") + ) + .exclude(inn="") + .order_by("inn") + .values_list("inn", flat=True) + .distinct() + ) + return [str(value).strip() for value in queryset[:limit] if str(value).strip()] + + +def _fetch_fns_registry_reports( + client: FNSApiClient, + *, + target_inns: list[str], +) -> FNSApiFetchResult: + """Получить отчёты реестровых организаций, изолируя ошибки одной цели.""" + reports: list[FNSApiReport] = [] + successful_targets = 0 + failed_targets = 0 + first_target_error = "" + for target_inn in target_inns: + try: + reports.extend(client.fetch_reports_by_inn(target_inn)) + successful_targets += 1 + except (HTTPClientError, ValueError) as exc: + failed_targets += 1 + if not first_target_error: + first_target_error = str(exc) + logger.warning( + "FNS API sync failed for registry INN %s: %s", + target_inn, + exc, + ) + + if failed_targets and successful_targets == 0: + raise RuntimeError( + "FNS API sync failed for every selected registry organization; " + f"first error: {first_target_error}" + ) + return FNSApiFetchResult( + reports=reports, + target_count=len(target_inns), + successful_targets=successful_targets, + failed_targets=failed_targets, + ) + + +def _fetch_fns_api_reports( + *, + normalized_inn: str, + normalized_organization_id: str, + registry_limit: int, + max_periods: int, + proxies: list[str] | None, +) -> FNSApiFetchResult: + """Выбрать только явную цель или ограниченный набор реестровых ИНН.""" + target_inns: list[str] | None = None + if not normalized_inn and not normalized_organization_id: + target_inns = _active_fns_registry_inns(limit=registry_limit) + if not target_inns: + return FNSApiFetchResult([], 0, 0, 0) + + with FNSApiClient(max_periods=max_periods, proxies=proxies) as client: + if normalized_organization_id: + reports = client.fetch_reports_by_organization_id( + normalized_organization_id + ) + return FNSApiFetchResult(reports, 1, 1, 0) + if normalized_inn: + reports = client.fetch_reports_by_inn(normalized_inn) + return FNSApiFetchResult(reports, 1, 1, 0) + return _fetch_fns_registry_reports(client, target_inns=target_inns or []) + + +def _save_fns_api_reports( + reports: list[FNSApiReport], + *, + batch_id: int, +) -> tuple[int, int, int]: + """Сохранить API-отчёты через общую FNS ingestion boundary.""" + saved_reports = 0 + saved_lines = 0 + skipped_reports = 0 + for report in reports: + try: + FNSReportService.save_report( + **report.to_save_report_kwargs(batch_id=batch_id) + ) + except FNSReportOrganizationResolutionSkipped as exc: + skipped_reports += 1 + logger.warning( + "FNS API report %s skipped: %s", + report.external_id, + exc.reason, + ) + continue + saved_reports += 1 + saved_lines += len(report.lines) + return saved_reports, saved_lines, skipped_reports + + +@shared_task( + bind=True, + soft_time_limit=INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS, + time_limit=INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS, +) +def sync_fns_financial_reports( + self, + *, + inn: str | None = None, + organization_id: int | str | None = None, + registry_organization_limit: int | None = None, + max_periods: int | None = None, + proxies: list[str] | None = None, + requested_by_id: int | None = None, +) -> dict: + """Синхронизировать ГИР БО точечно, не сканируя общую выдачу ФНС.""" + normalized_inn = "" if inn is None else str(inn).strip() + normalized_organization_id = ( + "" if organization_id is None else str(organization_id).strip() + ) + if normalized_inn and normalized_organization_id: + raise ValueError("Provide either inn or organization_id, not both") + + registry_limit = _resolve_fns_api_registry_limit(registry_organization_limit) + resolved_max_periods = _resolve_fns_api_max_periods(max_periods) + resolved_proxies = _resolve_proxies(proxies) + if normalized_organization_id: + sync_mode = "organization_id" + elif normalized_inn: + sync_mode = "inn" + else: + sync_mode = "active_registry" + + source = ParserLoadLog.Source.FNS_REPORTS + task_name = "apps.parsers.tasks.sync_fns_financial_reports" + load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( + source=source, + status="in_progress", + ) + task_id = self.request.id or str(uuid.uuid4()) + job = _get_or_create_background_job( + task_id=task_id, + task_name=task_name, + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, + meta={ + "source_key": "fns_financial", + "sync_mode": sync_mode, + "max_periods": resolved_max_periods, + }, + ) + job.mark_started() + job.update_progress(0, "Подготовка точечной синхронизации ГИР БО...") + + try: + job.update_progress(15, "Загрузка отчётности выбранных организаций...") + fetch_result = _fetch_fns_api_reports( + normalized_inn=normalized_inn, + normalized_organization_id=normalized_organization_id, + registry_limit=registry_limit, + max_periods=resolved_max_periods, + proxies=resolved_proxies, + ) + job.update_progress( + 70, + f"Сохранение {len(fetch_result.reports)} отчётов ГИР БО...", + ) + saved_reports, saved_lines, skipped_reports = _save_fns_api_reports( + fetch_result.reports, + batch_id=batch_id, + ) + + ParserLoadLogService.update( + load_log, + status="success", + records_count=saved_lines, + ) + result = { + "batch_id": batch_id, + "status": "success", + "sync_mode": sync_mode, + "targets": fetch_result.target_count, + "successful_targets": fetch_result.successful_targets, + "failed_targets": fetch_result.failed_targets, + "fetched_reports": len(fetch_result.reports), + "saved": saved_reports, + "saved_reports": saved_reports, + "saved_lines": saved_lines, + "skipped_reports": skipped_reports, + } + job.complete(result=result) + return result + except Exception as exc: + logger.error("%s failed: %s", task_name, exc, exc_info=True) + ParserLoadLogService.mark_failed(load_log, str(exc)) + job.fail(error=str(exc)) + raise + + @shared_task(bind=True) def scan_fns_directory( self, diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 12f158d..0574db9 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -138,6 +138,7 @@ TASKS_BY_NAME = { "apps.parsers.tasks.parse_unfair_suppliers": tasks.parse_unfair_suppliers, "apps.parsers.tasks.parse_fas_goz_evasion": tasks.parse_fas_goz_evasion, "apps.parsers.tasks.scan_fns_directory": tasks.scan_fns_directory, + "apps.parsers.tasks.sync_fns_financial_reports": (tasks.sync_fns_financial_reports), "apps.parsers.tasks.parse_arbitration_cases": tasks.parse_arbitration_cases, "apps.parsers.tasks.parse_registry_inspections": tasks.parse_registry_inspections, "apps.parsers.tasks.parse_fedresurs_bankruptcy": tasks.parse_fedresurs_bankruptcy, diff --git a/src/core/celery.py b/src/core/celery.py index 7f477b6..6ad336e 100644 --- a/src/core/celery.py +++ b/src/core/celery.py @@ -70,10 +70,6 @@ app.conf.beat_schedule = { "task": "apps.parsers.tasks.parse_industrial_products", "schedule": 7 * 24 * 60 * 60, # Every 7 days }, - "parse-inspections-weekly": { - "task": "apps.parsers.tasks.parse_inspections", - "schedule": 7 * 24 * 60 * 60, # Every 7 days - }, "sync-ru-proxies-hourly": { "task": "apps.parsers.tasks.sync_ru_proxies", "schedule": getattr(settings, "PROXY_TOOLS_SYNC_INTERVAL_SECONDS", 3600), diff --git a/src/organizations/cache.py b/src/organizations/cache.py index 5023bb9..9532f1a 100644 --- a/src/organizations/cache.py +++ b/src/organizations/cache.py @@ -7,6 +7,7 @@ import time from django.core.cache import cache ORGANIZATION_API_CACHE_PREFIX = "api:v2:organizations" +ORGANIZATION_API_CACHE_CONTRACT_VERSION = 2 ORGANIZATION_API_CACHE_VERSION_KEY = f"{ORGANIZATION_API_CACHE_PREFIX}:version" DEFAULT_ORGANIZATION_API_CACHE_VERSION = 1 DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS = 24 * 60 * 60 diff --git a/src/organizations/filters.py b/src/organizations/filters.py index c69d0c3..92ce9d3 100644 --- a/src/organizations/filters.py +++ b/src/organizations/filters.py @@ -45,7 +45,6 @@ class OrganizationFilter(filters.FilterSet): ) registry = filters.CharFilter(method="filter_registry") registry_name = filters.CharFilter(method="filter_registry_name") - has_registry = filters.BooleanFilter(method="filter_has_registry") source_group = filters.CharFilter(method="filter_source_group") has_financial_indicators = filters.BooleanFilter(method="filter_source_presence") @@ -83,7 +82,6 @@ class OrganizationFilter(filters.FilterSet): "identity_status", "registry", "registry_name", - "has_registry", "source_group", ] @@ -93,9 +91,6 @@ class OrganizationFilter(filters.FilterSet): def filter_registry_name(self, queryset, _name, value): return self._filter_by_registry_membership(queryset, registry_name=value) - def filter_has_registry(self, queryset, _name, value): - return self._filter_by_registry_membership(queryset, has_registry=value) - def filter_source_group(self, queryset, _name, value): source_group = SOURCE_FILTER_ALIASES.get(str(value), str(value)) return self._filter_by_source_group(queryset, source_group, True) @@ -123,15 +118,12 @@ class OrganizationFilter(filters.FilterSet): *, registry_id: str | None = None, registry_name: str | None = None, - has_registry: bool = True, ): query = cls._registry_directory_query( registry_id=registry_id, registry_name=registry_name, ) - if has_registry: - return queryset.filter(query) - return queryset.exclude(query) + return queryset.filter(query) @staticmethod def _registry_directory_query( @@ -139,11 +131,7 @@ class OrganizationFilter(filters.FilterSet): registry_id: str | None = None, registry_name: str | None = None, ) -> Q: - query = ( - Q(opk_registry_membership=True) - | Q(goz_participation=True) - | ~Q(ropk_num="") - ) + query = Q(opk_registry_membership=True) if registry_id: query &= Q(ropk_razdel_num=str(registry_id)) | Q(gk_code=str(registry_id)) if registry_name: @@ -153,20 +141,3 @@ class OrganizationFilter(filters.FilterSet): | Q(business_activity__icontains=registry_name) ) return query - - @staticmethod - def _registry_identity_value_querysets( - *, - registry_id: str | None = None, - registry_name: str | None = None, - ): - organizations = Organization.objects.filter( - OrganizationFilter._registry_directory_query( - registry_id=registry_id, - registry_name=registry_name, - ) - ) - return ( - organizations.exclude(inn="").values_list("inn", flat=True), - organizations.exclude(ogrn="").values_list("ogrn", flat=True), - ) diff --git a/src/organizations/management/commands/create_test_companies.py b/src/organizations/management/commands/create_test_companies.py new file mode 100644 index 0000000..bff9dba --- /dev/null +++ b/src/organizations/management/commands/create_test_companies.py @@ -0,0 +1,29 @@ +"""Create or refresh the fixed frontend test-company dataset.""" + +import json + +from apps.core.management.commands.base import BaseAppCommand + +from organizations.test_companies import TestCompanyDatasetService + + +class Command(BaseAppCommand): + """Create twenty deterministic organizations with every source dataset.""" + + help = "Создает или обновляет 20 тестовых компаний со всеми наборами данных" + use_transaction = True + + def execute_command(self, *args, **options) -> str: + result = TestCompanyDatasetService.create_or_update() + rendered = json.dumps( + { + "organizations_created": result.organizations_created, + "organizations_updated": result.organizations_updated, + "extensions": result.extensions, + "records": result.records, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/organizations/management/commands/delete_test_companies.py b/src/organizations/management/commands/delete_test_companies.py new file mode 100644 index 0000000..44f3ccd --- /dev/null +++ b/src/organizations/management/commands/delete_test_companies.py @@ -0,0 +1,24 @@ +"""Delete the fixed frontend test-company dataset.""" + +import json + +from apps.core.management.commands.base import BaseAppCommand + +from organizations.test_companies import TestCompanyDatasetService + + +class Command(BaseAppCommand): + """Delete only the twenty deterministic test organizations.""" + + help = "Удаляет 20 фиксированных тестовых компаний и связанные данные" + use_transaction = True + + def execute_command(self, *args, **options) -> str: + result = TestCompanyDatasetService.delete() + rendered = json.dumps( + {"organizations_deleted": result.organizations_deleted}, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/organizations/test_companies.py b/src/organizations/test_companies.py new file mode 100644 index 0000000..1fea671 --- /dev/null +++ b/src/organizations/test_companies.py @@ -0,0 +1,539 @@ +"""Deterministic, realistic test-company dataset for frontend demonstrations.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import date +from decimal import Decimal +from hashlib import sha256 +from uuid import UUID, uuid5 + +from apps.parsers.models import ParserLoadLog +from django.db.models import Count, Max, Min +from django.utils import timezone + +from organizations.cache import invalidate_organization_api_cache +from organizations.models import ( + Organization, + OrganizationSourceFinancialLine, + OrganizationSourceRecord, + SourceExtensionStatus, +) +from organizations.source_cache import invalidate_source_data_cache +from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS + +TEST_COMPANY_COUNT = 20 +TEST_COMPANY_NAMESPACE = UUID("59b36ae9-bcf8-4b7c-b77a-77578f485a01") +TEST_RECORD_PREFIX = "mostovik-test-company" + + +@dataclass(frozen=True) +class TestCompanyDatasetResult: + """Counters returned by create and delete operations.""" + + organizations_created: int = 0 + organizations_updated: int = 0 + organizations_deleted: int = 0 + extensions: int = 0 + records: int = 0 + + +class TestCompanyDatasetService: + """Create, refresh, and remove the fixed frontend demonstration dataset.""" + + @classmethod + def company_uids(cls) -> list[UUID]: + return [ + uuid5(TEST_COMPANY_NAMESPACE, f"company:{index}") + for index in range(1, TEST_COMPANY_COUNT + 1) + ] + + @classmethod + def create_or_update(cls) -> TestCompanyDatasetResult: + now = timezone.now() + created_count = 0 + updated_count = 0 + total_extensions = 0 + total_records = 0 + + descriptors_by_group = cls._descriptors_by_group() + for index, uid in enumerate(cls.company_uids(), start=1): + organization, created = Organization.objects.update_or_create( + uid=uid, + defaults=cls._organization_defaults(index=index, now=now), + ) + created_count += int(created) + updated_count += int(not created) + + extensions = {} + for source_group, descriptors in descriptors_by_group.items(): + descriptor = descriptors[0] + extension, _ = descriptor.extension_model.objects.update_or_create( + organization=organization, + defaults={ + "title": descriptor.title, + "status": SourceExtensionStatus.ACTIVE, + "first_seen_at": now, + "last_seen_at": now, + "last_load_batch": 9_000_000 + index, + "metadata": { + "dataset": TEST_RECORD_PREFIX, + "sources": [item.source for item in descriptors], + }, + }, + ) + extensions[source_group] = extension + + expected_external_ids = [] + for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items(): + if source not in ParserLoadLog.Source.values: + continue + external_id = f"{TEST_RECORD_PREFIX}:{index:02d}:{source}" + expected_external_ids.append(external_id) + record_defaults = cls._record_defaults( + source=source, + index=index, + organization=organization, + ) + record, _ = OrganizationSourceRecord.objects.update_or_create( + source=source, + external_id=external_id, + defaults={ + "extension": extensions[descriptor.source_group], + "record_type": descriptor.record_type, + "load_batch": 9_000_000 + index, + **record_defaults, + }, + ) + if source == ParserLoadLog.Source.FNS_REPORTS: + cls._refresh_financial_lines(record=record, index=index) + + OrganizationSourceRecord.objects.filter( + extension__organization=organization, + external_id__startswith=f"{TEST_RECORD_PREFIX}:", + ).exclude(external_id__in=expected_external_ids).delete() + cls._refresh_extension_counters(organization=organization) + total_extensions += len(extensions) + total_records += len(expected_external_ids) + + cls._invalidate_caches() + return TestCompanyDatasetResult( + organizations_created=created_count, + organizations_updated=updated_count, + extensions=total_extensions, + records=total_records, + ) + + @classmethod + def delete(cls) -> TestCompanyDatasetResult: + company_uids = cls.company_uids() + queryset = Organization.objects.filter(uid__in=company_uids) + deleted_count = queryset.count() + extension_models = { + descriptor.extension_model + for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items() + if source in ParserLoadLog.Source.values + } + for extension_model in extension_models: + extension_model.objects.filter(organization_id__in=company_uids).delete() + queryset.delete() + cls._invalidate_caches() + return TestCompanyDatasetResult(organizations_deleted=deleted_count) + + @staticmethod + def _descriptors_by_group(): + descriptors = {} + for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items(): + if source in ParserLoadLog.Source.values: + descriptors.setdefault(descriptor.source_group, []).append(descriptor) + return descriptors + + @classmethod + def _organization_defaults(cls, *, index: int, now): + is_rosatom = index <= 10 + corporation_name = ( + 'Госкорпорация "Росатом"' if is_rosatom else 'Госкорпорация "Роскосмос"' + ) + corporation_ministry = ( + 'Государственная корпорация по атомной энергии "Росатом"' + if is_rosatom + else 'Государственная корпорация по космической деятельности "Роскосмос"' + ) + industry = ( + "Атомная промышленность" + if is_rosatom + else "Ракетно-космическая промышленность" + ) + name = f"Тестовая компания {index}" + inn = cls._legal_entity_inn(index) + ogrn = cls._legal_entity_ogrn(index) + return { + "rn": 9_900_000 + index, + "gk_code": "2" if is_rosatom else "1", + "gk_name": corporation_name, + "in_korp_code": "1", + "in_korp_name": "Входит в состав", + "name": name, + "full_name": f'Акционерное общество "{name}"', + "short_name": name, + "pn_name": name, + "pn_name_en": f"Test Company {index}", + "inn": inn, + "kpp": f"7709{index:02d}001", + "ogrn": ogrn, + "okpo": f"90{index:06d}", + "filial": ".F.", + "is_branch": False, + "registration_date": date(2010 + index % 10, (index - 1) % 12 + 1, 15), + "create_date": str(2010 + index % 10), + "organizational_legal_form": "12267", + "organizational_legal_form1": "Непубличные акционерные общества", + "ownership_form": "61" if is_rosatom else "16", + "ownership_form1": ( + "Собственность государственных корпораций" + if is_rosatom + else "Частная собственность" + ), + "authorized_capital": Decimal(1_000_000 + index * 100_000), + "legal_address": ( + f"{101000 + index}, г. Москва, Тестовый проезд, д. {index}" + ), + "business_act_cod": "4" if is_rosatom else "2", + "business_activity": "Прочая" if is_rosatom else "Производственная", + "general_director": f"Иванов Иван Иванович {index}", + "general_director_tax_id": f"770100{index:06d}", + "appointment_date": date(2024, 1, min(index, 28)), + "akc_fs": "0", + "akc_sf": "0", + "re_za": False, + "re_zasf": False, + "goz_participation": True, + "opk_registry_membership": True, + "ropk_num": f"ТЕСТ-ОПК-{index:04d}", + "ropk_razdel_num": "3" if is_rosatom else "2", + "ropk_razdel_name": ( + "Организации, подведомственные и находящиеся в сфере деятельности " + f"{corporation_name}" + ), + "min": corporation_ministry, + "dep": "Департамент тестовых данных", + "otr": industry, + "integrated_structure": f"Тестовая интегрированная структура {index}", + "state_sector_code": "2", + "state_sector_name": ( + "контролируются РФ косвенно или совместно с организациями госсектора" + ), + "directory_source_file_hash": sha256( + f"{TEST_RECORD_PREFIX}:{index}".encode() + ).hexdigest(), + "directory_source_row_number": index, + "directory_imported_at": now, + } + + @classmethod + def _record_defaults(cls, *, source: str, index: int, organization): + common = { + "inn": organization.inn, + "ogrn": organization.ogrn, + "organisation_name": organization.name, + "source": source, + "load_batch": 9_000_000 + index, + } + url = f"https://example.test/{source}/{index}" + values = { + ParserLoadLog.Source.FNS_REPORTS: { + "title": f"Бухгалтерская отчетность за 2025 год — {organization.name}", + "record_date": "2025", + "status": "processed", + "payload": { + **common, + "file_name": f"fin_test_{organization.ogrn}.xlsx", + "file_hash": sha256( + f"financial:{organization.ogrn}".encode() + ).hexdigest(), + "lines_count": 4, + }, + }, + ParserLoadLog.Source.PROCUREMENTS: cls._procurement_values( + common, index, organization, law="44-ФЗ/223-ФЗ", source=source + ), + ParserLoadLog.Source.PROCUREMENTS_44FZ: cls._procurement_values( + common, index, organization, law="44-ФЗ", source=source + ), + ParserLoadLog.Source.PROCUREMENTS_223FZ: cls._procurement_values( + common, index, organization, law="223-ФЗ", source=source + ), + ParserLoadLog.Source.CONTRACTS: { + "title": f"Контракт на поставку оборудования № {index}", + "record_date": "15.03.2026", + "amount": Decimal(3_000_000 + index * 25_000), + "status": "Исполнение", + "url": url, + "payload": { + **common, + "contract_number": f"КОНТРАКТ-{index:05d}", + "subject": "Поставка испытательного оборудования", + "customer": organization.name, + "price": str(3_000_000 + index * 25_000), + "law": "44-ФЗ", + "status": "Исполнение", + "url": url, + }, + }, + ParserLoadLog.Source.INDUSTRIAL: { + "title": f"Сертификат промышленного производства {index:05d}/26", + "record_date": "15.01.2026", + "url": url, + "payload": { + **common, + "certificate_number": f"{index:05d}/26", + "issue_date": "15.01.2026", + "issue_date_normalized": "2026-01-15", + "expiry_date": "15.01.2029", + "expiry_date_normalized": "2029-01-15", + "certificate_file_url": url, + }, + }, + ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: { + "title": f"Испытательный комплекс ТК-{index}", + "payload": { + **common, + "full_organisation_name": organization.full_name, + "registry_number": f"ПРОД-{index:06d}", + "product_name": f"Испытательный комплекс ТК-{index}", + "product_model": f"ТК-{index}.2026", + "okpd2_code": "28.99.39.190", + "tnved_code": "8479899707", + "regulatory_document": "ТУ 28.99.39-001-2026", + }, + }, + ParserLoadLog.Source.MANUFACTURES: { + "title": organization.full_name, + "payload": { + **common, + "full_legal_name": organization.full_name, + "address": organization.legal_address, + }, + }, + ParserLoadLog.Source.INSPECTIONS: { + "title": f"Плановая выездная проверка {organization.name}", + "record_date": "01.06.2026", + "status": "Запланирована", + "payload": { + **common, + "registration_number": f"24800000{index:04d}", + "control_authority": "Ростехнадзор", + "inspection_type": "scheduled", + "inspection_form": "documentary_and_on_site", + "legal_basis": "Федеральный закон № 248-ФЗ", + "start_date": "01.06.2026", + "start_date_normalized": "2026-06-01", + "end_date": "15.06.2026", + "end_date_normalized": "2026-06-15", + "data_year": 2026, + "data_month": 6, + "status": "Запланирована", + }, + }, + ParserLoadLog.Source.FEDRESURS_BANKRUPTCY: { + "title": f"Сообщение по делу А40-{10000 + index}/2026", + "record_date": "01.05.2026", + "status": "Наблюдение", + "url": url, + "payload": { + **common, + "case_number": f"А40-{10000 + index}/2026", + "message_type": "Введение наблюдения", + "message_date": "01.05.2026", + "messages_count": 3, + "messages": [ + { + "type": "Введение наблюдения", + "date": "01.05.2026", + } + ], + "url": url, + }, + }, + ParserLoadLog.Source.UNFAIR_SUPPLIERS: { + "title": f"Реестровая запись РНП-{index:05d}", + "record_date": "20.01.2026", + "status": "Включен", + "url": url, + "payload": { + **common, + "supplier": organization.name, + "registry_number": f"РНП-{index:05d}", + "included_at": "20.01.2026", + "reason": "Нарушение условий закупки", + "url": url, + }, + }, + ParserLoadLog.Source.FAS_GOZ: { + "title": f"Постановление ФАС по ГОЗ № {index}", + "record_date": "20.01.2026", + "status": "Исполнено", + "payload": { + **common, + "registry_number": str(index), + "authority": "ФАС России", + "decision": "Постановление по ГОЗ", + "decision_date": "20.01.2026", + "execution_status": "Исполнено", + }, + }, + ParserLoadLog.Source.ARBITRATION: { + "title": f"Дело А40-{20000 + index}/2026", + "record_date": "10.02.2026", + "amount": Decimal(500_000 + index * 10_000), + "status": "В производстве", + "url": url, + "payload": { + **common, + "case_number": f"А40-{20000 + index}/2026", + "court": "Арбитражный суд города Москвы", + "role": "ответчик", + "claim_amount": str(500_000 + index * 10_000), + "status": "В производстве", + "url": url, + }, + }, + ParserLoadLog.Source.FSTEC: { + "title": f"Сертификат ФСТЭК № ТЕСТ-{index:04d}", + "record_date": "15.01.2026", + "status": "Действует", + "payload": { + **common, + "licensee": organization.name, + "registry_number": f"ТЕСТ-{index:04d}", + "issued_at": "15.01.2026", + "status": "Действует", + "№ сертификата": f"ТЕСТ-{index:04d}", + "Заявитель": organization.name, + "Дата внесения в реестр": "15.01.2026", + "Срок действия сертификата": "15.01.2029", + }, + }, + ParserLoadLog.Source.TRUDVSEM: { + "title": f"Инженер-испытатель {index} категории", + "record_date": "20.05.2026", + "status": "Открыта", + "url": url, + "payload": { + **common, + "id": f"VAC-{index:05d}", + "company": organization.name, + "job-name": f"Инженер-испытатель {index} категории", + "creation-date": "2026-05-20", + "salary": "120000.00", + "salary_min": 110000, + "salary_max": 140000, + "currency": "RUB", + "employment": "Полная занятость", + "schedule": "Полный день", + "region": "Москва", + "status": "Открыта", + "vacancy_source": "trudvsem", + "vac_url": url, + }, + }, + } + return values[source] + + @staticmethod + def _procurement_values(common, index, organization, *, law: str, source: str): + purchase_number = f"03732000000{index:08d}"[:19] + amount = Decimal(1_250_000 + index * 50_000) + url = f"https://example.test/{source}/{index}" + return { + "title": f"Поставка оборудования для {organization.name}", + "record_date": "01.02.2026", + "amount": amount, + "status": "Размещено", + "url": url, + "payload": { + **common, + "purchase_number": purchase_number, + "subject": "Поставка испытательного оборудования", + "customer": organization.name, + "customer_inn": organization.inn, + "price": str(amount), + "law": law, + "published_at": "01.02.2026", + "status": "Размещено", + "region_code": "77", + "data_year": 2026, + "data_month": 2, + "url": url, + }, + } + + @staticmethod + def _refresh_financial_lines(*, record, index: int) -> None: + expected = { + ("1", "1600", "Баланс (актив)", 10_000_000 + index * 100_000), + ("1", "1300", "Капитал и резервы", 4_000_000 + index * 50_000), + ("2", "2110", "Выручка", 20_000_000 + index * 200_000), + ("2", "2400", "Чистая прибыль", 2_000_000 + index * 25_000), + } + expected_keys = [] + for form_code, line_code, line_name, period_end in expected: + expected_keys.append((form_code, line_code, 2025)) + OrganizationSourceFinancialLine.objects.update_or_create( + source_record=record, + form_code=form_code, + line_code=line_code, + year=2025, + defaults={ + "line_name": line_name, + "period_start": period_end - 100_000, + "period_end": period_end, + }, + ) + lines = OrganizationSourceFinancialLine.objects.filter(source_record=record) + for line in lines: + key = (line.form_code, line.line_code, line.year) + if key not in expected_keys: + line.delete() + + @staticmethod + def _refresh_extension_counters(*, organization) -> None: + aggregates = { + item["extension_id"]: item + for item in OrganizationSourceRecord.objects.filter( + extension__organization=organization + ) + .values("extension_id") + .annotate( + records_count=Count("uid"), + first_seen_at=Min("created_at"), + last_seen_at=Max("updated_at"), + ) + } + for extension in organization.source_extensions.all(): + aggregate = aggregates.get(extension.uid, {}) + extension.records_count = aggregate.get("records_count", 0) + extension.first_seen_at = aggregate.get("first_seen_at") + extension.last_seen_at = aggregate.get("last_seen_at") + extension.save( + update_fields=["records_count", "first_seen_at", "last_seen_at"] + ) + + @staticmethod + def _legal_entity_inn(index: int) -> str: + base = f"770900{index:03d}" + weights = (2, 4, 10, 3, 5, 9, 4, 6, 8) + checksum = sum( + int(digit) * weight for digit, weight in zip(base, weights, strict=True) + ) + return f"{base}{checksum % 11 % 10}" + + @staticmethod + def _legal_entity_ogrn(index: int) -> str: + base = f"1267700{index:05d}" + return f"{base}{int(base) % 11 % 10}" + + @staticmethod + def _invalidate_caches() -> None: + invalidate_organization_api_cache() + invalidate_source_data_cache() diff --git a/src/organizations/views.py b/src/organizations/views.py index 101b2ee..851a978 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -29,6 +29,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from organizations.cache import ( DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS, + ORGANIZATION_API_CACHE_CONTRACT_VERSION, ORGANIZATION_API_CACHE_PREFIX, get_organization_api_cache_version, invalidate_organization_api_cache, @@ -56,7 +57,6 @@ from organizations.serializers import ( from organizations.source_record_export import build_source_records_export_archive ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations") -FALSE_QUERY_VALUES = {"0", "false", "no", "off"} def _query_parameter( @@ -80,10 +80,6 @@ def _query_parameter( ) -def _is_truthy_query_value(value: str) -> bool: - return value.strip().lower() not in FALSE_QUERY_VALUES - - SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup] ORGANIZATION_LIST_PARAMS = [ @@ -129,15 +125,6 @@ ORGANIZATION_LIST_PARAMS = [ "registry_name", description="Фильтр по части наименования реестра.", ), - _query_parameter( - "has_registry", - description=( - "Фильтр наличия активного участия в любом реестре; по умолчанию true " - "для list endpoint, если параметр не передан." - ), - param_type=openapi.TYPE_BOOLEAN, - default=True, - ), _query_parameter( "source_group", description="Фильтр по группе источников организации.", @@ -180,11 +167,6 @@ SOURCE_RECORD_LIST_PARAMS = [ ), _query_parameter("source", description="Фильтр по legacy source внутри группы."), _query_parameter("record_type", description="Фильтр по типу записи."), - _query_parameter( - "has_registry", - description="Фильтр наличия активного участия организации записи в любом реестре.", - param_type=openapi.TYPE_BOOLEAN, - ), _query_parameter( "organization", description="UID организации.", @@ -228,6 +210,7 @@ class CachedReadOnlyMixin: cache_version = get_organization_api_cache_version() raw_key = ( + f"c{ORGANIZATION_API_CACHE_CONTRACT_VERSION}:" f"v{cache_version}:{request.method}:" f"{request.get_full_path()}:{user_marker}" ) @@ -287,18 +270,12 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): return super().get_permissions() def get_queryset(self): - queryset = super().get_queryset().prefetch_related("source_extensions") - if self.action != "list" or "has_registry" in self.request.query_params: - return queryset - - filterset = OrganizationFilter( - data={"has_registry": "true"}, - queryset=queryset, - request=self.request, + return ( + super() + .get_queryset() + .filter(opk_registry_membership=True) + .prefetch_related("source_extensions") ) - if filterset.is_valid(): - return filterset.qs - return queryset @swagger_auto_schema( tags=[ORGANIZATIONS_TAG], @@ -306,8 +283,8 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): operation_summary="Список организаций", operation_description=( "Возвращает канонический справочник организаций API v2. " - "По умолчанию показывает только организации с активным участием " - "в реестрах; передайте has_registry=false, чтобы снять это ограничение. " + "Контур frontend всегда ограничен организациями, для которых " + "opk_registry_membership=true; снять это ограничение query-параметром нельзя. " "Данные источников возвращаются компактным списком sources; детальные " "записи доступны через endpoints расширений источников." ), @@ -435,11 +412,13 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): class OrganizationSourceExtensionViewSet(ReadOnlyModelViewSet): """Read-only API for source extensions and their records.""" - queryset = OrganizationSourceExtension.objects.select_related( - "organization" - ).order_by( - "organization__name", - "source_group", + queryset = ( + OrganizationSourceExtension.objects.select_related("organization") + .filter(organization__opk_registry_membership=True) + .order_by( + "organization__name", + "source_group", + ) ) serializer_class = OrganizationSourceExtensionSerializer permission_classes = [IsAuthenticated] @@ -488,6 +467,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): "extension", "extension__organization", ) + .filter(extension__organization__opk_registry_membership=True) .prefetch_related("financial_lines") .order_by("-created_at", "-uid") ) @@ -542,7 +522,6 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): source = params.get("source") record_type = params.get("record_type") organization = params.get("organization") - has_registry = params.get("has_registry") search_terms = SearchFilter().get_search_terms(self.request) if source_group: @@ -553,30 +532,11 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): queryset = queryset.filter(record_type=record_type) if organization: queryset = queryset.filter(extension__organization_id=organization) - if has_registry is not None: - registry_query = self._registry_membership_query() - if _is_truthy_query_value(has_registry): - queryset = queryset.filter(registry_query) - else: - queryset = queryset.exclude(registry_query) if search_terms: queryset = self._filter_search_queryset(queryset, search_terms) return queryset - @staticmethod - def _registry_membership_query(): - ( - inn_values, - ogrn_values, - ) = OrganizationFilter._registry_identity_value_querysets() - - return ( - Q(extension__organization__inn__in=inn_values) - | Q(extension__organization__ogrn__in=ogrn_values) - | Q(extension__organization__ogrip__in=ogrn_values) - ) - @classmethod def _filter_search_queryset(cls, queryset, search_terms: list[str]): queryset = queryset.annotate( diff --git a/src/settings/base.py b/src/settings/base.py index c21761a..2535626 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -198,6 +198,13 @@ TEMPLATES = [ WSGI_APPLICATION = "core.wsgi.application" +# Registry archives can legitimately be close to the parser-level 200 MiB limit. +# Keep Django's request-body limit slightly higher to account for multipart framing; +# uploaded file content itself is still validated by ParserUploadRequestSerializer. +DATA_UPLOAD_MAX_MEMORY_SIZE = int( + os.getenv("DATA_UPLOAD_MAX_MEMORY_SIZE", str(210 * 1024 * 1024)) +) + # Database и Cache определяются в dev.py / production.py # ============================================================================= @@ -206,6 +213,7 @@ WSGI_APPLICATION = "core.wsgi.application" ZAKUPKI_TOKEN = os.getenv("ZAKUPKI_TOKEN", "") SUPERJOB_APP_ID = os.getenv("SUPERJOB_APP_ID", "").strip() +HH_USER_AGENT = os.getenv("HH_USER_AGENT", "").strip() FNS_LOCK_TTL_SECONDS = 3600 PARSER_PROXIES = [ item.strip() for item in os.getenv("PARSER_PROXIES", "").split(",") if item.strip() diff --git a/tests/apps/core/test_celery_module.py b/tests/apps/core/test_celery_module.py index 23eebf3..2d6b274 100644 --- a/tests/apps/core/test_celery_module.py +++ b/tests/apps/core/test_celery_module.py @@ -51,7 +51,7 @@ class CeleryModuleTest(SimpleTestCase): ) self.assertIn("parse-manufactures-daily", module.app.conf.beat_schedule) self.assertIn("parse-industrial-products-daily", module.app.conf.beat_schedule) - self.assertIn("parse-inspections-weekly", module.app.conf.beat_schedule) + self.assertNotIn("parse-inspections-weekly", module.app.conf.beat_schedule) self.assertIn("sync-ru-proxies-hourly", module.app.conf.beat_schedule) def test_startup_refresh_queues_when_lock_acquired(self): diff --git a/tests/apps/organizations/test_api_v2.py b/tests/apps/organizations/test_api_v2.py index 8b70985..6682fd7 100644 --- a/tests/apps/organizations/test_api_v2.py +++ b/tests/apps/organizations/test_api_v2.py @@ -31,6 +31,12 @@ from rest_framework.test import APITestCase from tests.apps.user.factories import UserFactory +def create_frontend_organization(**kwargs): + """Create an organization visible through the frontend API by default.""" + kwargs.setdefault("opk_registry_membership", True) + return Organization.objects.create(**kwargs) + + class OrganizationsApiV2Test(APITestCase): """Checks v2 canonical organizations endpoints.""" @@ -51,13 +57,13 @@ class OrganizationsApiV2Test(APITestCase): worksheet.append([key, value]) def test_list_is_paginated_and_available_only_under_v2(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Альфа"', inn="7707083893", kpp="770701001", ogrn="1027700132195", ) - Organization.objects.create( + create_frontend_organization( name="ИП Иванов Иван Иванович", inn="500100732259", ogrip="304500116000157", @@ -174,16 +180,13 @@ class OrganizationsApiV2Test(APITestCase): "search", "ordering", "registry", - "has_registry", "identity_status", "source_group", "has_financial_indicators", "has_planned_inspections", ): self.assertIn(expected_name, list_parameters) - self.assertIn( - "по умолчанию true", list_parameters["has_registry"]["description"] - ) + self.assertNotIn("has_registry", list_parameters) self.assertIn( "группе источников", list_parameters["source_group"]["description"], @@ -206,7 +209,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertIn("/api/v2/organization-sources/{uid}/records/", paths) def test_retrieve_returns_item_by_uid(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Ромашка"', inn="7712345678", kpp="771201001", @@ -224,7 +227,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["name"], 'ООО "Ромашка"') def test_response_includes_normalized_name(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='АКЦИОНЕРНОЕ ОБЩЕСТВО "СЕВЕРНЫЙ МОСТ"', inn="7712345679", kpp="771201002", @@ -242,7 +245,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["normalized_name"], 'АО "Северный Мост"') def test_list_returns_compact_source_summaries_without_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Легкий список"', inn="7712345682", kpp="771201005", @@ -279,7 +282,7 @@ class OrganizationsApiV2Test(APITestCase): ) def test_list_source_summaries_do_not_load_source_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Легкие источники"', inn="7712345685", kpp="771201008", @@ -316,7 +319,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(source_record_queries, []) def test_source_records_endpoint_returns_requested_source_payload(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Явные данные"', full_name='Общество с ограниченной ответственностью "Явные данные"', short_name='ООО "Явные данные"', @@ -387,7 +390,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(embedded_organization["is_branch"], False) def test_detail_returns_compact_source_summaries_by_default(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Полная карточка"', inn="7712345684", kpp="771201007", @@ -410,13 +413,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["sources"][0]["source_group"], "financial_indicators") def test_source_group_filter_limits_organizations_by_source_extension(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Сводка данных"', inn="7712345681", kpp="771201004", ogrn="1027700132199", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Без промышленности"', inn="7712345686", kpp="771201009", @@ -446,7 +449,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["data"][0]["uid"], str(organization.uid)) def test_normalized_name_compacts_scientific_production_forms(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='Научно-Производственное Предприятие "ИМПУЛЬС"', inn="7712345680", kpp="771201003", @@ -463,13 +466,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["normalized_name"], 'НПП "Импульс"') def test_list_filters_by_identifiers_and_searches_by_name(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Северный мост"', inn="7711111111", kpp="771101001", ogrn="1027700132111", ) - Organization.objects.create( + create_frontend_organization( name='АО "Южный путь"', inn="7722222222", kpp="772201001", @@ -494,13 +497,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(by_search.data["data"][0]["uid"], str(target.uid)) def test_list_searches_by_identifier_fragments_for_dashboard(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Поиск по реквизитам"', inn="7711111122", kpp="771102222", ogrn="1027700132122", ) - Organization.objects.create( + create_frontend_organization( name='ООО "Другая организация"', inn="7722222233", kpp="772203333", @@ -520,7 +523,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["data"][0]["uid"], str(target.uid)) def test_list_response_is_cached_by_query_string(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Кеш"', inn="7733333333", kpp="773301001", @@ -532,7 +535,7 @@ class OrganizationsApiV2Test(APITestCase): url, {"inn": "7733333333", "has_registry": "false"}, ) - Organization.objects.create( + create_frontend_organization( name='ООО "После кеша"', inn="7744444444", kpp="774401001", @@ -555,7 +558,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(different_query_response.data["data"][0]["inn"], "7744444444") def test_list_response_cache_is_invalidated_by_version_bump(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Версия кеша"', inn="7744444445", kpp="774401002", @@ -575,7 +578,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(third_response["X-Cache"], "MISS") def test_source_update_invalidates_organization_cache(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Источник сброса"', inn="7744444446", kpp="774401003", @@ -599,7 +602,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(third_response["X-Cache"], "MISS") def test_explicit_invalidation_refreshes_organization_cache(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Реестр сброса"', inn="7744444447", kpp="774401004", @@ -618,7 +621,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(third_response["X-Cache"], "MISS") def test_retrieve_response_is_cached(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Деталь"', inn="7755555555", kpp="775501001", @@ -639,7 +642,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(second_response.data["name"], 'ООО "Деталь"') def test_list_includes_source_groups_and_active_registries(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Данные"', inn="7777777777", kpp="777701001", @@ -783,8 +786,8 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(records_response.data["data"][0]["external_id"], "fin-1") self.assertEqual(len(records_response.data["data"][0]["financial_lines"]), 3) - def test_filters_by_registry_and_has_registry(self): - with_registry = Organization.objects.create( + def test_frontend_scope_cannot_be_lifted_by_has_registry_parameter(self): + with_registry = create_frontend_organization( name='ООО "В реестре"', inn="7788888888", kpp="778801001", @@ -793,11 +796,32 @@ class OrganizationsApiV2Test(APITestCase): ropk_razdel_num="opk-space", ropk_razdel_name="Роскосмос ОПК", ) - without_registry = Organization.objects.create( + without_registry = create_frontend_organization( name='ООО "Без реестра"', inn="7799999999", kpp="779901001", ogrn="1027700132999", + opk_registry_membership=False, + ) + rosatom_without_opk = create_frontend_organization( + name='АО "Росатом без ОПК"', + inn="7799999998", + kpp="779801001", + ogrn="1027700132998", + gk_code="rosatom", + gk_name="Росатом", + goz_participation=True, + opk_registry_membership=False, + ) + roscosmos_without_opk = create_frontend_organization( + name='АО "Роскосмос без ОПК"', + inn="7799999997", + kpp="779701001", + ogrn="1027700132997", + gk_code="roscosmos", + gk_name="Роскосмос", + ropk_num="historical-number-without-membership", + opk_registry_membership=False, ) by_registry = self.client.get( reverse("api_v2:organizations:organizations-list"), @@ -820,22 +844,31 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(has_registry.data["data"][0]["uid"], str(with_registry.uid)) self.assertEqual(no_registry.data["meta"]["pagination"]["total_count"], 1) - self.assertEqual(no_registry.data["data"][0]["uid"], str(without_registry.uid)) - - def test_has_registry_filter_uses_uncorrelated_identity_subqueries(self): - filterset = OrganizationFilter( - data={"has_registry": "true"}, - queryset=Organization.objects.all(), + self.assertEqual(no_registry.data["data"][0]["uid"], str(with_registry.uid)) + returned_uids = {item["uid"] for item in no_registry.data["data"]} + self.assertTrue( + returned_uids.isdisjoint( + { + str(without_registry.uid), + str(rosatom_without_opk.uid), + str(roscosmos_without_opk.uid), + } + ) ) - self.assertTrue(filterset.is_valid(), filterset.errors) - sql = str(filterset.qs.query).upper() + hidden_detail = self.client.get( + reverse( + "api_v2:organizations:organizations-detail", + args=[rosatom_without_opk.uid], + ) + ) + self.assertEqual(hidden_detail.status_code, status.HTTP_404_NOT_FOUND) - self.assertIn("OPK_REGISTRY_MEMBERSHIP", sql) - self.assertNotIn("EXISTS", sql) + def test_has_registry_is_not_a_public_filter(self): + self.assertNotIn("has_registry", OrganizationFilter.base_filters) def test_list_defaults_to_has_registry_true(self): - with_registry = Organization.objects.create( + with_registry = create_frontend_organization( name='ООО "Дефолтный реестр"', inn="7800000101", kpp="780001101", @@ -844,11 +877,21 @@ class OrganizationsApiV2Test(APITestCase): ropk_razdel_num="default", ropk_razdel_name="Дефолтный реестр", ) - Organization.objects.create( + create_frontend_organization( name='ООО "Скрыто по дефолту"', inn="7800000102", kpp="780001102", ogrn="1027700133102", + opk_registry_membership=False, + ) + create_frontend_organization( + name='АО "Росатом без ОПК скрыт по дефолту"', + inn="7800000103", + kpp="780001103", + ogrn="1027700133103", + gk_name="Росатом", + goz_participation=True, + opk_registry_membership=False, ) default_response = self.client.get( reverse("api_v2:organizations:organizations-list") @@ -869,24 +912,24 @@ class OrganizationsApiV2Test(APITestCase): 1, ) self.assertEqual( - explicit_false_response.data["data"][0]["inn"], - "7800000102", + {item["inn"] for item in explicit_false_response.data["data"]}, + {"7800000101"}, ) def test_filters_by_data_presence(self): - with_industrial = Organization.objects.create( + with_industrial = create_frontend_organization( name='ООО "С промышленностью"', inn="7800000001", kpp="780001001", ogrn="1027700133001", ) - with_fns = Organization.objects.create( + with_fns = create_frontend_organization( name='ООО "С отчетностью"', inn="7800000002", kpp="780001002", ogrn="1027700133002", ) - without_data = Organization.objects.create( + without_data = create_frontend_organization( name='ООО "Без данных"', inn="7800000003", kpp="780001003", @@ -937,7 +980,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(has_fns.data["data"][0]["uid"], str(with_fns.uid)) def test_has_fns_reports_filter_does_not_preload_report_identities(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Отчетность без preload"', inn="7800000004", kpp="780001004", @@ -966,13 +1009,13 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(distinct_report_queries, []) def test_sources_action_returns_only_current_organization_extensions(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Источник"', inn="7800000201", kpp="780002101", ogrn="1027700133201", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Другой источник"', inn="7800000205", kpp="780002105", @@ -1010,7 +1053,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(sources["industrial_production"]["uid"], str(industrial.uid)) def test_unknown_source_group_filter_returns_empty_page(self): - Organization.objects.create( + create_frontend_organization( name='ООО "Неверная группа"', inn="7800000202", kpp="780002102", @@ -1026,7 +1069,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(response.data["meta"]["pagination"]["total_count"], 0) def test_detail_source_summaries_do_not_load_source_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Без N+1"', inn="7800000203", kpp="780002103", @@ -1084,7 +1127,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(source_record_queries, []) def test_trudvsem_filter_alias_uses_vacancies_source_group(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Вакансии"', inn="7800000204", kpp="780002104", @@ -1128,7 +1171,7 @@ class OrganizationsApiV2Test(APITestCase): @override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True) def test_dev_flag_allows_anonymous_access(self): self.client.force_authenticate(user=None) - Organization.objects.create( + create_frontend_organization( name='ООО "Без токена"', inn="7766666666", kpp="776601001", diff --git a/tests/apps/organizations/test_api_v2_source_extensions.py b/tests/apps/organizations/test_api_v2_source_extensions.py index 97ed2ab..6adb849 100644 --- a/tests/apps/organizations/test_api_v2_source_extensions.py +++ b/tests/apps/organizations/test_api_v2_source_extensions.py @@ -13,6 +13,12 @@ from rest_framework.test import APITestCase from tests.apps.user.factories import UserFactory +def create_frontend_organization(**kwargs): + """Create an organization visible through the frontend API by default.""" + kwargs.setdefault("opk_registry_membership", True) + return Organization.objects.create(**kwargs) + + class OrganizationSourceExtensionsApiV2Test(APITestCase): """Checks organization-centric source extension API contract.""" @@ -22,7 +28,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.client.force_authenticate(self.user) def test_list_returns_compact_source_summaries_instead_of_embedded_data(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "API"', inn="7707083810", ogrn="1027700132010", @@ -56,7 +62,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(item["sources"][0]["records_count"], 1) def test_organization_sources_action_returns_extensions(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Sources"', inn="7707083811", ogrn="1027700132011", @@ -77,7 +83,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(response.data[0]["uid"], str(extension.uid)) def test_source_records_endpoint_returns_extension_records(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Records"', inn="7707083812", ogrn="1027700132012", @@ -110,12 +116,12 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): ) def test_flat_source_records_endpoint_filters_by_source_group(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Flat Records"', inn="7707083813", ogrn="1027700132013", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Other Records"', inn="7707083814", ogrn="1027700132014", @@ -160,18 +166,19 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(record["source_group"], "planned_inspections") self.assertEqual(record["organization"]["uid"], str(target.uid)) - def test_flat_source_records_endpoint_filters_by_has_registry(self): - with_registry = Organization.objects.create( + def test_flat_source_records_scope_cannot_be_lifted_by_has_registry(self): + with_registry = create_frontend_organization( name='ООО "With Registry Source"', inn="7707083815", ogrn="1027700132015", opk_registry_membership=True, ropk_razdel_num="source-records", ) - without_registry = Organization.objects.create( + without_registry = create_frontend_organization( name='ООО "Without Registry Source"', inn="7707083816", ogrn="1027700132016", + opk_registry_membership=False, ) with_registry_extension = PlannedInspectionExtension.objects.create( organization=with_registry, @@ -225,11 +232,11 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): ) self.assertEqual( without_registry_response.data["data"][0]["external_id"], - "INSP-WITHOUT-REGISTRY", + "INSP-WITH-REGISTRY", ) def test_source_record_organization_uses_active_registry_identity(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Реестровое имя"', inn="1800020960", kpp="180001001", @@ -266,7 +273,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(response_organization["ogrn"], organization.ogrn) def test_source_record_organization_keeps_canonical_inn_with_leading_zero(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='ООО "Башнефть-Добыча"', inn="0277106840", ogrn="1090280032699", @@ -298,12 +305,12 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(response_organization["ogrn"], organization.ogrn) def test_flat_source_records_searches_payload_values_displayed_in_tables(self): - target = Organization.objects.create( + target = create_frontend_organization( name='ООО "Поиск по payload"', inn="7707083817", ogrn="1027700132017", ) - other = Organization.objects.create( + other = create_frontend_organization( name='ООО "Другая payload"', inn="7707083818", ogrn="1027700132018", @@ -363,7 +370,7 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): ) def test_flat_source_records_searches_registry_identity_displayed_in_tables(self): - organization = Organization.objects.create( + organization = create_frontend_organization( name='АО "Реестровый поиск"', inn="1800020961", kpp="180001002", diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py new file mode 100644 index 0000000..65d525e --- /dev/null +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -0,0 +1,168 @@ +"""Tests for deterministic frontend demo company management commands.""" + +from io import StringIO + +from apps.exchange.state_corp_services import StateCorpExchangeService +from apps.parsers.models import ParserLoadLog +from django.core.management import call_command +from django.test import TestCase, override_settings +from organizations.models import ( + Organization, + OrganizationSourceExtension, + OrganizationSourceFinancialLine, + OrganizationSourceRecord, + SourceGroup, +) + +TEST_EXCHANGE_TOKEN = "test-exchange-token" # noqa: S105 + + +class TestCompaniesCommandsTest(TestCase): + """Checks creation, refresh, and removal of the fixed demo dataset.""" + + def test_create_builds_twenty_companies_with_every_source_dataset(self): + call_command("create_test_companies", stdout=StringIO()) + + companies = Organization.objects.filter(name__startswith="Тестовая компания ") + self.assertEqual(companies.count(), 20) + self.assertEqual( + set(companies.values_list("name", flat=True)), + {f"Тестовая компания {index}" for index in range(1, 21)}, + ) + self.assertEqual(companies.filter(opk_registry_membership=True).count(), 20) + self.assertEqual( + companies.filter( + gk_code="2", + gk_name='Госкорпорация "Росатом"', + ).count(), + 10, + ) + self.assertEqual( + companies.filter( + gk_code="1", + gk_name='Госкорпорация "Роскосмос"', + ).count(), + 10, + ) + + expected_groups = {choice.value for choice in SourceGroup} + expected_sources = {choice.value for choice in ParserLoadLog.Source} + for company in companies: + self.assertEqual( + set(company.source_extensions.values_list("source_group", flat=True)), + expected_groups, + ) + self.assertEqual( + set( + OrganizationSourceRecord.objects.filter( + extension__organization=company + ).values_list("source", flat=True) + ), + expected_sources, + ) + + self.assertEqual( + OrganizationSourceExtension.objects.filter( + organization__in=companies + ).count(), + 20 * len(expected_groups), + ) + self.assertEqual( + OrganizationSourceRecord.objects.filter( + extension__organization__in=companies + ).count(), + 20 * len(expected_sources), + ) + self.assertEqual( + OrganizationSourceFinancialLine.objects.filter( + source_record__extension__organization__in=companies + ).count(), + 20 * 4, + ) + + def test_create_updates_the_fixed_dataset_without_duplicates(self): + call_command("create_test_companies", stdout=StringIO()) + company_uids = set( + Organization.objects.filter( + name__startswith="Тестовая компания " + ).values_list("uid", flat=True) + ) + first_company = Organization.objects.get(name="Тестовая компания 1") + first_company.name = "Поврежденное тестовое имя" + first_company.gk_name = "Неверная корпорация" + first_company.save(update_fields=["name", "gk_name"]) + record = OrganizationSourceRecord.objects.filter( + extension__organization=first_company, + source=ParserLoadLog.Source.INSPECTIONS, + ).get() + record.title = "Устаревшие тестовые данные" + record.payload = {"stale": True} + record.save(update_fields=["title", "payload"]) + + call_command("create_test_companies", stdout=StringIO()) + + refreshed_uids = set( + Organization.objects.filter( + name__startswith="Тестовая компания " + ).values_list("uid", flat=True) + ) + self.assertEqual(refreshed_uids, company_uids) + self.assertEqual(len(refreshed_uids), 20) + first_company.refresh_from_db() + self.assertEqual(first_company.name, "Тестовая компания 1") + self.assertEqual(first_company.gk_name, 'Госкорпорация "Росатом"') + record.refresh_from_db() + self.assertNotEqual(record.title, "Устаревшие тестовые данные") + self.assertIn("registration_number", record.payload) + self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15) + + @override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN) + def test_created_source_records_are_exported_to_state_corp_package(self): + call_command("create_test_companies", stdout=StringIO()) + + company_inns = list( + Organization.objects.filter(name__startswith="Тестовая компания ") + .order_by("rn") + .values_list("inn", flat=True) + ) + package = StateCorpExchangeService.build_package( + organization_inns=company_inns + ) + + self.assertEqual( + package.payload_counts, + { + "organizations": 20, + "industrial_certificates": 20, + "manufacturers": 20, + "industrial_products": 20, + "prosecutor_checks": 20, + "public_procurements": 80, + "financial_reports": 20, + "arbitration_cases": 20, + "bankruptcy_procedures": 20, + "defense_unreliable_suppliers": 40, + "information_security_registries": 20, + "labor_vacancies": 20, + }, + ) + + def test_delete_removes_only_the_fixed_twenty_companies(self): + untouched = Organization.objects.create( + name="Тестовая компания 99", + inn="7700000099", + ogrn="1027700000099", + opk_registry_membership=True, + ) + call_command("create_test_companies", stdout=StringIO()) + + call_command("delete_test_companies", stdout=StringIO()) + + self.assertFalse( + Organization.objects.filter( + name__in=[f"Тестовая компания {index}" for index in range(1, 21)] + ).exists() + ) + self.assertTrue(Organization.objects.filter(uid=untouched.uid).exists()) + self.assertEqual(OrganizationSourceExtension.objects.count(), 0) + self.assertEqual(OrganizationSourceRecord.objects.count(), 0) diff --git a/tests/apps/parsers/test_checko.py b/tests/apps/parsers/test_checko.py index 11f7038..46f9f48 100644 --- a/tests/apps/parsers/test_checko.py +++ b/tests/apps/parsers/test_checko.py @@ -493,6 +493,41 @@ class CheckoClientApiTest(SimpleTestCase): ) ) + def test_http_403_quota_error_keeps_checko_rate_limit_classification(self): + with TestHTTPServer() as server: + server.add_json( + "/v2/company", + { + "meta": { + "status": "error", + "message": ( + "Превышен суточный лимит запросов для бесплатного тарифа" + ), + "balance": 0, + "today_request_count": 100, + } + }, + status=403, + ) + client = _client_for(server) + with self.assertRaises(CheckoRateLimitError) as context: + client.get_company(CompanyRequest(inn=_digits(10))) + + self.assertEqual(context.exception.status_code, 403) + self.assertEqual(context.exception.balance, 0) + self.assertEqual(context.exception.request_count, 100) + + def test_non_quota_http_error_remains_connection_error(self): + with TestHTTPServer() as server: + server.add_json( + "/v2/company", + {"meta": {"status": "error", "message": "Доступ запрещён"}}, + status=403, + ) + client = _client_for(server) + with self.assertRaises(CheckoConnectionError): + client.get_company(CompanyRequest(inn=_digits(10))) + def test_not_found_error_handling(self): with TestHTTPServer() as server: server.add_json( diff --git a/tests/apps/parsers/test_fns_api_client.py b/tests/apps/parsers/test_fns_api_client.py new file mode 100644 index 0000000..5af5020 --- /dev/null +++ b/tests/apps/parsers/test_fns_api_client.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from urllib.parse import parse_qs + +from apps.parsers.clients.fns import FNSApiClient, FNSApiClientError +from django.test import SimpleTestCase + +from tests.utils import Response +from tests.utils import TestHTTPServer as HTTPTestServer + + +class FNSApiClientTest(SimpleTestCase): + def test_fetch_reports_by_inn_uses_exact_search_and_maps_financial_lines(self): + search_queries: list[dict[str, list[str]]] = [] + + def search(_request, _body: bytes) -> Response: + search_queries.append(parse_qs(_request.query)) + return Response( + body=( + b'{"content":[' + b'{"id":1,"inn":"7700000000","ogrn":"1027700000000"},' + b'{"id":42,"inn":"7701000001",' + b'"ogrn":"1027700000001"}]}' + ), + headers={"Content-Type": "application/json"}, + ) + + bfo = [ + { + "period": "2025", + "actualBfoDate": "2026-03-10", + "typeCorrections": [ + { + "type": 12, + "correction": { + "id": 555, + "datePresent": "2026-03-10T12:00:00", + "balance": { + "id": 555, + "okud": "0710001", + "current1600": 120, + "previous1600": 100, + "beforePrevious1600": 90, + }, + "financialResult": { + "id": 555, + "okud": "0710002", + "current2110": 300, + "previous2110": 250, + "current2400": 50, + "previous2400": 40, + }, + }, + } + ], + }, + { + "period": "2024", + "typeCorrections": [], + }, + ] + + with HTTPTestServer() as server: + server.add_route("GET", "/advanced-search/organizations/search", search) + server.add_json( + "/nbo/organizations/42", + { + "id": 42, + "inn": "7701000001", + "ogrn": "1027700000001", + "fullName": "АО Завод", + }, + ) + server.add_json("/nbo/organizations/42/bfo/", bfo) + with FNSApiClient( + base_url=server.base_url, + max_periods=1, + http_adapter=server.adapter, + ) as client: + self.assertIn( + "Mozilla/5.0", client.http_client.session.headers["User-Agent"] + ) + reports = client.fetch_reports_by_inn("7701000001") + + self.assertEqual( + search_queries, + [{"query": ["7701000001"], "page": ["0"], "size": ["10"]}], + ) + self.assertEqual(len(reports), 1) + report = reports[0] + self.assertEqual(report.external_id, "girbo:555") + self.assertEqual(report.ogrn, "1027700000001") + self.assertEqual(report.year, 2025) + self.assertEqual(report.period_type, 12) + self.assertEqual( + [ + ( + line.form_code, + line.line_code, + line.period_start, + line.period_end, + ) + for line in report.lines + ], + [ + ("1", "1600", 100, 120), + ("2", "2110", 250, 300), + ("2", "2400", 40, 50), + ], + ) + kwargs = report.to_save_report_kwargs(batch_id=7) + self.assertEqual(kwargs["source"], "api") + self.assertEqual(kwargs["batch_id"], 7) + self.assertEqual(kwargs["file_hash"], report.file_hash) + self.assertEqual(len(kwargs["lines_data"]), 3) + + def test_fetch_reports_by_organization_id_does_not_run_search(self): + with HTTPTestServer() as server: + server.add_json( + "/nbo/organizations/42", + { + "id": 42, + "inn": "7701000001", + "ogrn": "1027700000001", + "fullName": "АО Завод", + }, + ) + server.add_json("/nbo/organizations/42/bfo/", []) + client = FNSApiClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + reports = client.fetch_reports_by_organization_id(42) + + self.assertEqual(reports, []) + + def test_fetch_reports_by_inn_fails_when_exact_match_is_absent(self): + with HTTPTestServer() as server: + server.add_json( + "/advanced-search/organizations/search", + {"content": [{"id": 1, "inn": "7700000000"}]}, + ) + client = FNSApiClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage(FNSApiClientError, "not found"): + client.fetch_reports_by_inn("7701000001") + + def test_rejects_invalid_inn_before_request(self): + client = FNSApiClient() + + with self.assertRaisesMessage(ValueError, "10 or 12 digits"): + client.fetch_reports_by_inn("not-an-inn") diff --git a/tests/apps/parsers/test_gisp_client.py b/tests/apps/parsers/test_gisp_client.py new file mode 100644 index 0000000..0c59d8f --- /dev/null +++ b/tests/apps/parsers/test_gisp_client.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import json + +from apps.parsers.clients.gisp import GispProductsClient, GispProductsClientError +from django.test import SimpleTestCase + +from tests.utils import Response +from tests.utils import TestHTTPServer as HTTPTestServer + + +def _gisp_item(number: int) -> dict: + return { + "_product_reg_number_2023": str(number), + "_product_reg_number_2022": f"old-{number}", + "_res_number": f"resolution-{number}", + "_org_inn": "7701000001", + "_org_ogrn": "1027700000001", + "_org_name": "АО Завод", + "_product_name": f"Станок {number}", + "_product_spec": f"Модель {number}", + "_product_okpd2": "28.41.1", + "_product_tnved": "8456", + "_basedondoc_name": "Заключение", + "_basedondoc_num": "42", + "_basedondoc_date": "2026-01-15", + } + + +class GispProductsClientTest(SimpleTestCase): + def test_rejects_page_size_above_safe_bound(self): + with self.assertRaisesMessage( + ValueError, + "page_size must be between 1 and 100", + ): + GispProductsClient(page_size=101) + + def test_rejects_max_pages_above_safe_bound(self): + with self.assertRaisesMessage( + ValueError, + "max_pages must be between 1 and 100", + ): + GispProductsClient(max_pages=101) + + def test_rejects_max_records_above_safe_bound(self): + with self.assertRaisesMessage( + ValueError, + "max_records must be between 1 and 10000", + ): + GispProductsClient(max_records=10_001) + + def test_fetch_products_follows_skip_pagination_and_maps_items(self): + requests: list[dict] = [] + + def handle_page(_request, body: bytes) -> Response: + payload = json.loads(body) + requests.append(payload) + options = payload["opt"] + skip = options["skip"] + take = options["take"] + items = [_gisp_item(number) for number in range(skip + 1, 6)][:take] + return Response( + body=json.dumps( + {"ok": True, "total_count": 5, "items": items} + ).encode(), + headers={"Content-Type": "application/json"}, + ) + + with HTTPTestServer() as server: + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_page) + with GispProductsClient( + base_url=server.base_url, + page_size=2, + max_pages=10, + max_records=5, + http_adapter=server.adapter, + ) as client: + result = client.fetch_products() + + self.assertEqual( + [payload["opt"] for payload in requests], + [ + {"skip": 0, "take": 2, "requireTotalCount": True}, + {"skip": 2, "take": 2, "requireTotalCount": True}, + {"skip": 4, "take": 1, "requireTotalCount": True}, + ], + ) + self.assertEqual(result.total_count, 5) + self.assertEqual(result.pages_fetched, 3) + self.assertEqual(result.items_fetched, 5) + self.assertFalse(result.truncated) + self.assertEqual(len(result.products), 5) + self.assertEqual(result.products[0].registry_number, "1") + self.assertEqual(result.products[0].inn, "7701000001") + self.assertEqual(result.products[0].product_name, "Станок 1") + self.assertEqual( + result.products[0].regulatory_document, + "Заключение | 42 | 2026-01-15", + ) + + def test_fetch_products_stops_at_max_pages_and_reports_truncation(self): + calls = 0 + + def handle_page(_request, body: bytes) -> Response: + nonlocal calls + calls += 1 + options = json.loads(body)["opt"] + items = [ + _gisp_item(number) + for number in range(options["skip"] + 1, options["skip"] + 3) + ] + return Response( + body=json.dumps( + {"ok": True, "total_count": 100, "items": items} + ).encode(), + headers={"Content-Type": "application/json"}, + ) + + with HTTPTestServer() as server: + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_page) + with GispProductsClient( + base_url=server.base_url, + page_size=2, + max_pages=2, + max_records=100, + http_adapter=server.adapter, + ) as client: + result = client.fetch_products() + + self.assertEqual(calls, 2) + self.assertEqual(result.items_fetched, 4) + self.assertTrue(result.truncated) + + def test_fetch_products_rejects_invalid_response_shape(self): + with HTTPTestServer() as server: + server.add_route( + "POST", + "/pp719v2/pub/prod/b/", + lambda _request, _body: Response( + body=b'{"ok":true,"total_count":1}', + headers={"Content-Type": "application/json"}, + ), + ) + client = GispProductsClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage( + GispProductsClientError, + "returned no items list", + ): + client.fetch_products() + + def test_fetch_products_rejects_empty_page_before_known_total(self): + with HTTPTestServer() as server: + server.add_route( + "POST", + "/pp719v2/pub/prod/b/", + lambda _request, _body: Response( + body=b'{"ok":true,"total_count":10,"items":[]}', + headers={"Content-Type": "application/json"}, + ), + ) + client = GispProductsClient( + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage( + GispProductsClientError, + "empty page before total_count", + ): + client.fetch_products() diff --git a/tests/apps/parsers/test_parser_schedules.py b/tests/apps/parsers/test_parser_schedules.py new file mode 100644 index 0000000..2f23ee5 --- /dev/null +++ b/tests/apps/parsers/test_parser_schedules.py @@ -0,0 +1,104 @@ +import importlib + +from django.apps import apps +from django.test import TestCase +from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask + +schedule_migration = importlib.import_module( + "apps.parsers.migrations.0025_consolidate_inspections_schedule" +) + + +class DefaultParserScheduleTest(TestCase): + def test_inspections_have_one_canonical_sync_schedule(self): + crontab = CrontabSchedule.objects.create( + minute="0", + hour="0", + day_of_week="6", + day_of_month="*", + month_of_year="*", + timezone="Europe/Moscow", + ) + interval = IntervalSchedule.objects.create( + every=7, + period=IntervalSchedule.DAYS, + ) + PeriodicTask.objects.create( + name="parser:inspections:weekly-saturday-msk", + task="apps.parsers.tasks.parse_inspections", + crontab=crontab, + enabled=True, + ) + PeriodicTask.objects.create( + name="parse-inspections-weekly", + task="apps.parsers.tasks.parse_inspections", + interval=interval, + enabled=True, + ) + PeriodicTask.objects.create( + name="parser:sync_inspections:weekly-saturday-msk", + task="apps.parsers.tasks.parse_inspections", + args='["stale"]', + kwargs='{"year": 2020}', + crontab=crontab, + enabled=True, + ) + + schedule_migration.consolidate_inspections_schedule(apps, None) + schedule_migration.consolidate_inspections_schedule(apps, None) + + schedule_names = { + "parser:inspections:weekly-saturday-msk", + "parser:sync_inspections:weekly-saturday-msk", + "parse-inspections-weekly", + } + + schedules = list( + PeriodicTask.objects.filter(name__in=schedule_names) + .order_by("name") + .values_list("name", "task", "enabled", "args", "kwargs") + ) + + self.assertEqual( + schedules, + [ + ( + "parser:sync_inspections:weekly-saturday-msk", + "apps.parsers.tasks.sync_inspections", + True, + "[]", + "{}", + ) + ], + ) + + def test_fns_weekly_schedule_runs_bounded_api_sync(self): + fns_schedule_migration = importlib.import_module( + "apps.parsers.migrations.0026_update_fns_financial_schedule" + ) + interval = IntervalSchedule.objects.create( + every=7, + period=IntervalSchedule.DAYS, + ) + PeriodicTask.objects.create( + name="parser:fns_financial:weekly-saturday-msk", + task="apps.parsers.tasks.scan_fns_directory", + args='["stale"]', + kwargs='{"stale": true}', + interval=interval, + enabled=True, + ) + + fns_schedule_migration.update_fns_financial_schedule(apps, None) + fns_schedule_migration.update_fns_financial_schedule(apps, None) + + task = PeriodicTask.objects.get(name="parser:fns_financial:weekly-saturday-msk") + self.assertEqual( + task.task, + "apps.parsers.tasks.sync_fns_financial_reports", + ) + self.assertEqual(task.args, "[]") + self.assertEqual(task.kwargs, "{}") + self.assertTrue(task.enabled) + self.assertIsNotNone(task.crontab_id) + self.assertIsNone(task.interval_id) diff --git a/tests/apps/parsers/test_proverki_client.py b/tests/apps/parsers/test_proverki_client.py index a8b6aae..2bd8a65 100644 --- a/tests/apps/parsers/test_proverki_client.py +++ b/tests/apps/parsers/test_proverki_client.py @@ -332,6 +332,25 @@ class ProverkiDownloadParseTest(SimpleTestCase): ) self.assertEqual(len(inspections), 1) + def test_download_and_parse_closes_playwright_when_portal_fails(self): + class _PortalClient(ProverkiClient): + playwright_closed = False + + def _download_from_portal(self, *args, **kwargs): # type: ignore[override] + raise RuntimeError("navigation failed") + + def _close_playwright(self): # type: ignore[override] + self.playwright_closed = True + + client = _PortalClient(use_playwright=True) + + with self.assertRaises(RuntimeError): + client._download_and_parse( + "http://portal.example.com", file_format="portal" + ) + + self.assertTrue(client.playwright_closed) + def test_download_from_portal_does_not_wait_for_networkidle(self): archive = build_zip( [("data.xml", _xml_with_tag("INSPECTION", _inspection_attrs()))] @@ -654,6 +673,63 @@ class ProverkiParseXMLTest(SimpleTestCase): self.assertEqual(inspections[0].inspection_type, inspection_type) self.assertEqual(inspections[0].status, status) + def test_parse_xml_record_fz248_current_production_shape(self): + xml = b""" + + + + + + + + +""" + client = ProverkiClient() + + inspections = client._parse_xml_content(xml) + + self.assertEqual(len(inspections), 1) + inspection = inspections[0] + self.assertEqual(inspection.registration_number, "248000000000") + self.assertEqual(inspection.inn, "7700000000") + self.assertEqual(inspection.ogrn, "1027700000000") + self.assertEqual(inspection.organisation_name, "Example Industrial Company") + self.assertEqual( + inspection.control_authority, + "Example Supervisory Authority", + ) + self.assertEqual(inspection.inspection_type, "Unscheduled inspection") + self.assertEqual(inspection.inspection_form, "On-site inspection") + self.assertEqual(inspection.start_date, "2026-07-01") + self.assertEqual(inspection.end_date, "2026-07-15") + self.assertEqual(inspection.status, "Completed") + + def test_parse_xml_record_fz248_kind_control_is_type_fallback(self): + element = ET.fromstring( # noqa: S314 + """ + + + + """ + ) + client = ProverkiClient() + + inspection = client._parse_xml_record(element) + + self.assertIsNotNone(inspection) + self.assertEqual( + inspection.inspection_type, + "Federal industrial supervision", + ) + def test_parse_xml_record_namespace_text_child_fallback(self): ns = "http://example.com/ns" inn = _digits(10) @@ -1171,3 +1247,42 @@ class ProverkiPlaywrightStubTest(SimpleTestCase): sys.modules.pop("playwright.sync_api", None) else: sys.modules["playwright.sync_api"] = original_module + + def test_get_browser_launch_error_stops_playwright(self): + class _BrokenChromium: + def launch(self, **_kwargs): + raise RuntimeError("browser missing") + + class _FakePlaywright: + chromium = _BrokenChromium() + + def __init__(self): + self.stopped = False + + def stop(self): + self.stopped = True + + playwright = _FakePlaywright() + + class _FakeSyncPlaywright: + def start(self): + return playwright + + fake_module = types.SimpleNamespace( + sync_playwright=lambda: _FakeSyncPlaywright() + ) + client = ProverkiClient() + original_module = sys.modules.get("playwright.sync_api") + sys.modules["playwright.sync_api"] = fake_module + try: + with self.assertRaises(ProverkiClientError): + client._get_browser() + finally: + if original_module is None: + sys.modules.pop("playwright.sync_api", None) + else: + sys.modules["playwright.sync_api"] = original_module + + self.assertTrue(playwright.stopped) + self.assertIsNone(client._playwright) + self.assertIsNone(client._browser) diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index 55d21eb..a6049e8 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -176,6 +176,29 @@ class SourceCardServiceUnitTest(SimpleTestCase): ) self.assertEqual(enqueue_mock.call_count, 3) + @patch( + "apps.parsers.source_cards.SourceCardService._enqueue_task", + return_value={ + "task_id": "task-fns", + "task_name": "apps.parsers.tasks.sync_fns_financial_reports", + }, + ) + def test_refresh_card_for_financial_indicators_uses_api_sync(self, enqueue_mock): + result = SourceCardService.refresh_card( + slug="financial-indicators", + requested_by_id=12, + ) + + self.assertEqual(result["tasks"][0]["task_id"], "task-fns") + self.assertEqual( + enqueue_mock.call_args.kwargs["task_name"], + "apps.parsers.tasks.sync_fns_financial_reports", + ) + self.assertEqual( + enqueue_mock.call_args.kwargs["kwargs"], + {"requested_by_id": 12}, + ) + @patch( "apps.parsers.source_cards.SourceCardService._enqueue_task", return_value={ diff --git a/tests/apps/parsers/test_source_cards_views.py b/tests/apps/parsers/test_source_cards_views.py index da040fa..7b65485 100644 --- a/tests/apps/parsers/test_source_cards_views.py +++ b/tests/apps/parsers/test_source_cards_views.py @@ -3,8 +3,6 @@ from __future__ import annotations import hashlib -from pathlib import Path -from tempfile import TemporaryDirectory from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import ParserLoadLog @@ -236,21 +234,17 @@ class SourceCardsApiTestCase(APITestCase): self.assertIn("status_label", row) self.assertIn("active_tasks", row) + @override_settings(CELERY_TASK_ALWAYS_EAGER=False) def test_refresh_creates_background_job_and_returns_task(self): self.client.force_authenticate(self.admin) - with TemporaryDirectory() as tmp_dir, override_settings( - FNS_WATCH_DIRECTORY=str(Path(tmp_dir) / "watch"), - FNS_PROCESSED_DIRECTORY=str(Path(tmp_dir) / "processed"), - FNS_FAILED_DIRECTORY=str(Path(tmp_dir) / "failed"), - ): - response = self.client.post( - reverse( - "api_v1:sources:source-cards-refresh", - kwargs={"slug": "financial-indicators"}, - ), - {}, - format="json", - ) + response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "financial-indicators"}, + ), + {}, + format="json", + ) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(response.data["status"], "accepted") @@ -260,7 +254,7 @@ class SourceCardsApiTestCase(APITestCase): self.assertTrue( BackgroundJob.objects.filter( task_id=task_id, - task_name="apps.parsers.tasks.scan_fns_directory", + task_name="apps.parsers.tasks.sync_fns_financial_reports", user_id=self.admin.id, ).exists() ) diff --git a/tests/apps/parsers/test_source_registry.py b/tests/apps/parsers/test_source_registry.py new file mode 100644 index 0000000..fd47794 --- /dev/null +++ b/tests/apps/parsers/test_source_registry.py @@ -0,0 +1,42 @@ +from apps.parsers import tasks +from apps.parsers.clients.common.structured import MAX_FILE_SIZE_BYTES +from apps.parsers.source_registry import PARSER_SOURCES +from apps.parsers.views import TASKS_BY_NAME +from django.conf import settings +from django.test import SimpleTestCase + + +class ParserSourceRegistryEisTest(SimpleTestCase): + def test_procurement_urls_select_the_expected_law(self): + common_search_url = ( + "https://zakupki.gov.ru/epz/order/extendedsearch/results.html" + ) + + self.assertEqual( + PARSER_SOURCES["procurements_44fz"].upstream_url, + f"{common_search_url}?fz44=on", + ) + self.assertEqual( + PARSER_SOURCES["procurements_223fz"].upstream_url, + f"{common_search_url}?fz223=on", + ) + + +class ParserSourceRegistryFNSTest(SimpleTestCase): + def test_request_body_limit_allows_maximum_registry_archive(self): + self.assertGreater(settings.DATA_UPLOAD_MAX_MEMORY_SIZE, MAX_FILE_SIZE_BYTES) + + def test_fns_refresh_uses_bounded_api_task_and_preserves_file_upload(self): + source = PARSER_SOURCES["fns_financial"] + + self.assertEqual( + source.task_name, + "apps.parsers.tasks.sync_fns_financial_reports", + ) + self.assertEqual(source.parser_strategy, "fns_bfo_bounded_registry_sync") + self.assertTrue(source.supports_file_upload) + self.assertNotIn("query=", source.upstream_url) + self.assertIs( + TASKS_BY_NAME[source.task_name], + tasks.sync_fns_financial_reports, + ) diff --git a/tests/apps/parsers/test_structured_data_client.py b/tests/apps/parsers/test_structured_data_client.py index 4f506de..6fb65b0 100644 --- a/tests/apps/parsers/test_structured_data_client.py +++ b/tests/apps/parsers/test_structured_data_client.py @@ -1,7 +1,10 @@ from __future__ import annotations from apps.parsers.clients.base import HTTPClientError -from apps.parsers.clients.common.structured import StructuredDataClient +from apps.parsers.clients.common.structured import ( + EIS_CA_BUNDLE_PATH, + StructuredDataClient, +) from django.test import SimpleTestCase @@ -18,7 +21,84 @@ class _FakeHTTPClient: return response +class _FakeGispHTTPClient: + def __init__(self) -> None: + self.endpoint = "" + self.payload: dict | None = None + + def post_json( + self, + endpoint: str, + *, + payload: dict | None = None, + **_kwargs, + ) -> dict: + self.endpoint = endpoint + self.payload = payload + return {"items": [{"res_number": "123", "_product_name": "Станок"}]} + + +class StructuredDataClientGispTest(SimpleTestCase): + def test_products_request_omits_unsupported_res_date_sort(self): + client = StructuredDataClient(source="mpt_products") + fake_http_client = _FakeGispHTTPClient() + client._http_client = fake_http_client + + records = client.fetch_records(file_url="https://gisp.gov.ru/pp719v2/pub/prod/") + + self.assertEqual(len(records), 1) + self.assertEqual(records[0].external_id, "123") + self.assertEqual( + fake_http_client.endpoint, + "https://gisp.gov.ru/pp719v2/pub/prod/b/", + ) + self.assertEqual( + fake_http_client.payload, + { + "opt": { + "skip": 0, + "take": 100, + "requireTotalCount": True, + } + }, + ) + + +class StructuredDataClientFasGozParsingTest(SimpleTestCase): + def test_person_name_populates_record_title(self): + client = StructuredDataClient(source="fas_goz") + + records = client.fetch_records( + content=""" + + + + + + + + + + + + +
РГОЗ-001ФАС РоссииПостановление № 114.07.2026ИсполненоООО ПРОГРЕССПРОГРЕССг. Москва7707083893
+ """.encode(), + file_name="register.html", + ) + + self.assertEqual(len(records), 1) + self.assertEqual(records[0].organisation_name, "ООО ПРОГРЕСС") + self.assertEqual(records[0].title, "ООО ПРОГРЕСС") + + class StructuredDataClientEisCardEnrichmentTest(SimpleTestCase): + def test_eis_client_uses_project_scoped_trusted_ca_bundle(self): + client = StructuredDataClient(source="procurements_44fz") + + self.assertTrue(EIS_CA_BUNDLE_PATH.is_file()) + self.assertEqual(client.http_client.verify_ssl, str(EIS_CA_BUNDLE_PATH)) + def test_procurement_card_enriches_customer_identity_from_detail_page(self): detail_url = ( "https://zakupki.gov.ru/epz/order/notice/ea20/view/" @@ -104,3 +184,170 @@ class StructuredDataClientEisCardEnrichmentTest(SimpleTestCase): self.assertEqual(records[0].inn, "") self.assertEqual(records[0].ogrn, "") self.assertEqual(records[0].organisation_name, "ФЕДЕРАЛЬНОЕ ГБУ НАУКИ") + + def test_procurement_card_prefers_organization_page_for_identity(self): + detail_url = ( + "https://zakupki.gov.ru/epz/order/notice/zk20/view/" + "common-info.html?regNumber=0322300001726000667" + ) + identity_url = ( + "https://zakupki.gov.ru/epz/organization/view/" + "info.html?organizationCode=03223000017" + ) + client = StructuredDataClient(source="procurements_44fz") + client._http_client = _FakeHTTPClient( + { + identity_url: """ +
Заказчик
+
21.07.2016
+
ОГРН
1022701405737
+
ИНН
2725006476
+
КПП
272501001
+
Полное наименование
КГБУЗ БОЛЬНИЦА
+ """.encode() + } + ) + + records = client.fetch_records( + content=f""" +
+
44-ФЗ
+ № 0322300001726000667 +
Объект закупки
Поставка оборудования
+
Заказчик
КГБУЗ БОЛЬНИЦА
+ КГБУЗ БОЛЬНИЦА +
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual(records[0].url, detail_url) + self.assertEqual(records[0].inn, "2725006476") + self.assertEqual(records[0].ogrn, "1022701405737") + self.assertEqual(records[0].organisation_name, "КГБУЗ БОЛЬНИЦА") + self.assertEqual(client._http_client.downloaded_urls, [identity_url]) + + def test_procurement_card_rejects_untrusted_detail_urls(self): + loopback_identity_url = ( + "http://127.0.0.1/epz/organization/view/" "info.html?inn=2725006476" + ) + foreign_detail_url = ( + "https://example.test/epz/order/notice/zk20/view/" + "common-info.html?regNumber=0322300001726000667" + ) + client = StructuredDataClient(source="procurements_44fz") + client._http_client = _FakeHTTPClient( + { + loopback_identity_url: b"", + foreign_detail_url: b"", + } + ) + + records = client.fetch_records( + content=f""" +
+
44-ФЗ
+ № 0322300001726000667 +
Объект закупки
Поставка оборудования
+
Заказчик
КГБУЗ БОЛЬНИЦА
+ КГБУЗ БОЛЬНИЦА +
+ """.encode(), + file_name="results.html", + ) + + self.assertNotIn("detail_url", records[0].payload) + self.assertNotIn("identity_url", records[0].payload) + self.assertEqual(client._http_client.downloaded_urls, []) + + def test_procurement_card_resolves_relative_official_urls(self): + detail_href = ( + "/epz/order/notice/zk20/view/" + "common-info.html?regNumber=0322300001726000667" + ) + identity_href = "/epz/organization/view/info.html?organizationCode=03223000017" + client = StructuredDataClient( + source="procurements_44fz", + enrich_eis_detail_pages=False, + ) + + records = client.fetch_records( + content=f""" +
+
44-ФЗ
+ № 0322300001726000667 +
Объект закупки
Поставка оборудования
+
Заказчик
КГБУЗ БОЛЬНИЦА
+ КГБУЗ БОЛЬНИЦА +
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual( + records[0].payload["detail_url"], + f"https://zakupki.gov.ru{detail_href}", + ) + self.assertEqual( + records[0].payload["identity_url"], + f"https://zakupki.gov.ru{identity_href}", + ) + + +class StructuredDataClientEisCardParsingTest(SimpleTestCase): + def test_contract_card_prefers_contract_details_over_order_notice(self): + contract_url = ( + "https://zakupki.gov.ru/epz/contract/contractCard/" + "common-info.html?reestrNumber=3310300251025000007" + ) + notice_url = ( + "https://zakupki.gov.ru/epz/order/notice/view/" + "common-info.html?regNumber=0126300014725000080" + ) + client = StructuredDataClient( + source="contracts", + enrich_eis_detail_pages=False, + ) + + records = client.fetch_records( + content=f""" +
+ № 3310300251025000007 +
Заказчик
МБДОУ СКАЗКА
+
Объекты закупки
Мясо птицы
+
Размещен контракт в реестре контрактов
+
04.12.2025
+ Сведения закупки +
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual(records[0].url, contract_url) + self.assertEqual(records[0].organisation_name, "МБДОУ СКАЗКА") + self.assertEqual(records[0].record_date, "04.12.2025") + + def test_unfair_supplier_skips_empty_placed_label(self): + client = StructuredDataClient( + source="unfair_suppliers", + enrich_eis_detail_pages=False, + ) + + records = client.fetch_records( + content=""" +
+
44-ФЗ
+
№ 26007353
+
Размещено
+
Наименование (ФИО) недобросовестного поставщика
+
ООО ПРОГРЕСС
+
ИНН (аналог ИНН)
9728104322
+
Включено
14.07.2026
+
+ """.encode(), + file_name="results.html", + ) + + self.assertEqual(records[0].title, "ООО ПРОГРЕСС") + self.assertEqual(records[0].organisation_name, "ООО ПРОГРЕСС") + self.assertEqual(records[0].record_date, "14.07.2026") diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 9c773f5..05286de 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -4,10 +4,12 @@ from __future__ import annotations import hashlib import io +import json import os import tempfile import threading import unittest +from contextlib import ExitStack from datetime import date from pathlib import Path from types import SimpleNamespace @@ -17,7 +19,9 @@ from urllib.parse import urlparse from apps.core.services import BackgroundJobService from apps.parsers import tasks as parser_tasks from apps.parsers.clients.base import HTTPError +from apps.parsers.clients.checko import CheckoRateLimitError from apps.parsers.clients.common.schemas import GenericParserItem +from apps.parsers.clients.gisp import GispProductsClient, GispProductsClientError from apps.parsers.clients.minpromtorg.industrial import ( IndustrialProductionClient, IndustrialProductionClientError, @@ -26,10 +30,6 @@ from apps.parsers.clients.minpromtorg.manufactures import ( ManufacturesClient, ManufacturesClientError, ) -from apps.parsers.clients.minpromtorg.products import ( - IndustrialProductsClient, - IndustrialProductsClientError, -) from apps.parsers.clients.proverki.client import ProverkiClientError from apps.parsers.clients.zakupki import ZakupkiClientError from apps.parsers.models import ( @@ -38,6 +38,8 @@ from apps.parsers.models import ( ) from apps.parsers.services import FNSReportService, ParserLoadLogService from apps.parsers.tasks import ( + FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT, + FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT, INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS, INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS, _move_to_dir, @@ -55,11 +57,12 @@ from apps.parsers.tasks import ( parse_trudvsem_vacancies, process_fns_files_batch, scan_fns_directory, + sync_fns_financial_reports, sync_inspections, sync_procurements, sync_ru_proxies, ) -from django.test import TestCase, override_settings +from django.test import SimpleTestCase, TestCase, override_settings from openpyxl import Workbook from organizations.models import Organization, OrganizationSourceRecord @@ -71,7 +74,7 @@ from tests.apps.parsers.factories import ( ProxyFactory, ) from tests.apps.parsers.organization_helpers import create_directory_organization -from tests.utils import TestHTTPServer +from tests.utils import Response, TestHTTPServer from tests.utils.fixtures import ( build_minpromtorg_certificates_excel, build_minpromtorg_manufacturers_excel, @@ -277,6 +280,69 @@ class SyncRuProxiesTaskTestCase(TestCase): sync_mock.assert_called_once_with() +class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase): + @override_settings( + CHECKO_API_KEY="test-key", + FEDRESURS_CHECKO_FALLBACK_LIMIT=100, + ) + def test_rate_limit_stops_remaining_checko_lookups(self): + targets = [ + parser_tasks.RegistryLookupTarget( + organization_id=str(index), + inn=f"77010000{index:02d}", + ogrn=f"10277000000{index:02d}", + name=f"Организация {index}", + ) + for index in range(3) + ] + + class _RateLimitedCheckoClient: + calls = 0 + + def __init__(self, **_kwargs): + return + + def get_company(self, _request): + self.__class__.calls += 1 + raise CheckoRateLimitError( + "Превышен суточный лимит запросов", + status_code=403, + ) + + with ( + patch.object( + parser_tasks, + "_active_registry_lookup_targets", + return_value=targets, + ), + patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient), + ): + records = parser_tasks._fetch_checko_bankruptcy_records(proxies=None) + + self.assertEqual(records, []) + self.assertEqual(_RateLimitedCheckoClient.calls, 1) + + def test_non_http_official_error_is_not_silenced_when_fallback_is_empty(self): + with ( + patch.object( + parser_tasks, + "_fetch_structured_records", + side_effect=ValueError("invalid official payload"), + ), + patch.object( + parser_tasks, + "_fetch_checko_bankruptcy_records", + return_value=[], + ), + self.assertRaisesRegex(ValueError, "invalid official payload"), + ): + parser_tasks._fetch_fedresurs_bankruptcy_records( + file_url=None, + file_path=None, + proxies=None, + ) + + class GenericSourceFetchTestCase(TestCase): """Tests for source-specific generic fetch configuration.""" @@ -561,6 +627,52 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(captured_inns, [str(organization.mn_inn)]) self.assertNotIn(str(no_membership.mn_inn), captured_inns) + def test_fedresurs_falls_back_to_checko_when_official_source_is_empty(self): + fallback_records = [SimpleNamespace(source="fedresurs_bankruptcy")] + + with ( + patch.object( + parser_tasks, + "_fetch_structured_records", + return_value=[], + ), + patch.object( + parser_tasks, + "_fetch_checko_bankruptcy_records", + return_value=fallback_records, + ) as fallback, + ): + records = parser_tasks._fetch_fedresurs_bankruptcy_records( + file_url=None, + file_path=None, + proxies=[], + ) + + self.assertIs(records, fallback_records) + fallback.assert_called_once_with(proxies=[]) + + def test_fedresurs_skips_when_official_source_and_fallback_are_empty(self): + with ( + patch.object( + parser_tasks, + "_fetch_structured_records", + return_value=[], + ), + patch.object( + parser_tasks, + "_fetch_checko_bankruptcy_records", + return_value=[], + ), + ): + result = parser_tasks.parse_fedresurs_bankruptcy(proxies=[]) + + log = ParserLoadLog.objects.get( + source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY + ) + self.assertEqual(result["status"], "skipped") + self.assertEqual(log.status, ParserLoadLog.Status.SKIPPED) + self.assertIn("returned no bankruptcy records", log.error_message) + def test_checko_bankruptcy_items_group_messages_by_procedure(self): company = SimpleNamespace( ogrn="1052452047450", @@ -961,6 +1073,14 @@ class GenericSourceFetchTestCase(TestCase): cleanup_mock.assert_called_once_with(max_age_minutes=45) jobs_cleanup_mock.assert_called_once() self.assertEqual(jobs_cleanup_mock.call_args.kwargs["max_age_minutes"], 45) + self.assertIn( + "apps.parsers.tasks.scan_fns_directory", + jobs_cleanup_mock.call_args.kwargs["task_names"], + ) + self.assertIn( + "apps.parsers.tasks.sync_fns_financial_reports", + jobs_cleanup_mock.call_args.kwargs["task_names"], + ) self.assertEqual(result["status"], "success") self.assertEqual(result["marked_failed"], 2) self.assertEqual(result["marked_jobs_failed"], 3) @@ -992,6 +1112,186 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(job.meta["batch_id"], 7) +class FNSApiSyncTaskTestCase(TestCase): + @staticmethod + def _report(*, external_id: str = "girbo:101", lines_count: int = 2): + lines = [SimpleNamespace() for _ in range(lines_count)] + + def _save_kwargs(*, batch_id: int) -> dict: + return { + "external_id": external_id, + "ogrn": "1027700132195", + "file_name": f"{external_id}.json", + "file_hash": "a" * 64, + "source": "api", + "batch_id": batch_id, + "lines_data": [], + } + + return SimpleNamespace( + external_id=external_id, + lines=lines, + to_save_report_kwargs=_save_kwargs, + ) + + def test_sync_by_explicit_inn_saves_reports_through_fns_service(self): + report = self._report() + with ( + patch.object(parser_tasks, "FNSApiClient") as client_class, + patch.object( + parser_tasks.FNSReportService, + "save_report", + return_value=SimpleNamespace(uid=fake.uuid4()), + ) as save_report, + patch.object(parser_tasks, "_active_fns_registry_inns") as registry_inns, + ): + client = client_class.return_value.__enter__.return_value + client.fetch_reports_by_inn.return_value = [report] + + result = sync_fns_financial_reports( + inn="7707083893", + max_periods=3, + proxies=[], + ) + + client_class.assert_called_once_with(max_periods=3, proxies=[]) + client.fetch_reports_by_inn.assert_called_once_with("7707083893") + registry_inns.assert_not_called() + save_kwargs = save_report.call_args.kwargs + self.assertEqual(save_kwargs["external_id"], report.external_id) + self.assertEqual(save_kwargs["batch_id"], result["batch_id"]) + self.assertEqual(result["saved_reports"], 1) + self.assertEqual(result["saved_lines"], 2) + self.assertEqual(result["sync_mode"], "inn") + + def test_sync_by_explicit_fns_organization_id_bypasses_search(self): + report = self._report(lines_count=1) + with ( + patch.object(parser_tasks, "FNSApiClient") as client_class, + patch.object(parser_tasks.FNSReportService, "save_report"), + patch.object(parser_tasks, "_active_fns_registry_inns") as registry_inns, + ): + client = client_class.return_value.__enter__.return_value + client.fetch_reports_by_organization_id.return_value = [report] + + result = sync_fns_financial_reports( + organization_id=12345, + proxies=[], + ) + + client.fetch_reports_by_organization_id.assert_called_once_with("12345") + client.fetch_reports_by_inn.assert_not_called() + registry_inns.assert_not_called() + self.assertEqual(result["sync_mode"], "organization_id") + self.assertEqual(result["targets"], 1) + + def test_registry_sync_clamps_limit_and_continues_after_target_error(self): + report = self._report(lines_count=1) + with ( + patch.object( + parser_tasks, + "_active_fns_registry_inns", + return_value=["7707083893", "7736050003"], + ) as registry_inns, + patch.object(parser_tasks, "FNSApiClient") as client_class, + patch.object(parser_tasks.FNSReportService, "save_report"), + ): + client = client_class.return_value.__enter__.return_value + client.fetch_reports_by_inn.side_effect = [ + HTTPError("organization unavailable"), + [report], + ] + + result = sync_fns_financial_reports( + registry_organization_limit=2_600_000, + proxies=[], + ) + + registry_inns.assert_called_once_with( + limit=FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT + ) + self.assertEqual(result["targets"], 2) + self.assertEqual(result["successful_targets"], 1) + self.assertEqual(result["failed_targets"], 1) + self.assertEqual(result["saved_reports"], 1) + + def test_registry_sync_uses_bounded_default(self): + with ( + patch.object( + parser_tasks, + "_active_fns_registry_inns", + return_value=[], + ) as registry_inns, + patch.object(parser_tasks, "FNSApiClient") as client_class, + ): + result = sync_fns_financial_reports(proxies=[]) + + registry_inns.assert_called_once_with( + limit=FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT + ) + client_class.assert_not_called() + self.assertEqual(result["targets"], 0) + self.assertEqual(result["saved_reports"], 0) + + def test_rejects_combined_explicit_selectors_before_creating_load(self): + with self.assertRaisesRegex(ValueError, "either inn or organization_id"): + sync_fns_financial_reports( + inn="7707083893", + organization_id=12345, + proxies=[], + ) + + self.assertFalse( + ParserLoadLog.objects.filter( + source=ParserLoadLog.Source.FNS_REPORTS + ).exists() + ) + + def test_zero_organization_id_is_rejected_instead_of_running_registry_sync(self): + with ( + patch.object( + parser_tasks, + "_active_fns_registry_inns", + ) as registry_inns, + self.assertRaisesRegex(ValueError, "positive integer"), + ): + sync_fns_financial_reports(organization_id=0, proxies=[]) + + registry_inns.assert_not_called() + self.assertTrue( + ParserLoadLog.objects.filter( + source=ParserLoadLog.Source.FNS_REPORTS, + status=ParserLoadLog.Status.FAILED, + ).exists() + ) + + def test_active_registry_targets_include_all_registry_signals_and_respect_limit( + self, + ): + OrganizationFactory( + inn="1000000001", + opk_registry_membership=True, + ) + OrganizationFactory( + inn="1000000002", + goz_participation=True, + ) + OrganizationFactory( + inn="1000000003", + ropk_num="РОПК-3", + ) + OrganizationFactory( + inn="1000000000", + opk_registry_membership=False, + goz_participation=False, + ropk_num="", + ) + + targets = parser_tasks._active_fns_registry_inns(limit=2) + + self.assertEqual(targets, ["1000000001", "1000000002"]) + + @override_settings( CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, @@ -1378,7 +1678,7 @@ class MinpromtorgTasksTestCase(TestCase): def _add_minpromtorg_routes(self, server: TestHTTPServer): certificates_bytes, cert_rows = build_minpromtorg_certificates_excel(count=2) manufacturers_bytes, manuf_rows = build_minpromtorg_manufacturers_excel(count=2) - products_bytes, product_rows = build_minpromtorg_products_excel(count=2) + _products_bytes, product_rows = build_minpromtorg_products_excel(count=2) _ensure_directory_organizations_for_rows(cert_rows) _ensure_directory_organizations_for_rows(manuf_rows) _ensure_directory_organizations_for_rows(product_rows) @@ -1388,7 +1688,6 @@ class MinpromtorgTasksTestCase(TestCase): ) cert_file = f"data_resolutions_{date_str}.xlsx" manuf_file = f"data_orgs_{date_str}.xlsx" - products_file = f"industrial_products_{date_str}.xlsx" server.add_json( "/api/kss-document-preview", @@ -1402,15 +1701,6 @@ class MinpromtorgTasksTestCase(TestCase): "name": ManufacturesClient().query, "files": [{"name": manuf_file, "url": f"/files/{manuf_file}"}], }, - { - "name": IndustrialProductsClient().query, - "files": [ - { - "name": products_file, - "url": f"/files/{products_file}", - } - ], - }, ] }, ) @@ -1428,13 +1718,37 @@ class MinpromtorgTasksTestCase(TestCase): "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ), ) - server.add_bytes( - f"/files/{products_file}", - products_bytes, - content_type=( - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - ), - ) + gisp_items = [ + { + "_product_reg_number_2023": row.registry_number, + "_org_inn": row.inn, + "_org_ogrn": row.ogrn, + "_org_name": row.full_organisation_name, + "_product_name": row.product_name, + "_product_spec": row.product_model, + "_product_okpd2": row.okpd2_code, + "_product_tnved": row.tnved_code, + "_basedondoc_name": row.regulatory_document, + } + for row in product_rows + ] + + def handle_gisp_products(_request, body: bytes) -> Response: + options = json.loads(body)["opt"] + skip = options["skip"] + take = options["take"] + return Response( + body=json.dumps( + { + "ok": True, + "total_count": len(gisp_items), + "items": gisp_items[skip : skip + take], + } + ).encode(), + headers={"Content-Type": "application/json"}, + ) + + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_gisp_products) return cert_rows, product_rows, manuf_rows def test_parse_all_minpromtorg_success(self): @@ -1470,11 +1784,24 @@ class MinpromtorgTasksTestCase(TestCase): def test_parse_all_sources_success(self): with TestHTTPServer() as server: cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) - result = parse_all_sources( - proxies=[], - client_adapter=server.adapter, - inspections_use_playwright=False, - ) + with patch.object( + parser_tasks.sync_inspections, + "apply", + return_value=SimpleNamespace(id="inspections-test-task"), + ) as inspections_apply: + result = parse_all_sources( + proxies=[], + client_adapter=server.adapter, + inspections_use_playwright=False, + ) + + inspections_apply.assert_called_once_with( + kwargs={ + "proxies": [], + "client_adapter": server.adapter, + "use_playwright": False, + } + ) self.assertIn("industrial", result) self.assertIn("industrial_products", result) @@ -1505,6 +1832,44 @@ class MinpromtorgTasksTestCase(TestCase): 0, ) + @override_settings(CELERY_TASK_ALWAYS_EAGER=False) + def test_parse_all_sources_queues_fns_api_sync(self): + task_names = ( + "parse_industrial_production", + "parse_industrial_products", + "parse_manufactures", + "sync_inspections", + "parse_procurements_44fz", + "parse_procurements_223fz", + "parse_contracts", + "parse_unfair_suppliers", + "parse_fas_goz_evasion", + "sync_fns_financial_reports", + "parse_arbitration_cases", + "parse_fedresurs_bankruptcy", + "parse_fstec_registers", + "parse_trudvsem_vacancies", + ) + with ExitStack() as stack: + delays = { + task_name: stack.enter_context( + patch.object( + getattr(parser_tasks, task_name), + "delay", + return_value=SimpleNamespace(id=f"{task_name}-id"), + ) + ) + for task_name in task_names + } + + result = parse_all_sources(proxies=[]) + + self.assertEqual( + result["fns_financial"], + "sync_fns_financial_reports-id", + ) + delays["sync_fns_financial_reports"].assert_called_once_with(proxies=[]) + def test_parse_all_minpromtorg_without_adapter(self): with TestHTTPServer() as server: cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) @@ -1514,7 +1879,7 @@ class MinpromtorgTasksTestCase(TestCase): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) - class _LocalIndustrialProductsClient(IndustrialProductsClient): + class _LocalIndustrialProductsClient(GispProductsClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) @@ -1525,7 +1890,7 @@ class MinpromtorgTasksTestCase(TestCase): super().__init__(*args, **kwargs) original_industrial = parser_tasks.IndustrialProductionClient - original_industrial_products = parser_tasks.IndustrialProductsClient + original_industrial_products = parser_tasks.GispProductsClient original_manufactures = parser_tasks.ManufacturesClient original_industrial_delay = parser_tasks.parse_industrial_production.delay original_industrial_products_delay = ( @@ -1533,7 +1898,7 @@ class MinpromtorgTasksTestCase(TestCase): ) original_manufactures_delay = parser_tasks.parse_manufactures.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient - parser_tasks.IndustrialProductsClient = _LocalIndustrialProductsClient + parser_tasks.GispProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): @@ -1563,7 +1928,7 @@ class MinpromtorgTasksTestCase(TestCase): result = parse_all_minpromtorg(proxies=[]) finally: parser_tasks.IndustrialProductionClient = original_industrial - parser_tasks.IndustrialProductsClient = original_industrial_products + parser_tasks.GispProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures parser_tasks.parse_industrial_production.delay = ( original_industrial_delay @@ -1604,7 +1969,7 @@ class MinpromtorgTasksTestCase(TestCase): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) - class _LocalIndustrialProductsClient(IndustrialProductsClient): + class _LocalIndustrialProductsClient(GispProductsClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) @@ -1615,16 +1980,16 @@ class MinpromtorgTasksTestCase(TestCase): super().__init__(*args, **kwargs) original_industrial = parser_tasks.IndustrialProductionClient - original_industrial_products = parser_tasks.IndustrialProductsClient + original_industrial_products = parser_tasks.GispProductsClient original_manufactures = parser_tasks.ManufacturesClient original_industrial_delay = parser_tasks.parse_industrial_production.delay original_industrial_products_delay = ( parser_tasks.parse_industrial_products.delay ) original_manufactures_delay = parser_tasks.parse_manufactures.delay - original_inspections_delay = parser_tasks.parse_inspections.delay + original_sync_inspections_delay = parser_tasks.sync_inspections.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient - parser_tasks.IndustrialProductsClient = _LocalIndustrialProductsClient + parser_tasks.GispProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): @@ -1653,12 +2018,12 @@ class MinpromtorgTasksTestCase(TestCase): _industrial_products_eager_delay ) parser_tasks.parse_manufactures.delay = _manufactures_eager_delay - parser_tasks.parse_inspections.delay = _inspections_stub_delay + parser_tasks.sync_inspections.delay = _inspections_stub_delay try: result = parse_all_sources(proxies=[], inspections_use_playwright=None) finally: parser_tasks.IndustrialProductionClient = original_industrial - parser_tasks.IndustrialProductsClient = original_industrial_products + parser_tasks.GispProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures parser_tasks.parse_industrial_production.delay = ( original_industrial_delay @@ -1667,7 +2032,7 @@ class MinpromtorgTasksTestCase(TestCase): original_industrial_products_delay ) parser_tasks.parse_manufactures.delay = original_manufactures_delay - parser_tasks.parse_inspections.delay = original_inspections_delay + parser_tasks.sync_inspections.delay = original_sync_inspections_delay self.assertIn("industrial", result) self.assertIn("industrial_products", result) @@ -1733,30 +2098,16 @@ class MinpromtorgTasksTestCase(TestCase): ) def test_parse_industrial_products_failure(self): - date_str = fake.date_between(start_date="-30d", end_date="today").strftime( - "%Y%m%d" - ) - products_file = f"industrial_products_{date_str}.xlsx" - with TestHTTPServer() as server: - server.add_json( - "/api/kss-document-preview", - { - "data": [ - { - "name": IndustrialProductsClient().query, - "files": [ - { - "name": products_file, - "url": f"/files/{products_file}", - } - ], - } - ] - }, + server.add_route( + "POST", + "/pp719v2/pub/prod/b/", + lambda _request, _body: Response( + body=b'{"ok":true,"total_count":1}', + headers={"Content-Type": "application/json"}, + ), ) - server.add_bytes("/files/" + products_file, b"not-an-excel") - with self.assertRaises(IndustrialProductsClientError): + with self.assertRaises(GispProductsClientError): parse_industrial_products(client_adapter=server.adapter) def test_parse_industrial_products_with_default_proxies(self): @@ -1765,6 +2116,9 @@ class MinpromtorgTasksTestCase(TestCase): result = parse_industrial_products(client_adapter=server.adapter) self.assertEqual(result["status"], "success") + self.assertEqual(result["fetched"], len(product_rows)) + self.assertEqual(result["total"], len(product_rows)) + self.assertFalse(result["truncated"]) self.assertEqual( OrganizationSourceRecord.objects.filter( source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS @@ -1772,6 +2126,35 @@ class MinpromtorgTasksTestCase(TestCase): len(product_rows), ) + def test_parse_industrial_products_applies_bounded_pagination(self): + with TestHTTPServer() as server: + _cert_rows, _product_rows, _manuf_rows = self._add_minpromtorg_routes( + server + ) + result = parse_industrial_products( + client_adapter=server.adapter, + page_size=1, + max_pages=1, + max_records=1, + ) + + self.assertEqual(result["fetched"], 1) + self.assertEqual(result["total"], 2) + self.assertTrue(result["truncated"]) + self.assertEqual(result["saved"], 1) + self.assertEqual( + OrganizationSourceRecord.objects.filter( + source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS + ).count(), + 1, + ) + + def test_parse_industrial_products_keeps_public_task_name(self): + self.assertEqual( + parse_industrial_products.name, + "apps.parsers.tasks.parse_industrial_products", + ) + def test_parse_manufactures_failure(self): date_str = fake.date_between(start_date="-30d", end_date="today").strftime( "%Y%m%d" @@ -2782,7 +3165,10 @@ class ParseVacanciesTaskTestCase(TestCase): self.assertEqual(result["saved"], 1) self.assertEqual(captured_offsets, [0]) - @override_settings(SUPERJOB_APP_ID="test-superjob-app-id") + @override_settings( + SUPERJOB_APP_ID="test-superjob-app-id", + HH_USER_AGENT="Mostovik/1.0 (ops@example.test)", + ) def test_parse_trudvsem_vacancies_uses_combined_vacancies_client(self): for inn in ("7701000401", "7701000402", "7701000403"): _ensure_directory_organization(inn=inn) @@ -2842,6 +3228,10 @@ class ParseVacanciesTaskTestCase(TestCase): self.assertEqual(result["status"], "success") self.assertEqual(result["saved"], 3) self.assertEqual(captured_kwargs["superjob_app_id"], "test-superjob-app-id") + self.assertEqual( + captured_kwargs["hh_user_agent"], + "Mostovik/1.0 (ops@example.test)", + ) self.assertEqual(captured_kwargs["sources"], ["trudvsem", "hh", "superjob"]) self.assertEqual(captured_fetch_kwargs["limit"], 25) self.assertEqual(captured_fetch_kwargs["offset"], 5) diff --git a/tests/apps/parsers/test_vacancy_clients.py b/tests/apps/parsers/test_vacancy_clients.py index 1124466..2b4672f 100644 --- a/tests/apps/parsers/test_vacancy_clients.py +++ b/tests/apps/parsers/test_vacancy_clients.py @@ -17,6 +17,15 @@ from tests.utils import TestHTTPServer class TrudvsemClientTest(SimpleTestCase): + def test_default_base_url_uses_https_transport(self): + with TestHTTPServer() as server: + server.add_json("/api/v1/vacancies", {"results": {"vacancies": []}}) + with TrudvsemClient(http_adapter=server.adapter) as client: + records = client.fetch_vacancies(limit=1) + + self.assertEqual(records, []) + self.assertEqual(client.base_url, "https://opendata.trudvsem.ru/api/v1") + def test_fetch_vacancies_by_company_inn_uses_company_endpoint(self): with TestHTTPServer() as server: server.add_json( @@ -78,6 +87,7 @@ class HHVacanciesClientTest(SimpleTestCase): }, ) client = HHVacanciesClient( + user_agent="Mostovik/1.0 (ops@example.test)", base_url=server.base_url, http_adapter=server.adapter, ) @@ -93,6 +103,10 @@ class HHVacanciesClientTest(SimpleTestCase): self.assertEqual(record.amount, Decimal("120000")) self.assertEqual(record.url, "https://hh.ru/vacancy/123") self.assertEqual(record.payload["vacancy_source"], "hh") + self.assertEqual( + client.http_client.session.headers["HH-User-Agent"], + "Mostovik/1.0 (ops@example.test)", + ) def test_fetch_vacancies_tolerates_unexpected_nested_shapes(self): with TestHTTPServer() as server: @@ -111,6 +125,7 @@ class HHVacanciesClientTest(SimpleTestCase): }, ) client = HHVacanciesClient( + user_agent="Mostovik/1.0 (ops@example.test)", base_url=server.base_url, http_adapter=server.adapter, ) @@ -122,6 +137,21 @@ class HHVacanciesClientTest(SimpleTestCase): self.assertEqual(records[0].organisation_name, "") self.assertIsNone(records[0].amount) + def test_fetch_vacancies_requires_hh_user_agent_before_request(self): + with TestHTTPServer() as server: + server.add_json("/vacancies", {"items": []}) + client = HHVacanciesClient( + user_agent=" ", + base_url=server.base_url, + http_adapter=server.adapter, + ) + + with self.assertRaisesMessage( + VacanciesClientError, + "HH_USER_AGENT is required", + ): + client.fetch_vacancies(limit=1) + class SuperJobVacanciesClientTest(SimpleTestCase): def test_fetch_vacancies_maps_superjob_objects_to_generic_records(self):