"""Access-gated SRO reads with explicit dev opt-in, pacing and bounded retries.""" from __future__ import annotations import re import time from dataclasses import dataclass from datetime import UTC, datetime from email.utils import parsedate_to_datetime from urllib.parse import parse_qs, urljoin, urlsplit import requests from apps.core.exceptions import ConflictError from apps.parsers.registry_http import limited_bytes from apps.parsers.registry_snapshots import SnapshotValidationError from django.conf import settings def require_sro_access_approved() -> None: if settings.SRO_DEV_COLLECTION_ENABLED: return if not ( settings.SRO_UPSTREAM_ACCESS_APPROVED and settings.SRO_UPSTREAM_APPROVAL_REFERENCE.strip() ): raise ConflictError( message="Автоматизированный доступ к реестру СРО ещё не согласован", code="upstream_access_not_approved", ) def safe_sro_url(value: str, *, base: str | None = None) -> str: value = urljoin(base, value) if base else value try: parts = urlsplit(value) host = (parts.hostname or "").lower() valid = ( parts.scheme == "https" and (host == "reestr-sro.ru" or host.endswith(".reestr-sro.ru")) and not parts.username and not parts.password and parts.port in (None, 443) and not parts.fragment and not any(ord(char) < 33 for char in value) ) except (TypeError, ValueError): valid = False if not valid: raise SnapshotValidationError("unsafe_sro_url") return value def is_sro_lookup_url(value: str) -> bool: """Only the observed exact identifier lookup may carry a negative 404 body.""" try: parts = urlsplit(safe_sro_url(value)) query = parse_qs(parts.query, keep_blank_values=True, max_num_fields=20) except (SnapshotValidationError, ValueError): return False identifiers = query.get("q", []) return ( parts.path == "/proverka_dopuska/" and len(identifiers) == 1 and re.fullmatch(r"[0-9]{10}(?:[0-9]{3})?", identifiers[0]) is not None ) @dataclass(frozen=True) class SroPage: url: str body: bytes status_code: int = 200 class SroHttpClient: """One sequential task owns the global site interval, including redirects.""" def __init__(self, *, session=None, clock=time.monotonic, sleep=time.sleep): require_sro_access_approved() self.session = session or requests.Session() self.owns_session = session is None if self.owns_session: self.session.trust_env = False self.session.headers["User-Agent"] = "Mostovik SRO integration" self.clock, self.sleep = clock, sleep self.last_request = None self.requests_count = 0 self.http_errors_count = 0 def get(self, url: str) -> SroPage: for _ in range(5): url = safe_sro_url(url) response = self._request(url) try: if response.is_redirect: url = safe_sro_url(response.headers.get("Location", ""), base=url) continue if response.status_code == 404: if not is_sro_lookup_url(url): raise SnapshotValidationError("sro_page_not_found") elif response.status_code != 200: raise SnapshotValidationError("sro_upstream_http_error") return SroPage( url, limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES), status_code=response.status_code, ) except requests.RequestException as exc: self.http_errors_count += 1 raise SnapshotValidationError("sro_upstream_transport_error") from exc finally: response.close() raise SnapshotValidationError("sro_redirect_limit") def _pace(self) -> None: require_sro_access_approved() if self.requests_count >= settings.SRO_MAX_REQUESTS_PER_RUN: raise SnapshotValidationError("sro_request_budget_exceeded") interval = max(3.0, settings.SRO_REQUEST_INTERVAL_SECONDS) if self.last_request is not None: self.sleep(max(0.0, interval - (self.clock() - self.last_request))) self.last_request = self.clock() self.requests_count += 1 def _request(self, url: str): for attempt in range(3): self._pace() try: response = self.session.get( url, stream=True, allow_redirects=False, timeout=(10, settings.SRO_HTTP_TIMEOUT_SECONDS), ) except requests.RequestException as exc: self.http_errors_count += 1 if attempt == 2: raise SnapshotValidationError( "sro_upstream_transport_error" ) from exc continue if response.status_code >= 400: self.http_errors_count += 1 if response.status_code not in (429, 500, 502, 503, 504): return response retry_after = response.headers.get("Retry-After", "") response.close() self._retry_pause(retry_after) raise SnapshotValidationError("sro_upstream_unavailable") def _retry_pause(self, value: str) -> None: delay = 3.0 if value.isdigit(): delay = max(delay, int(value)) elif value: try: target = parsedate_to_datetime(value) delay = max(delay, (target - datetime.now(UTC)).total_seconds()) except (TypeError, ValueError): raise SnapshotValidationError("sro_invalid_retry_after") from None if delay > 300: raise SnapshotValidationError("sro_retry_after_exceeds_budget") self.sleep(delay) def close(self) -> None: if self.owns_session: self.session.close()