mostovik-backend/src/apps/parsers/sro_http.py

"""Access-gated SRO reads with explicit dev opt-in, pacing and bounded retries."""

from __future__ import annotations

import re
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from urllib.parse import parse_qs, urljoin, urlsplit

import requests
from apps.core.exceptions import ConflictError
from apps.parsers.registry_http import limited_bytes
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.conf import settings


def require_sro_access_approved() -> None:
    if settings.SRO_DEV_COLLECTION_ENABLED:
        return
    if not (
        settings.SRO_UPSTREAM_ACCESS_APPROVED
        and settings.SRO_UPSTREAM_APPROVAL_REFERENCE.strip()
    ):
        raise ConflictError(
            message="Автоматизированный доступ к реестру СРО ещё не согласован",
            code="upstream_access_not_approved",
        )


def safe_sro_url(value: str, *, base: str | None = None) -> str:
    value = urljoin(base, value) if base else value
    try:
        parts = urlsplit(value)
        host = (parts.hostname or "").lower()
        valid = (
            parts.scheme == "https"
            and (host == "reestr-sro.ru" or host.endswith(".reestr-sro.ru"))
            and not parts.username
            and not parts.password
            and parts.port in (None, 443)
            and not parts.fragment
            and not any(ord(char) < 33 for char in value)
        )
    except (TypeError, ValueError):
        valid = False
    if not valid:
        raise SnapshotValidationError("unsafe_sro_url")
    return value


def is_sro_lookup_url(value: str) -> bool:
    """Only the observed exact identifier lookup may carry a negative 404 body."""
    try:
        parts = urlsplit(safe_sro_url(value))
        query = parse_qs(parts.query, keep_blank_values=True, max_num_fields=20)
    except (SnapshotValidationError, ValueError):
        return False
    identifiers = query.get("q", [])
    return (
        parts.path == "/proverka_dopuska/"
        and len(identifiers) == 1
        and re.fullmatch(r"[0-9]{10}(?:[0-9]{3})?", identifiers[0]) is not None
    )


@dataclass(frozen=True)
class SroPage:
    url: str
    body: bytes
    status_code: int = 200


class SroHttpClient:
    """One sequential task owns the global site interval, including redirects."""

    def __init__(self, *, session=None, clock=time.monotonic, sleep=time.sleep):
        require_sro_access_approved()
        self.session = session or requests.Session()
        self.owns_session = session is None
        if self.owns_session:
            self.session.trust_env = False
            self.session.headers["User-Agent"] = "Mostovik SRO integration"
        self.clock, self.sleep = clock, sleep
        self.last_request = None
        self.requests_count = 0
        self.http_errors_count = 0

    def get(self, url: str) -> SroPage:
        for _ in range(5):
            url = safe_sro_url(url)
            response = self._request(url)
            try:
                if response.is_redirect:
                    url = safe_sro_url(response.headers.get("Location", ""), base=url)
                    continue
                if response.status_code == 404:
                    if not is_sro_lookup_url(url):
                        raise SnapshotValidationError("sro_page_not_found")
                elif response.status_code != 200:
                    raise SnapshotValidationError("sro_upstream_http_error")
                return SroPage(
                    url,
                    limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES),
                    status_code=response.status_code,
                )
            except requests.RequestException as exc:
                self.http_errors_count += 1
                raise SnapshotValidationError("sro_upstream_transport_error") from exc
            finally:
                response.close()
        raise SnapshotValidationError("sro_redirect_limit")

    def _pace(self) -> None:
        require_sro_access_approved()
        if self.requests_count >= settings.SRO_MAX_REQUESTS_PER_RUN:
            raise SnapshotValidationError("sro_request_budget_exceeded")
        interval = max(3.0, settings.SRO_REQUEST_INTERVAL_SECONDS)
        if self.last_request is not None:
            self.sleep(max(0.0, interval - (self.clock() - self.last_request)))
        self.last_request = self.clock()
        self.requests_count += 1

    def _request(self, url: str):
        for attempt in range(3):
            self._pace()
            try:
                response = self.session.get(
                    url,
                    stream=True,
                    allow_redirects=False,
                    timeout=(10, settings.SRO_HTTP_TIMEOUT_SECONDS),
                )
            except requests.RequestException as exc:
                self.http_errors_count += 1
                if attempt == 2:
                    raise SnapshotValidationError(
                        "sro_upstream_transport_error"
                    ) from exc
                continue
            if response.status_code >= 400:
                self.http_errors_count += 1
            if response.status_code not in (429, 500, 502, 503, 504):
                return response
            retry_after = response.headers.get("Retry-After", "")
            response.close()
            self._retry_pause(retry_after)
        raise SnapshotValidationError("sro_upstream_unavailable")

    def _retry_pause(self, value: str) -> None:
        delay = 3.0
        if value.isdigit():
            delay = max(delay, int(value))
        elif value:
            try:
                target = parsedate_to_datetime(value)
                delay = max(delay, (target - datetime.now(UTC)).total_seconds())
            except (TypeError, ValueError):
                raise SnapshotValidationError("sro_invalid_retry_after") from None
        if delay > 300:
            raise SnapshotValidationError("sro_retry_after_exceeds_budget")
        self.sleep(delay)

    def close(self) -> None:
        if self.owns_session:
            self.session.close()