feat: complete published registry contracts and gated SRO ingestion

src/apps/parsers/sro_http.py

@@ -0,0 +1,147 @@
+"""Permission-gated, paced reads of the SRO site; no automatic hidden retries."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from urllib.parse import urljoin, urlsplit
+
+import requests
+from apps.core.exceptions import ConflictError
+from apps.parsers.registry_http import limited_bytes
+from apps.parsers.registry_snapshots import SnapshotValidationError
+from django.conf import settings
+
+
+def require_sro_access_approved() -> None:
+    if not (
+        settings.SRO_UPSTREAM_ACCESS_APPROVED
+        and settings.SRO_UPSTREAM_APPROVAL_REFERENCE.strip()
+    ):
+        raise ConflictError(
+            message="Автоматизированный доступ к реестру СРО ещё не согласован",
+            code="upstream_access_not_approved",
+        )
+
+
+def safe_sro_url(value: str, *, base: str | None = None) -> str:
+    value = urljoin(base, value) if base else value
+    try:
+        parts = urlsplit(value)
+        host = (parts.hostname or "").lower()
+        valid = (
+            parts.scheme == "https"
+            and (host == "reestr-sro.ru" or host.endswith(".reestr-sro.ru"))
+            and not parts.username
+            and not parts.password
+            and parts.port in (None, 443)
+            and not parts.fragment
+            and not any(ord(char) < 33 for char in value)
+        )
+    except (TypeError, ValueError):
+        valid = False
+    if not valid:
+        raise SnapshotValidationError("unsafe_sro_url")
+    return value
+
+
+@dataclass(frozen=True)
+class SroPage:
+    url: str
+    body: bytes
+
+
+class SroHttpClient:
+    """One sequential task owns the global site interval, including redirects."""
+
+    def __init__(self, *, session=None, clock=time.monotonic, sleep=time.sleep):
+        require_sro_access_approved()
+        self.session = session or requests.Session()
+        self.owns_session = session is None
+        if self.owns_session:
+            self.session.trust_env = False
+            self.session.headers[
+                "User-Agent"
+            ] = "Mostovik SRO integration (approved access)"
+        self.clock, self.sleep = clock, sleep
+        self.last_request = None
+        self.requests_count = 0
+        self.http_errors_count = 0
+
+    def get(self, url: str) -> SroPage:
+        for _ in range(5):
+            url = safe_sro_url(url)
+            response = self._request(url)
+            try:
+                if response.is_redirect:
+                    url = safe_sro_url(response.headers.get("Location", ""), base=url)
+                    continue
+                if response.status_code == 404:
+                    raise SnapshotValidationError("sro_page_not_found")
+                if response.status_code != 200:
+                    raise SnapshotValidationError("sro_upstream_http_error")
+                return SroPage(
+                    url, limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES)
+                )
+            except requests.RequestException as exc:
+                self.http_errors_count += 1
+                raise SnapshotValidationError("sro_upstream_transport_error") from exc
+            finally:
+                response.close()
+        raise SnapshotValidationError("sro_redirect_limit")
+
+    def _pace(self) -> None:
+        require_sro_access_approved()
+        if self.requests_count >= settings.SRO_MAX_REQUESTS_PER_RUN:
+            raise SnapshotValidationError("sro_request_budget_exceeded")
+        interval = max(3.0, settings.SRO_REQUEST_INTERVAL_SECONDS)
+        if self.last_request is not None:
+            self.sleep(max(0.0, interval - (self.clock() - self.last_request)))
+        self.last_request = self.clock()
+        self.requests_count += 1
+
+    def _request(self, url: str):
+        for attempt in range(3):
+            self._pace()
+            try:
+                response = self.session.get(
+                    url,
+                    stream=True,
+                    allow_redirects=False,
+                    timeout=(10, settings.SRO_HTTP_TIMEOUT_SECONDS),
+                )
+            except requests.RequestException as exc:
+                self.http_errors_count += 1
+                if attempt == 2:
+                    raise SnapshotValidationError(
+                        "sro_upstream_transport_error"
+                    ) from exc
+                continue
+            if response.status_code >= 400:
+                self.http_errors_count += 1
+            if response.status_code not in (429, 500, 502, 503, 504):
+                return response
+            retry_after = response.headers.get("Retry-After", "")
+            response.close()
+            self._retry_pause(retry_after)
+        raise SnapshotValidationError("sro_upstream_unavailable")
+
+    def _retry_pause(self, value: str) -> None:
+        delay = 3.0
+        if value.isdigit():
+            delay = max(delay, int(value))
+        elif value:
+            try:
+                target = parsedate_to_datetime(value)
+                delay = max(delay, (target - datetime.now(UTC)).total_seconds())
+            except (TypeError, ValueError):
+                raise SnapshotValidationError("sro_invalid_retry_after") from None
+        if delay > 300:
+            raise SnapshotValidationError("sro_retry_after_exceeds_budget")
+        self.sleep(delay)
+
+    def close(self) -> None:
+        if self.owns_session:
+            self.session.close()