fix: enable dev SRO collection and handle real lookup responses

src/apps/parsers/sro_http.py

@@ -1,12 +1,13 @@
-"""Permission-gated, paced reads of the SRO site; no automatic hidden retries."""
+"""Access-gated SRO reads with explicit dev opt-in, pacing and bounded retries."""
 
 from __future__ import annotations
 
+import re
 import time
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
-from urllib.parse import urljoin, urlsplit
+from urllib.parse import parse_qs, urljoin, urlsplit
 
 import requests
 from apps.core.exceptions import ConflictError
@@ -16,6 +17,8 @@ from django.conf import settings
 
 
 def require_sro_access_approved() -> None:
+    if settings.SRO_DEV_COLLECTION_ENABLED:
+        return
     if not (
         settings.SRO_UPSTREAM_ACCESS_APPROVED
         and settings.SRO_UPSTREAM_APPROVAL_REFERENCE.strip()
@@ -47,10 +50,26 @@ def safe_sro_url(value: str, *, base: str | None = None) -> str:
     return value
 
 
+def is_sro_lookup_url(value: str) -> bool:
+    """Only the observed exact identifier lookup may carry a negative 404 body."""
+    try:
+        parts = urlsplit(safe_sro_url(value))
+        query = parse_qs(parts.query, keep_blank_values=True, max_num_fields=20)
+    except (SnapshotValidationError, ValueError):
+        return False
+    identifiers = query.get("q", [])
+    return (
+        parts.path == "/proverka_dopuska/"
+        and len(identifiers) == 1
+        and re.fullmatch(r"[0-9]{10}(?:[0-9]{3})?", identifiers[0]) is not None
+    )
+
+
 @dataclass(frozen=True)
 class SroPage:
     url: str
     body: bytes
+    status_code: int = 200
 
 
 class SroHttpClient:
@@ -62,9 +81,7 @@ class SroHttpClient:
         self.owns_session = session is None
         if self.owns_session:
             self.session.trust_env = False
-            self.session.headers[
-                "User-Agent"
-            ] = "Mostovik SRO integration (approved access)"
+            self.session.headers["User-Agent"] = "Mostovik SRO integration"
         self.clock, self.sleep = clock, sleep
         self.last_request = None
         self.requests_count = 0
@@ -79,11 +96,14 @@ class SroHttpClient:
                     url = safe_sro_url(response.headers.get("Location", ""), base=url)
                     continue
                 if response.status_code == 404:
-                    raise SnapshotValidationError("sro_page_not_found")
-                if response.status_code != 200:
+                    if not is_sro_lookup_url(url):
+                        raise SnapshotValidationError("sro_page_not_found")
+                elif response.status_code != 200:
                     raise SnapshotValidationError("sro_upstream_http_error")
                 return SroPage(
-                    url, limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES)
+                    url,
+                    limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES),
+                    status_code=response.status_code,
                 )
             except requests.RequestException as exc:
                 self.http_errors_count += 1