fix: enable dev SRO collection and handle real lookup responses

src/apps/parsers/sro_http.py

@@ -1,12 +1,13 @@
-"""Permission-gated, paced reads of the SRO site; no automatic hidden retries."""
+"""Access-gated SRO reads with explicit dev opt-in, pacing and bounded retries."""
 
 from __future__ import annotations
 
+import re
 import time
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
-from urllib.parse import urljoin, urlsplit
+from urllib.parse import parse_qs, urljoin, urlsplit
 
 import requests
 from apps.core.exceptions import ConflictError
@@ -16,6 +17,8 @@ from django.conf import settings
 
 
 def require_sro_access_approved() -> None:
+    if settings.SRO_DEV_COLLECTION_ENABLED:
+        return
     if not (
         settings.SRO_UPSTREAM_ACCESS_APPROVED
         and settings.SRO_UPSTREAM_APPROVAL_REFERENCE.strip()
@@ -47,10 +50,26 @@ def safe_sro_url(value: str, *, base: str | None = None) -> str:
     return value
 
 
+def is_sro_lookup_url(value: str) -> bool:
+    """Only the observed exact identifier lookup may carry a negative 404 body."""
+    try:
+        parts = urlsplit(safe_sro_url(value))
+        query = parse_qs(parts.query, keep_blank_values=True, max_num_fields=20)
+    except (SnapshotValidationError, ValueError):
+        return False
+    identifiers = query.get("q", [])
+    return (
+        parts.path == "/proverka_dopuska/"
+        and len(identifiers) == 1
+        and re.fullmatch(r"[0-9]{10}(?:[0-9]{3})?", identifiers[0]) is not None
+    )
+
+
 @dataclass(frozen=True)
 class SroPage:
     url: str
     body: bytes
+    status_code: int = 200
 
 
 class SroHttpClient:
@@ -62,9 +81,7 @@ class SroHttpClient:
         self.owns_session = session is None
         if self.owns_session:
             self.session.trust_env = False
-            self.session.headers[
-                "User-Agent"
-            ] = "Mostovik SRO integration (approved access)"
+            self.session.headers["User-Agent"] = "Mostovik SRO integration"
         self.clock, self.sleep = clock, sleep
         self.last_request = None
         self.requests_count = 0
@@ -79,11 +96,14 @@ class SroHttpClient:
                     url = safe_sro_url(response.headers.get("Location", ""), base=url)
                     continue
                 if response.status_code == 404:
-                    raise SnapshotValidationError("sro_page_not_found")
-                if response.status_code != 200:
+                    if not is_sro_lookup_url(url):
+                        raise SnapshotValidationError("sro_page_not_found")
+                elif response.status_code != 200:
                     raise SnapshotValidationError("sro_upstream_http_error")
                 return SroPage(
-                    url, limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES)
+                    url,
+                    limited_bytes(response, settings.SRO_HTTP_MAX_RESPONSE_BYTES),
+                    status_code=response.status_code,
                 )
             except requests.RequestException as exc:
                 self.http_errors_count += 1

src/apps/parsers/sro_membership.py

@@ -1,8 +1,4 @@
-"""SRO HTML mapping and complete, permission-gated membership snapshots.
-
-Selectors follow the approved source-first fixtures. Live query/search acceptance
-requires upstream approval and has deliberately not been performed.
-"""
+"""SRO HTML mapping and complete snapshots under configured source access."""
 
 from __future__ import annotations
 
@@ -13,7 +9,7 @@ import tempfile
 import zipfile
 from collections import Counter
 from datetime import datetime
-from urllib.parse import urlencode, urlsplit
+from urllib.parse import parse_qs, urlencode, urlsplit
 
 from apps.parsers.models import (
     ParserSourceArtifact,
@@ -31,6 +27,7 @@ from apps.parsers.registry_snapshots import (
 from apps.parsers.sro_http import (
     SroHttpClient,
     SroPage,
+    is_sro_lookup_url,
     require_sro_access_approved,
     safe_sro_url,
 )
@@ -49,6 +46,10 @@ SRO_STATUSES = {
 }
 SRO_ID = re.compile(r"/sro-id-(\d+)(?:/|$)")
 ZERO_MARKERS = ("ничего не найдено", "сведения не найдены", "найдено 0")
+LOOKUP_NOT_FOUND_TITLE = (
+    "Проверка членства в реестре СРО, проверить допуск организации в СРО по ИНН"
+)
+LOOKUP_NOT_FOUND_HEADING = "Запрашиваемая страница на сайте отсутствует."
 MONTHS = {
     name: index
     for index, name in enumerate(
@@ -158,8 +159,29 @@ def _table_rows(document: BeautifulSoup, *, admission: bool = False) -> list[dic
     return result
 
 
-def parse_sro_lookup(page: SroPage) -> tuple[list[dict], str | None]:
+def parse_sro_lookup(
+    page: SroPage, *, expected_lookup_value: str | None = None
+) -> tuple[list[dict], str | None]:
     document = BeautifulSoup(page.body, "html.parser")
+    if page.status_code == 404:
+        titles, headings = document.find_all("title"), document.find_all("h1")
+        if (
+            is_sro_lookup_url(page.url)
+            and len(titles) == len(headings) == 1
+            and _text(titles[0].get_text(" ", strip=True)) == LOOKUP_NOT_FOUND_TITLE
+            and _text(headings[0].get_text(" ", strip=True)) == LOOKUP_NOT_FOUND_HEADING
+            and document.select_one("table.sro-members") is None
+        ):
+            if expected_lookup_value is not None and parse_qs(
+                urlsplit(page.url).query
+            ).get("q") != [expected_lookup_value]:
+                raise SnapshotValidationError("sro_lookup_identifier_mismatch")
+            # This observed error template is ambiguous in isolation. The scan
+            # requires a valid 200 lookup before it may publish such negatives.
+            return [], None
+        raise SnapshotValidationError("sro_lookup_response_unrecognized")
+    if page.status_code != 200:
+        raise SnapshotValidationError("sro_lookup_response_unrecognized")
     result = []
     for cells in _table_rows(document):
         identity = cells["identity"].get_text(" ", strip=True)
@@ -424,6 +446,7 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
     manifest, pending = [], []
     raw_count, not_found, missing_dates = 0, 0, 0
     parse_errors = 0
+    recognized_lookup_404, successful_lookup_200 = 0, 0
     reasons, dates = Counter(), set()
 
     def fetch(url):
@@ -432,7 +455,9 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
         safe_sro_url(page.url)
         filename = f"response-{len(manifest) + 1:06d}.html"
         raw.writestr(filename, page.body)
-        manifest.append({"file": filename, "url": page.url})
+        manifest.append(
+            {"file": filename, "url": page.url, "status_code": page.status_code}
+        )
         return page
 
     resolver = SroResolver(fetch)
@@ -452,7 +477,9 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
                 }
             )
             page = fetch(f"{SRO_LOOKUP_URL}?{query}")
-            rows, version = parse_sro_lookup(page)
+            rows, version = parse_sro_lookup(page, expected_lookup_value=value)
+            recognized_lookup_404 += page.status_code == 404
+            successful_lookup_200 += page.status_code == 200
             if version:
                 dates.add(version)
             if len(dates) > 1:
@@ -495,6 +522,7 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
         parse_errors += str(exc).startswith(
             (
                 "sro_members",
+                "sro_lookup_",
                 "invalid_sro_registry_date",
                 "sro_registry_changed",
                 "sro_sitemap_parse_error",
@@ -517,6 +545,8 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
             candidate_organizations_count=len(candidates),
             queried_organizations_count=completed,
             not_found_organizations_count=not_found,
+            recognized_lookup_404_count=recognized_lookup_404,
+            successful_lookup_200_count=successful_lookup_200,
             raw_memberships_count=raw_count,
             found_memberships_count=raw_count,
             quarantined_records_count=sum(reasons.values()),
@@ -546,15 +576,24 @@ def _sro_candidates(mode: str) -> list[Organization]:
         .select_related("sro_lookup")
         .order_by("uid")
     )
-    if mode == "full":
-        return list(candidates)
-    return [
-        organization
-        for organization in candidates
-        if not hasattr(organization, "sro_lookup")
-        or organization.sro_lookup.organization_fingerprint
-        != _organization_fingerprint(organization)
-    ]
+    own_index = OwnOrganizationIndex()
+    selected = []
+    for organization in candidates:
+        resolved, reason = own_index.resolve(
+            inn=organization.inn, ogrn=organization.ogrn
+        )
+        # Shared branch identities belong to the unambiguous canonical head.
+        # Keep conflicts/incomplete identities for the existing strict validation.
+        if not reason and resolved is not None and resolved.uid != organization.uid:
+            continue
+        if (
+            mode == "full"
+            or not hasattr(organization, "sro_lookup")
+            or organization.sro_lookup.organization_fingerprint
+            != _organization_fingerprint(organization)
+        ):
+            selected.append(organization)
+    return selected
 
 
 def _save_sro_checkpoints(candidates, artifact) -> None:
@@ -607,6 +646,14 @@ def refresh_sro_membership(
             # Keep raw/quarantine diagnostics, but never advance any checkpoint or
             # replace the previous complete dataset with this incomplete scan.
             raise SnapshotValidationError("sro_incomplete_membership_scan")
+        if (
+            candidates
+            and artifact.metadata["recognized_lookup_404_count"] == len(candidates)
+            and not artifact.metadata["successful_lookup_200_count"]
+        ):
+            # An all-404 scan cannot distinguish absence from a site-wide outage.
+            # Raw responses stay available, but previous rows/checkpoints survive.
+            raise SnapshotValidationError("sro_ambiguous_empty_scan")
         if mode == "incremental" and not candidates:
             previous = (
                 ParserSourceArtifact.objects.filter(

src/settings/base.py

@@ -503,7 +503,8 @@ FNS_FAILED_DIRECTORY = PROJECT_ROOT / "input" / "fns" / "failed"
 
 CHECKO_API_KEY = os.getenv("CHECKO_API_KEY", "")
 
-# SRO collection stays disabled until the site owner approves automated access.
+# Default closed; the explicit collection override is configured only on dev.
+SRO_DEV_COLLECTION_ENABLED = _env_bool("SRO_DEV_COLLECTION_ENABLED", default=False)
 SRO_UPSTREAM_ACCESS_APPROVED = (
     os.getenv("SRO_UPSTREAM_ACCESS_APPROVED", "false").lower() == "true"
 )