fix: enable dev SRO collection and handle real lookup responses

tests/apps/parsers/test_sro_canonical_candidates.py

@@ -0,0 +1,83 @@
+"""SRO queries use the directory's canonical organization without hiding conflicts."""
+
+from apps.parsers.models import SroOrganizationLookup
+from apps.parsers.sro_membership import _organization_fingerprint, _sro_candidates
+from django.test import TestCase
+from django.utils import timezone
+from organizations.models import Organization
+
+
+class SroCanonicalCandidatesTest(TestCase):
+    def create_organization(self, **overrides):
+        values = {
+            "name": "АО Фикстура",
+            "inn": "1234567890",
+            "ogrn": "1027700132195",
+            "okpo": "00123456",
+            "directory_imported_at": timezone.now(),
+        }
+        values.update(overrides)
+        return Organization.objects.create(**values)
+
+    def candidate_ids(self, mode):
+        return [organization.uid for organization in _sro_candidates(mode)]
+
+    def test_head_and_branches_are_queried_as_one_canonical_organization(self):
+        head = self.create_organization()
+        self.create_organization(name="Первый филиал", is_branch=True)
+        self.create_organization(name="Второй филиал", is_branch=True)
+
+        for mode in ("full", "incremental"):
+            with self.subTest(mode=mode):
+                self.assertEqual(self.candidate_ids(mode), [head.uid])
+
+    def test_ambiguous_heads_are_not_silently_discarded_or_chosen(self):
+        first = self.create_organization()
+        second = self.create_organization(name="Другой головной офис")
+
+        for mode in ("full", "incremental"):
+            with self.subTest(mode=mode):
+                self.assertEqual(set(self.candidate_ids(mode)), {first.uid, second.uid})
+
+    def test_unchanged_head_skips_incremental_and_branches_do_not_retrigger_it(self):
+        head = self.create_organization()
+        self.create_organization(name="Филиал без контрольной точки", is_branch=True)
+        SroOrganizationLookup.objects.create(
+            organization=head,
+            checked_at=timezone.now(),
+            organization_fingerprint=_organization_fingerprint(head),
+        )
+
+        self.assertEqual(self.candidate_ids("incremental"), [])
+        self.assertEqual(self.candidate_ids("full"), [head.uid])
+
+        head.name = "Новое наименование головной организации"
+        head.save(update_fields=["name"])
+        self.assertEqual(self.candidate_ids("incremental"), [head.uid])
+
+    def test_branch_with_distinct_identity_remains_a_candidate(self):
+        head = self.create_organization()
+        branch = self.create_organization(
+            name="Филиал с собственными реквизитами",
+            inn="1234567891",
+            ogrn="1027700132196",
+            is_branch=True,
+        )
+
+        for mode in ("full", "incremental"):
+            with self.subTest(mode=mode):
+                self.assertEqual(set(self.candidate_ids(mode)), {head.uid, branch.uid})
+
+    def test_incomplete_resolution_does_not_discard_the_candidate(self):
+        head = self.create_organization(okpo="")
+        branch = self.create_organization(name="Филиал", is_branch=True)
+
+        for mode in ("full", "incremental"):
+            with self.subTest(mode=mode):
+                self.assertEqual(set(self.candidate_ids(mode)), {head.uid, branch.uid})
+
+    def test_organization_outside_directory_does_not_become_a_candidate(self):
+        head = self.create_organization()
+        self.create_organization(directory_imported_at=None)
+
+        self.assertEqual(self.candidate_ids("full"), [head.uid])

tests/apps/parsers/test_sro_dev_access.py

@@ -0,0 +1,119 @@
+"""Explicit dev collection does not fabricate upstream approval or perform HTTP."""
+
+from importlib import import_module
+from unittest.mock import Mock, patch
+from uuid import UUID
+
+import pytest
+from apps.core.exceptions import ConflictError
+from apps.core.models import BackgroundJob, JobStatus
+from apps.parsers.sro_http import SroHttpClient, require_sro_access_approved
+from core.celery import app as celery_app
+from django.core.cache import cache
+from rest_framework.test import APIClient
+
+from tests.apps.user.factories import UserFactory
+
+
+@pytest.fixture(autouse=True)
+def closed_sro_gate(settings):
+    settings.SRO_DEV_COLLECTION_ENABLED = False
+    settings.SRO_UPSTREAM_ACCESS_APPROVED = False
+    settings.SRO_UPSTREAM_APPROVAL_REFERENCE = ""
+
+
+@pytest.fixture
+def admin_client(db):
+    cache.clear()
+    client = APIClient()
+    client.force_authenticate(UserFactory.create_user(is_staff=True))
+    return client
+
+
+@pytest.mark.parametrize(
+    "dev_enabled,approved,reference,allowed",
+    (
+        (False, False, "", False),
+        (False, True, "", False),
+        (False, False, "written approval", False),
+        (False, True, "written approval", True),
+        (True, False, "", True),
+        (True, True, "", True),
+    ),
+)
+def test_gate_keeps_dev_override_separate_from_upstream_approval(
+    settings, dev_enabled, approved, reference, allowed
+):
+    settings.SRO_DEV_COLLECTION_ENABLED = dev_enabled
+    settings.SRO_UPSTREAM_ACCESS_APPROVED = approved
+    settings.SRO_UPSTREAM_APPROVAL_REFERENCE = reference
+    if allowed:
+        require_sro_access_approved()
+    else:
+        with pytest.raises(ConflictError) as error:
+            require_sro_access_approved()
+        assert error.value.code == "upstream_access_not_approved"
+    assert settings.SRO_UPSTREAM_ACCESS_APPROVED is approved
+    assert reference == settings.SRO_UPSTREAM_APPROVAL_REFERENCE
+
+
+@pytest.mark.parametrize(
+    "url",
+    (
+        "/api/v1/sources/sro-membership-check/refresh/",
+        "/api/v1/parsers/run/sro_membership_check/",
+    ),
+)
+@pytest.mark.parametrize("enabled", (False, True))
+def test_dev_refresh_entrypoints_enqueue_only_with_explicit_opt_in(
+    admin_client, settings, url, enabled
+):
+    settings.SRO_DEV_COLLECTION_ENABLED = enabled
+    import_module("apps.parsers.tasks")
+    task = celery_app.tasks["parsers.sro_membership_check.refresh"]
+    with (
+        patch.object(task, "apply_async") as dispatch,
+        patch("apps.parsers.sro_http.requests.Session") as http_session,
+    ):
+        response = admin_client.post(url, {}, format="json")
+    http_session.assert_not_called()
+    if not enabled:
+        assert response.status_code == 409
+        assert response.data["errors"][0]["code"] == "upstream_access_not_approved"
+        assert not BackgroundJob.objects.exists()
+        dispatch.assert_not_called()
+        return
+    assert response.status_code == 202
+    payload = response.data.get("data", response.data)
+    assert payload["status"] == "queued"
+    assert payload["task_ids"] == [payload["task_id"]]
+    assert str(UUID(payload["task_id"])) == payload["task_id"]
+    job = BackgroundJob.objects.get(task_id=payload["task_id"])
+    assert job.status == JobStatus.PENDING
+    assert job.task_name == "parsers.sro_membership_check.refresh"
+    dispatch.assert_called_once()
+    assert settings.SRO_UPSTREAM_ACCESS_APPROVED is False
+    assert settings.SRO_UPSTREAM_APPROVAL_REFERENCE == ""
+
+
+def test_dev_http_client_uses_neutral_user_agent(settings):
+    settings.SRO_DEV_COLLECTION_ENABLED = True
+    session = Mock(headers={})
+    with patch("apps.parsers.sro_http.requests.Session", return_value=session):
+        client = SroHttpClient()
+        client.close()
+    assert session.headers["User-Agent"] == "Mostovik SRO integration"
+    assert session.trust_env is False
+    session.get.assert_not_called()
+    session.close.assert_called_once()
+
+
+def test_revoking_dev_override_stops_before_http(settings):
+    settings.SRO_DEV_COLLECTION_ENABLED = True
+    session = Mock()
+    client = SroHttpClient(session=session)
+    settings.SRO_DEV_COLLECTION_ENABLED = False
+    with pytest.raises(ConflictError) as error:
+        client.get("https://reestr-sro.ru/")
+    assert error.value.code == "upstream_access_not_approved"
+    session.get.assert_not_called()

tests/apps/parsers/test_sro_upstream_empty.py

@@ -0,0 +1,260 @@
+"""Recognize the observed lookup 404 template without publishing an outage as empty."""
+
+import json
+import zipfile
+from unittest.mock import Mock
+from urllib.parse import parse_qs, urlsplit
+
+import pytest
+from apps.parsers.models import ParserSourceArtifact, SroOrganizationLookup
+from apps.parsers.registry_snapshots import SnapshotValidationError
+from apps.parsers.sro_http import SroHttpClient, SroPage
+from apps.parsers.sro_membership import (
+    SRO_LOOKUP_URL,
+    parse_sro_lookup,
+    refresh_sro_membership,
+)
+from django.utils import timezone
+from organizations.models import Organization, OrganizationSourceRecord
+
+from tests.apps.parsers.test_sro_membership import FixtureSite
+
+LOOKUP_TITLE = (
+    "Проверка членства в реестре СРО, проверить допуск организации в СРО по ИНН"
+)
+NOT_FOUND_HEADING = "Запрашиваемая страница на сайте отсутствует."
+LOOKUP_EMPTY = (
+    f"<html><head><title>{LOOKUP_TITLE}</title></head>"
+    f"<body><h1>{NOT_FOUND_HEADING}</h1></body></html>"
+).encode()
+
+
+@pytest.fixture(autouse=True)
+def sro_settings(settings):
+    settings.SRO_DEV_COLLECTION_ENABLED = True
+
+
+def response_404(body=LOOKUP_EMPTY):
+    response = Mock(status_code=404, is_redirect=False)
+    response.iter_content.return_value = [body]
+    return response
+
+
+@pytest.mark.parametrize("query", ("1234567890", "1027700132195"))
+def test_http_returns_bounded_lookup_404_with_status(query):
+    response = response_404()
+    session = Mock()
+    session.get.return_value = response
+    url = f"{SRO_LOOKUP_URL}?q={query}&sro_name=&search=Find"
+    page = SroHttpClient(session=session).get(url)
+    assert (page.url, page.body, page.status_code) == (url, LOOKUP_EMPTY, 404)
+    session.get.assert_called_once()
+    response.close.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    "suffix",
+    (
+        "/",
+        "/members/?q=1234567890",
+        "/proverka_dopuska/extra/?q=1234567890",
+        "/proverka_dopuska/",
+        "/proverka_dopuska/?q=12345678901",
+        "/proverka_dopuska/?q=1234567890&q=",
+        "/proverka_dopuska/?q=1234567890&q=1027700132195",
+        "/proverka_dopuska/?q=not-an-identifier",
+        "/proverka_dopuska/?q=１２３４５６７８９０",
+    ),
+)
+def test_other_404s_are_not_reclassified_or_read(suffix):
+    response = response_404()
+    session = Mock()
+    session.get.return_value = response
+    with pytest.raises(SnapshotValidationError, match="^sro_page_not_found$"):
+        SroHttpClient(session=session).get(f"https://www.reestr-sro.ru{suffix}")
+    response.iter_content.assert_not_called()
+    response.close.assert_called_once()
+
+
+def test_lookup_404_response_body_size_limit(settings):
+    settings.SRO_HTTP_MAX_RESPONSE_BYTES = 4
+    response = response_404(b"12345")
+    session = Mock()
+    session.get.return_value = response
+    with pytest.raises(SnapshotValidationError, match="^source_response_too_large$"):
+        SroHttpClient(session=session).get(f"{SRO_LOOKUP_URL}?q=1234567890")
+    response.close.assert_called_once()
+
+
+def test_parser_recognizes_only_observed_negative_404():
+    page = SroPage(f"{SRO_LOOKUP_URL}?q=1234567890", LOOKUP_EMPTY, status_code=404)
+    assert parse_sro_lookup(page) == ([], None)
+    assert SroPage(page.url, b"fixture").status_code == 200
+
+
+@pytest.mark.parametrize(
+    "body",
+    (
+        b"<html><h1>404 Not Found</h1></html>",
+        b"<html><title>Login</title><h1>Sign in</h1></html>",
+        "Ничего не найдено".encode(),
+        LOOKUP_EMPTY.replace(LOOKUP_TITLE.encode(), b"Other source"),
+        LOOKUP_EMPTY.replace(NOT_FOUND_HEADING.encode(), b"Captcha"),
+        LOOKUP_EMPTY.replace(b"</body>", b"<h1>Other</h1></body>"),
+        LOOKUP_EMPTY.replace(b"</body>", b'<table class="sro-members"></table></body>'),
+    ),
+)
+def test_parser_rejects_unrecognized_404_even_with_empty_markers(body):
+    page = SroPage(f"{SRO_LOOKUP_URL}?q=1234567890", body, status_code=404)
+    with pytest.raises(
+        SnapshotValidationError, match="^sro_lookup_response_unrecognized$"
+    ):
+        parse_sro_lookup(page)
+
+
+@pytest.mark.parametrize(
+    "url,status",
+    (
+        (f"{SRO_LOOKUP_URL}?q=1234567890", 200),
+        (f"{SRO_LOOKUP_URL}?q=1234567890", 503),
+        ("https://www.reestr-sro.ru/members/?q=1234567890", 404),
+        ("https://other.invalid/proverka_dopuska/?q=1234567890", 404),
+    ),
+)
+def test_template_is_not_empty_for_other_status_or_url(url, status):
+    with pytest.raises(SnapshotValidationError):
+        parse_sro_lookup(SroPage(url, LOOKUP_EMPTY, status_code=status))
+
+
+class MixedFixtureSite(FixtureSite):
+    def __init__(self, organizations, negative_ids):
+        super().__init__(organizations)
+        self.negative_ids = negative_ids
+
+    def get(self, url):
+        parts = urlsplit(url)
+        query = parse_qs(parts.query).get("q", [None])[0]
+        if parts.path == "/proverka_dopuska/" and query in self.negative_ids:
+            self.urls.append(url)
+            return SroPage(url, LOOKUP_EMPTY, status_code=404)
+        return super().get(url)
+
+
+@pytest.fixture
+def organizations(db, settings, tmp_path):
+    settings.MEDIA_ROOT = str(tmp_path / "media")
+    settings.PARSER_PRIVATE_ARTIFACT_ROOT = str(tmp_path / "private")
+    return [
+        Organization.objects.create(
+            name=f"АО Фикстура {index}",
+            inn=f"{index}234567890",
+            ogrn=f"{index}027700132195",
+            okpo=f"0012345{index}",
+            directory_imported_at=timezone.now(),
+        )
+        for index in (1, 2)
+    ]
+
+
+def test_mixed_positive_and_known_404_publishes_with_status_provenance(organizations):
+    positive, negative = organizations
+    refresh_sro_membership(load_batch=1, mode="full", client=FixtureSite(organizations))
+    artifact, result = refresh_sro_membership(
+        load_batch=2,
+        mode="full",
+        client=MixedFixtureSite(organizations, {negative.ogrn}),
+    )
+    assert result.published == 1
+    assert (
+        OrganizationSourceRecord.objects.get().extension.organization_id == positive.uid
+    )
+    assert artifact.metadata["recognized_lookup_404_count"] == 1
+    assert artifact.metadata["successful_lookup_200_count"] == 1
+    assert artifact.metadata["not_found_organizations_count"] == 1
+    assert (
+        SroOrganizationLookup.objects.get(organization=negative).artifact_id
+        == artifact.uid
+    )
+    with zipfile.ZipFile(artifact.file.path) as raw:
+        assert raw.testzip() is None
+        manifest = json.loads(raw.read("manifest.json"))
+        assert sorted(item["status_code"] for item in manifest) == [200, 200, 404]
+        negative_entry = next(item for item in manifest if item["status_code"] == 404)
+        assert raw.read(negative_entry["file"]) == LOOKUP_EMPTY
+
+
+@pytest.mark.parametrize("mode", ("full", "incremental"))
+def test_all_known_404_rejects_without_changing_previous_snapshot_or_checkpoints(
+    organizations, mode
+):
+    previous, _ = refresh_sro_membership(
+        load_batch=1, mode="full", client=FixtureSite(organizations)
+    )
+    records = list(OrganizationSourceRecord.objects.order_by("uid").values())
+    checkpoints = list(
+        SroOrganizationLookup.objects.order_by("organization_id").values()
+    )
+    for organization in organizations:
+        organization.name += " изменённая"
+        organization.save(update_fields=["name"])
+    on_publish = Mock()
+    with pytest.raises(SnapshotValidationError, match="^sro_ambiguous_empty_scan$"):
+        refresh_sro_membership(
+            load_batch=2,
+            mode=mode,
+            client=MixedFixtureSite(organizations, {org.ogrn for org in organizations}),
+            on_publish=on_publish,
+        )
+    on_publish.assert_not_called()
+    assert list(OrganizationSourceRecord.objects.order_by("uid").values()) == records
+    assert (
+        list(SroOrganizationLookup.objects.order_by("organization_id").values())
+        == checkpoints
+    )
+    previous.refresh_from_db()
+    assert previous.status == ParserSourceArtifact.Status.PUBLISHED
+    rejected = ParserSourceArtifact.objects.get(load_batch=2)
+    assert rejected.status == ParserSourceArtifact.Status.REJECTED
+    assert rejected.metadata["recognized_lookup_404_count"] == 2
+    assert rejected.metadata["successful_lookup_200_count"] == 0
+    with zipfile.ZipFile(rejected.file.path) as raw:
+        assert raw.testzip() is None
+        assert [
+            item["status_code"] for item in json.loads(raw.read("manifest.json"))
+        ] == [404, 404]
+
+
+def test_negative_redirect_to_other_identifier_preserves_snapshot(organizations):
+    positive, negative = organizations
+    refresh_sro_membership(load_batch=1, mode="full", client=FixtureSite(organizations))
+    records = list(OrganizationSourceRecord.objects.order_by("uid").values())
+    checkpoints = list(
+        SroOrganizationLookup.objects.order_by("organization_id").values()
+    )
+
+    class RedirectedSite(MixedFixtureSite):
+        def get(self, url):
+            page = super().get(url)
+            if page.status_code == 404:
+                return SroPage(
+                    f"{SRO_LOOKUP_URL}?q={positive.ogrn}", page.body, status_code=404
+                )
+            return page
+
+    with pytest.raises(
+        SnapshotValidationError, match="^sro_lookup_identifier_mismatch$"
+    ):
+        refresh_sro_membership(
+            load_batch=2,
+            mode="full",
+            client=RedirectedSite(organizations, {negative.ogrn}),
+        )
+    assert list(OrganizationSourceRecord.objects.order_by("uid").values()) == records
+    assert (
+        list(SroOrganizationLookup.objects.order_by("organization_id").values())
+        == checkpoints
+    )
+    rejected = ParserSourceArtifact.objects.get(load_batch=2)
+    assert rejected.status == ParserSourceArtifact.Status.REJECTED
+    assert rejected.metadata["recognized_lookup_404_count"] == 0
+    assert rejected.metadata["parse_errors_count"] == 1