feat: restore registry procurement collection

tests/apps/parsers/test_checko_parsers.py

@@ -37,6 +37,58 @@ class CheckoClientParsingTest(SimpleTestCase):
             data["\u0417\u0430\u043f\u0438\u0441\u0438"][0]["\u041e\u0413\u0420\u041d"],
         )
 
+    def test_parse_company_maps_russian_unfair_supplier_records(self):
+        data = _map_ru_keys(
+            {
+                "ОГРН": "1027700000001",
+                "ИНН": "7701000001",
+                "НаимСокр": "ООО ОПК",
+                "НедобПост": True,
+                "НедобПостЗап": [
+                    {
+                        "РеестрНомер": "RNP-1",
+                        "ДатаПуб": "2026-07-01",
+                        "ДатаУтв": "2026-06-30",
+                        "ЗаказНаимСокр": "Заказчик",
+                        "ЗаказНаимПолн": "Заказчик полный",
+                        "ЗаказИНН": "7702000002",
+                        "ЗаказКПП": "770201001",
+                        "ЗакупНомер": "PURCHASE-1",
+                        "ЗакупОпис": "Поставка оборудования",
+                        "ЦенаКонтр": 1500000,
+                    },
+                    {
+                        "РеестрНомер": "RNP-2",
+                        "ЗакупНомер": "PURCHASE-2",
+                    },
+                ],
+            }
+        )
+
+        company = self.client._parse_company_data(data)
+
+        self.assertEqual(len(company.unfair_supplier), 2)
+        record = company.unfair_supplier[0]
+        self.assertEqual(record.registry_number, "RNP-1")
+        self.assertEqual(record.customer_inn, "7702000002")
+        self.assertEqual(record.purchase_description, "Поставка оборудования")
+        self.assertEqual(record.contract_price, 1500000)
+        self.assertEqual(company.unfair_supplier[1].registry_number, "RNP-2")
+
+    def test_parse_company_handles_empty_unfair_supplier_records(self):
+        company = self.client._parse_company_data(
+            _map_ru_keys(
+                {
+                    "ОГРН": "1027700000001",
+                    "ИНН": "7701000001",
+                    "НедобПост": False,
+                    "НедобПостЗап": [],
+                }
+            )
+        )
+
+        self.assertEqual(company.unfair_supplier, ())
+
     def test_parse_okved_info_with_additional(self):
         info = self.client._parse_okved_info(
             {"code": "62.01", "name": "Development", "version": "2001"},

tests/apps/parsers/test_eis_registry_client.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from datetime import date
+from decimal import Decimal
+from urllib.parse import parse_qs, urlsplit
+
+from apps.parsers.clients.base import HTTPClientError
+from apps.parsers.clients.common.eis_registry import EisRegistryProcurementClient
+from apps.parsers.clients.common.schemas import GenericParserItem
+from django.test import SimpleTestCase
+
+
+def _record(*, external_id: str, inn: str, amount: str = "1") -> GenericParserItem:
+    return GenericParserItem(
+        source="procurements_44fz",
+        external_id=external_id,
+        inn=inn,
+        organisation_name="Наименование из ЕИС",
+        title="Закупка",
+        amount=Decimal(amount),
+        payload={"raw": external_id},
+    )
+
+
+class _FakeHttpClient:
+    def __init__(self, pages: dict[int, bytes], *, error_page: int | None = None):
+        self.pages = pages
+        self.error_page = error_page
+        self.urls: list[str] = []
+
+    def download_file(self, url: str, **_kwargs) -> bytes:
+        self.urls.append(url)
+        page = int(parse_qs(urlsplit(url).query)["pageNumber"][0])
+        if page == self.error_page:
+            raise HTTPClientError("page failed")
+        return self.pages[page]
+
+
+class _FakeStructuredClient:
+    def __init__(self, pages: dict[int, bytes], records: dict[bytes, list]):
+        self.http_client = _FakeHttpClient(pages)
+        self.records = records
+
+    def fetch_records(self, *, content: bytes, file_name: str):
+        assert file_name == "results.html"
+        return self.records[content]
+
+
+class EisRegistryProcurementClientTest(SimpleTestCase):
+    def test_fetches_all_pages_filters_inn_and_deduplicates_external_id(self):
+        page_1 = b'<a data-pagenumber="2"></a>'
+        page_2 = b'<a data-pagenumber="2"></a>'
+        structured = _FakeStructuredClient(
+            {1: page_1, 2: page_2},
+            {
+                page_1: [
+                    _record(external_id="same", inn="7701000001", amount="1"),
+                    _record(external_id="foreign", inn="7701000099"),
+                ],
+                page_2: [
+                    _record(external_id="same", inn="7701000001", amount="2"),
+                    _record(external_id="other", inn="7701000001"),
+                ],
+            },
+        )
+        client = EisRegistryProcurementClient(
+            source="procurements_44fz",
+            law="44",
+            _structured_client=structured,
+        )
+
+        records = client.fetch_for_customer(
+            inn="7701000001",
+            organization_name="ОПК Заказчик",
+            organization_id="org-1",
+            today=date(2026, 7, 19),
+        )
+
+        self.assertEqual([row.external_id for row in records], ["same", "other"])
+        self.assertEqual(records[0].amount, Decimal("2"))
+        self.assertTrue(all(row.inn == "7701000001" for row in records))
+        self.assertEqual(records[0].payload["organization_id"], "org-1")
+        query = parse_qs(urlsplit(structured.http_client.urls[0]).query)
+        self.assertEqual(query["fz44"], ["on"])
+        self.assertEqual(query["strictEqual"], ["true"])
+        self.assertEqual(query["publishDateFrom"], ["19.07.2025"])
+        self.assertEqual(query["publishDateTo"], ["19.07.2026"])
+
+    def test_builds_223fz_query_and_accepts_empty_page(self):
+        empty_page = b"<html></html>"
+        structured = _FakeStructuredClient({1: empty_page}, {empty_page: []})
+        client = EisRegistryProcurementClient(
+            source="procurements_223fz",
+            law="223",
+            _structured_client=structured,
+        )
+
+        records = client.fetch_for_customer(
+            inn="7701000001",
+            organization_name="ОПК Заказчик",
+            organization_id="org-1",
+        )
+
+        self.assertEqual(records, [])
+        query = parse_qs(urlsplit(structured.http_client.urls[0]).query)
+        self.assertEqual(query["fz223"], ["on"])
+        self.assertNotIn("fz44", query)
+
+    def test_page_error_is_not_silenced(self):
+        page_1 = b'<a data-pagenumber="2"></a>'
+        structured = _FakeStructuredClient({1: page_1}, {page_1: []})
+        structured.http_client.error_page = 2
+        client = EisRegistryProcurementClient(
+            source="procurements_44fz",
+            law="44",
+            _structured_client=structured,
+        )
+
+        with self.assertRaises(HTTPClientError):
+            client.fetch_for_customer(
+                inn="7701000001",
+                organization_name="ОПК Заказчик",
+                organization_id="org-1",
+            )

tests/apps/parsers/test_registry_procurement_tasks.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+from contextlib import ExitStack
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from apps.parsers import tasks as parser_tasks
+from apps.parsers.clients.common.schemas import GenericParserItem
+from apps.parsers.models import CheckoCollectionAttempt, ParserLoadLog
+from django.test import TestCase, override_settings
+
+from tests.apps.parsers.organization_helpers import create_directory_organization
+
+
+def _organization(index: int, *, membership: bool = True):
+    return create_directory_organization(
+        name=f"Организация {index}",
+        inn=f"7701000{index:03d}",
+        ogrn=f"1027700000{index:03d}",
+        opk_registry_membership=membership,
+    )
+
+
+class RegistryProcurementTasksTest(TestCase):
+    def test_daily_orchestrator_queues_addressed_sources_with_same_batch_limit(self):
+        task_names = (
+            "parse_procurements_44fz",
+            "parse_procurements_223fz",
+            "parse_registry_contracts",
+            "parse_unfair_suppliers",
+            "parse_arbitration_cases",
+            "parse_fedresurs_bankruptcy",
+            "parse_registry_inspections",
+            "parse_trudvsem_vacancies",
+        )
+        with ExitStack() as stack:
+            mocks = [
+                stack.enter_context(
+                    patch.object(
+                        getattr(parser_tasks, task_name),
+                        "delay",
+                        return_value=SimpleNamespace(id=f"{task_name}-id"),
+                    )
+                )
+                for task_name in task_names
+            ]
+            result = parser_tasks.parse_registry_enrichment_sources(
+                limit=20,
+                proxies=[],
+            )
+
+        self.assertEqual(
+            set(result),
+            {
+                "procurements_44fz",
+                "procurements_223fz",
+                "contracts",
+                "unfair_suppliers",
+                "arbitration",
+                "bankruptcy",
+                "inspections",
+                "vacancies",
+            },
+        )
+        for task_mock in mocks[:5]:
+            task_mock.assert_called_once_with(
+                proxies=[],
+                requested_by_id=None,
+                limit=20,
+            )
+        mocks[5].assert_called_once_with(proxies=[], requested_by_id=None)
+        mocks[6].assert_called_once_with(proxies=[], requested_by_id=None, limit=20)
+        mocks[7].assert_called_once_with(proxies=[], requested_by_id=None)
+
+    def test_eis_monthly_claims_advance_to_the_next_batch(self):
+        organizations = [_organization(index) for index in range(1, 4)]
+        outside_registry = _organization(9, membership=False)
+        requested_inns: list[str] = []
+
+        class _Client:
+            def __init__(self, **_kwargs):
+                return
+
+            def fetch_for_customer(self, **kwargs):
+                requested_inns.append(kwargs["inn"])
+                return [
+                    GenericParserItem(
+                        source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
+                        external_id=f"notice-{kwargs['inn']}",
+                        inn=kwargs["inn"],
+                        organisation_name=kwargs["organization_name"],
+                    )
+                ]
+
+        with patch.object(parser_tasks, "EisRegistryProcurementClient", _Client):
+            first = parser_tasks._fetch_eis_registry_procurement_records(
+                source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
+                law="44",
+                limit=2,
+                proxies=[],
+            )
+            second = parser_tasks._fetch_eis_registry_procurement_records(
+                source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
+                law="44",
+                limit=2,
+                proxies=[],
+            )
+
+        self.assertEqual(len(first), 2)
+        self.assertEqual(len(second), 1)
+        self.assertEqual(requested_inns, [item.inn for item in organizations])
+        self.assertNotIn(outside_registry.inn, requested_inns)
+        self.assertEqual(
+            CheckoCollectionAttempt.objects.filter(
+                source=CheckoCollectionAttempt.Source.PROCUREMENTS_44FZ,
+                status=CheckoCollectionAttempt.Status.SUCCESS,
+            ).count(),
+            3,
+        )
+
+    def test_eis_request_error_creates_failed_monthly_claim(self):
+        organization = _organization(1)
+
+        class _Client:
+            def __init__(self, **_kwargs):
+                return
+
+            def fetch_for_customer(self, **_kwargs):
+                raise parser_tasks.HTTPClientError("EIS unavailable")
+
+        with (
+            patch.object(parser_tasks, "EisRegistryProcurementClient", _Client),
+            self.assertRaises(parser_tasks.ParserSourceSkipped),
+        ):
+            parser_tasks._fetch_eis_registry_procurement_records(
+                source=ParserLoadLog.Source.PROCUREMENTS_223FZ,
+                law="223",
+                limit=1,
+                proxies=[],
+            )
+
+        attempt = CheckoCollectionAttempt.objects.get(
+            organization=organization,
+            source=CheckoCollectionAttempt.Source.PROCUREMENTS_223FZ,
+        )
+        self.assertEqual(attempt.status, CheckoCollectionAttempt.Status.FAILED)
+
+    def test_explicit_organization_ids_do_not_fall_through_to_next_batch(self):
+        selected = _organization(1)
+        _organization(2)
+        calls: list[str] = []
+
+        class _Client:
+            def __init__(self, **_kwargs):
+                return
+
+            def fetch_for_customer(self, **kwargs):
+                calls.append(kwargs["inn"])
+                return []
+
+        kwargs = {
+            "source": ParserLoadLog.Source.PROCUREMENTS_44FZ,
+            "law": "44",
+            "limit": 20,
+            "proxies": [],
+            "organization_ids": [str(selected.uid)],
+        }
+        with patch.object(parser_tasks, "EisRegistryProcurementClient", _Client):
+            self.assertEqual(
+                parser_tasks._fetch_eis_registry_procurement_records(**kwargs),
+                [],
+            )
+            with self.assertRaises(parser_tasks.ParserSourceSkipped):
+                parser_tasks._fetch_eis_registry_procurement_records(**kwargs)
+
+        self.assertEqual(calls, [selected.inn])
+
+    @override_settings(CHECKO_API_KEY="test-key")
+    def test_checko_rnp_is_supplier_bound_and_not_requested_twice(self):
+        organization = _organization(1)
+        calls: list[str] = []
+
+        class _Client:
+            def __init__(self, **_kwargs):
+                return
+
+            def get_company(self, request):
+                calls.append(request.inn)
+                return SimpleNamespace(
+                    data=SimpleNamespace(
+                        inn=organization.inn,
+                        ogrn=organization.ogrn,
+                        unfair_supplier=(
+                            SimpleNamespace(
+                                registry_number="RNP-1",
+                                publish_date="2026-07-01",
+                                approval_date="2026-06-30",
+                                customer_short_name="Заказчик",
+                                customer_full_name="Заказчик полный",
+                                customer_inn="7702000002",
+                                customer_kpp="770201001",
+                                purchase_number="PURCHASE-1",
+                                purchase_description="Поставка оборудования",
+                                contract_price=1500000,
+                            ),
+                        ),
+                    )
+                )
+
+        with patch.object(parser_tasks, "CheckoClient", _Client):
+            records = parser_tasks._fetch_checko_unfair_supplier_records(
+                limit=1,
+                proxies=[],
+            )
+            with self.assertRaises(parser_tasks.ParserSourceSkipped):
+                parser_tasks._fetch_checko_unfair_supplier_records(
+                    limit=1,
+                    proxies=[],
+                )
+
+        self.assertEqual(calls, [organization.inn])
+        self.assertEqual(len(records), 1)
+        self.assertEqual(records[0].inn, organization.inn)
+        self.assertEqual(records[0].payload["supplier"]["inn"], organization.inn)
+        self.assertEqual(records[0].payload["customer"]["inn"], "7702000002")
+        self.assertEqual(records[0].amount, 1500000)
+
+    @override_settings(CHECKO_API_KEY="test-key")
+    def test_checko_rnp_api_error_creates_failed_monthly_claim(self):
+        organization = _organization(1)
+
+        class _Client:
+            def __init__(self, **_kwargs):
+                return
+
+            def get_company(self, _request):
+                raise parser_tasks.CheckoError("Checko unavailable")
+
+        with (
+            patch.object(parser_tasks, "CheckoClient", _Client),
+            self.assertRaises(parser_tasks.ParserSourceSkipped),
+        ):
+            parser_tasks._fetch_checko_unfair_supplier_records(
+                limit=1,
+                proxies=[],
+            )
+
+        attempt = CheckoCollectionAttempt.objects.get(
+            organization=organization,
+            source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
+        )
+        self.assertEqual(attempt.status, CheckoCollectionAttempt.Status.FAILED)

tests/apps/parsers/test_source_cards_service.py

@@ -240,10 +240,12 @@ class SourceCardServiceUnitTest(SimpleTestCase):
         "apps.parsers.source_cards.SourceCardService._enqueue_task",
         return_value={
             "task_id": "task-9",
-            "task_name": "apps.parsers.tasks.sync_procurements",
+            "task_name": "apps.parsers.tasks.parse_registry_enrichment_sources",
         },
     )
-    def test_refresh_card_for_procurements_uses_default_law_type(self, enqueue_mock):
+    def test_refresh_card_for_procurements_uses_addressed_registry_task(
+        self, enqueue_mock
+    ):
         result = SourceCardService.refresh_card(
             slug="public-procurements",
             requested_by_id=10,
@@ -254,12 +256,7 @@ class SourceCardServiceUnitTest(SimpleTestCase):
         self.assertEqual(result["tasks"][0]["task_id"], "task-9")
         self.assertEqual(
             enqueue_mock.call_args.kwargs["kwargs"],
-            {
-                "requested_by_id": 10,
-                "region_code": "77",
-                "law_type": "44",
-                "current_year": 2026,
-            },
+            {"requested_by_id": 10},
         )
 
     @patch(
@@ -540,6 +537,7 @@ class SourceCardServiceDatabaseTest(TestCase):
         self.assertEqual(card["records_count"], 3)
         self.assertEqual(card["organizations_count"], 2)
         source_items = {item["code"]: item for item in card["source_items"]}
+        self.assertEqual(source_items["procurements"]["records_count"], 2)
         self.assertEqual(source_items["procurements_44fz"]["organizations_count"], 1)
         self.assertEqual(source_items["procurements_223fz"]["organizations_count"], 1)
         self.assertEqual(source_items["contracts"]["organizations_count"], 1)

tests/apps/parsers/test_source_cards_views.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import hashlib
+from types import SimpleNamespace
+from unittest.mock import patch
 
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.models import ParserLoadLog
@@ -259,19 +261,24 @@ class SourceCardsApiTestCase(APITestCase):
             ).exists()
         )
 
-    def test_refresh_procurements_requires_region_code(self):
+    @override_settings(CELERY_TASK_ALWAYS_EAGER=False)
+    def test_refresh_procurements_does_not_require_region_code(self):
         self.client.force_authenticate(self.admin)
-        response = self.client.post(
-            reverse(
-                "api_v1:sources:source-cards-refresh",
-                kwargs={"slug": "public-procurements"},
-            ),
-            {},
-            format="json",
-        )
+        with patch(
+            "apps.parsers.tasks.parse_registry_enrichment_sources.apply_async",
+            return_value=SimpleNamespace(id="task-procurements"),
+        ):
+            response = self.client.post(
+                reverse(
+                    "api_v1:sources:source-cards-refresh",
+                    kwargs={"slug": "public-procurements"},
+                ),
+                {},
+                format="json",
+            )
 
-        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertFalse(response.data["success"])
+        self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
+        self.assertEqual(response.data["task_id"], "task-procurements")
 
     def test_refresh_forbidden_for_regular_user(self):
         response = self.client.post(

tests/apps/parsers/test_sources_api_e2e.py

@@ -6,7 +6,6 @@ from unittest.mock import patch
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.models import FinancialReport, FinancialReportLine, ParserLoadLog
 from django.urls import reverse
-from organizations.models import Organization
 from organizations.source_backfill import OrganizationSourceBackfillService
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -206,7 +205,7 @@ class SourcesApiE2ETest(APITestCase):
             )
 
             with patch(
-                "apps.parsers.tasks.sync_procurements.apply_async",
+                "apps.parsers.tasks.parse_registry_enrichment_sources.apply_async",
                 return_value=SimpleNamespace(id="task-procurements"),
             ):
                 procurements_response = self.client.post(
@@ -241,7 +240,7 @@ class SourcesApiE2ETest(APITestCase):
         self.assertTrue(
             BackgroundJob.objects.filter(
                 task_id="task-procurements",
-                task_name="apps.parsers.tasks.sync_procurements",
+                task_name="apps.parsers.tasks.parse_registry_enrichment_sources",
                 user_id=self.admin.id,
             ).exists()
         )

tests/apps/parsers/test_tasks.py

@@ -1855,18 +1855,10 @@ class MinpromtorgTasksTestCase(TestCase):
             "parse_industrial_products",
             "parse_manufactures",
             "sync_inspections",
-            "parse_procurements_44fz",
-            "parse_procurements_223fz",
-            "parse_contracts",
-            "parse_registry_contracts",
-            "parse_unfair_suppliers",
+            "parse_registry_enrichment_sources",
             "parse_fas_goz_evasion",
             "sync_fns_financial_reports",
-            "parse_arbitration_cases",
-            "parse_fedresurs_bankruptcy",
-            "parse_registry_inspections",
             "parse_fstec_registers",
-            "parse_trudvsem_vacancies",
         )
         with ExitStack() as stack:
             delays = {
@@ -1887,8 +1879,10 @@ class MinpromtorgTasksTestCase(TestCase):
             "sync_fns_financial_reports-id",
         )
         delays["sync_fns_financial_reports"].assert_called_once_with(proxies=[])
-        delays["parse_registry_contracts"].assert_called_once_with(proxies=[])
-        delays["parse_registry_inspections"].assert_called_once_with(proxies=[])
+        delays["parse_registry_enrichment_sources"].assert_called_once_with(
+            limit=250,
+            proxies=[],
+        )
 
     def test_parse_all_minpromtorg_without_adapter(self):
         with TestHTTPServer() as server: