Add organizations v2 API and registry enrichment

tests/apps/parsers/test_views.py

@@ -18,6 +18,7 @@ from apps.parsers.models import (
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.urls import reverse
 from openpyxl import Workbook
+from organizations.models import Organization
 from rest_framework import status
 from rest_framework.test import APITestCase
 
@@ -28,6 +29,12 @@ from tests.apps.parsers.factories import (
     ManufacturerRecordFactory,
     ParserLoadLogFactory,
 )
+from tests.apps.registers.factories import (
+    OrganizationFactory as RegisterOrganizationFactory,
+)
+from tests.apps.registers.factories import (
+    RegistryMembershipPeriodFactory,
+)
 from tests.apps.user.factories import UserFactory
 from tests.utils.fixtures import fake
 
@@ -160,6 +167,126 @@ class ParsersViewSetTest(APITestCase):
         )
         self.assertEqual(detail.status_code, status.HTTP_200_OK)
 
+    def test_procurements_v1_enriches_missing_customer_fields_from_canonical_organization(
+        self,
+    ):
+        organization = Organization.objects.create(
+            name='ООО "КАНОНИЧЕСКИЙ ЗАКАЗЧИК"',
+            inn="7701000101",
+            kpp="770101001",
+            ogrn="1027700000001",
+        )
+        record = ProcurementRecord.objects.create(
+            load_batch=1,
+            purchase_number="0320100000126000001",
+            purchase_name="Поставка оборудования",
+            customer_inn="",
+            customer_kpp="",
+            customer_ogrn=organization.ogrn,
+            customer_name="",
+            max_price="1000.00",
+            status="published",
+            law_type="44-FZ",
+            href="https://zakupki.gov.ru/example",
+            region_code="77",
+        )
+
+        self.client.force_authenticate(self.user)
+
+        list_response = self.client.get(reverse("api_v1:zakupki:procurements-list"))
+        detail_response = self.client.get(
+            reverse("api_v1:zakupki:procurements-detail", args=[record.id])
+        )
+
+        self.assertEqual(list_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(detail_response.status_code, status.HTTP_200_OK)
+        list_row = list_response.data["data"][0]
+        detail_row = detail_response.data
+        self.assertEqual(list_row["customer_name"], organization.name)
+        self.assertEqual(list_row["customer_inn"], organization.inn)
+        self.assertEqual(list_row["customer_kpp"], organization.kpp)
+        self.assertEqual(list_row["customer_ogrn"], organization.ogrn)
+        self.assertEqual(detail_row["customer_name"], organization.name)
+        self.assertEqual(detail_row["customer_inn"], organization.inn)
+        self.assertEqual(detail_row["customer_kpp"], organization.kpp)
+        self.assertEqual(detail_row["customer_ogrn"], organization.ogrn)
+
+    def test_procurements_v1_uses_customer_kpp_for_branch_enrichment(self):
+        head = Organization.objects.create(
+            name='ООО "КАНОНИЧЕСКИЙ ЗАКАЗЧИК"',
+            inn="7701000101",
+            kpp="770101001",
+            ogrn="1027700000001",
+        )
+        branch = Organization.objects.create(
+            name='ООО "КАНОНИЧЕСКИЙ ЗАКАЗЧИК" ФИЛИАЛ',
+            inn=head.inn,
+            kpp="780101001",
+            ogrn=head.ogrn,
+        )
+        record = ProcurementRecord.objects.create(
+            load_batch=1,
+            purchase_number="0320100000126000002",
+            purchase_name="Поставка оборудования",
+            customer_inn=head.inn,
+            customer_kpp=branch.kpp,
+            customer_ogrn=head.ogrn,
+            customer_name="",
+            max_price="1000.00",
+            status="published",
+            law_type="44-FZ",
+            href="https://zakupki.gov.ru/example",
+            region_code="77",
+        )
+
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get(
+            reverse("api_v1:zakupki:procurements-detail", args=[record.id])
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["customer_name"], branch.name)
+        self.assertEqual(response.data["customer_kpp"], branch.kpp)
+
+    def test_procurements_v1_does_not_guess_branch_when_customer_kpp_is_missing(self):
+        head = Organization.objects.create(
+            name='ООО "КАНОНИЧЕСКИЙ ЗАКАЗЧИК"',
+            inn="7701000101",
+            kpp="770101001",
+            ogrn="1027700000001",
+        )
+        Organization.objects.create(
+            name='ООО "КАНОНИЧЕСКИЙ ЗАКАЗЧИК" ФИЛИАЛ',
+            inn=head.inn,
+            kpp="780101001",
+            ogrn=head.ogrn,
+        )
+        record = ProcurementRecord.objects.create(
+            load_batch=1,
+            purchase_number="0320100000126000003",
+            purchase_name="Поставка оборудования",
+            customer_inn=head.inn,
+            customer_kpp="",
+            customer_ogrn=head.ogrn,
+            customer_name="",
+            max_price="1000.00",
+            status="published",
+            law_type="44-FZ",
+            href="https://zakupki.gov.ru/example",
+            region_code="77",
+        )
+
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get(
+            reverse("api_v1:zakupki:procurements-detail", args=[record.id])
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["customer_name"], "")
+        self.assertEqual(response.data["customer_kpp"], "")
+
     def test_eis_result_endpoint_uses_generic_records_without_breaking_old_zakupki_api(
         self,
     ):
@@ -190,6 +317,46 @@ class ParsersViewSetTest(APITestCase):
         self.assertEqual(detail_response.status_code, status.HTTP_200_OK)
         self.assertEqual(detail_response.data["data"]["payload"], {"registry": "44fz"})
 
+    def test_parser_results_v1_enrich_missing_organization_fields_without_contract_change(
+        self,
+    ):
+        organization = Organization.objects.create(
+            name='ООО "КАНОНИЧЕСКИЙ ПОСТАВЩИК"',
+            inn="7701000102",
+            kpp="770102001",
+            ogrn="1027700000002",
+        )
+        generic_record = GenericParserRecord.objects.create(
+            load_batch=1,
+            source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
+            external_id="eis-44fz-enrichment",
+            inn="",
+            ogrn=organization.ogrn,
+            organisation_name="",
+            title="EIS notice",
+            status="published",
+            payload={"registry": "44fz"},
+        )
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get("/api/v1/parsers/results/procurements_44fz/")
+        detail_response = self.client.get(
+            f"/api/v1/parsers/results/procurements_44fz/{generic_record.id}/"
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(detail_response.status_code, status.HTTP_200_OK)
+        row = response.data["data"][0]
+        detail = detail_response.data["data"]
+        self.assertEqual(row["organisation_name"], organization.name)
+        self.assertEqual(row["inn"], organization.inn)
+        self.assertEqual(row["ogrn"], organization.ogrn)
+        self.assertNotIn("kpp", row)
+        self.assertEqual(detail["organisation_name"], organization.name)
+        self.assertEqual(detail["inn"], organization.inn)
+        self.assertEqual(detail["ogrn"], organization.ogrn)
+        self.assertNotIn("kpp", detail)
+
     def test_dashboard_data_exposes_source_groups_for_page(self):
         GenericParserRecord.objects.create(
             load_batch=1,
@@ -252,6 +419,197 @@ class ParsersViewSetTest(APITestCase):
         self.assertEqual(payload["source_counts"][ParserLoadLog.Source.TRUDVSEM], 1)
         self.assertEqual(payload["source_counts"][ParserLoadLog.Source.FNS_REPORTS], 1)
 
+    def test_dashboard_data_exposes_registry_data_coverage(self):
+        registry_organization = RegisterOrganizationFactory(
+            mn_inn=7720699480,
+            mn_ogrn=1107746880031,
+        )
+        RegistryMembershipPeriodFactory(organization=registry_organization)
+        FinancialReport.objects.create(
+            external_id=_digits(5),
+            ogrn=str(registry_organization.mn_ogrn),
+            file_name=f"fin_{_digits(5)}_{registry_organization.mn_ogrn}.xlsx",
+            file_hash=fake.sha256(raw_output=False),
+            load_batch=1,
+            status=FinancialReport.Status.SUCCESS,
+            source=FinancialReport.SourceType.API,
+        )
+        GenericParserRecord.objects.create(
+            load_batch=1,
+            source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
+            external_id="unfair-1",
+            title="Unfair supplier record",
+            inn=str(registry_organization.mn_inn),
+            ogrn=str(registry_organization.mn_ogrn),
+        )
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get("/api/v1/parsers/dashboard/")
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        coverage = response.data["data"]["registry_data_coverage"]
+        self.assertEqual(coverage["total_organizations"], 1)
+        items = {item["source"]: item for item in coverage["items"]}
+        self.assertEqual(items[ParserLoadLog.Source.FNS_REPORTS]["organizations_count"], 1)
+        self.assertEqual(items[ParserLoadLog.Source.FNS_REPORTS]["coverage_percent"], 100)
+        self.assertNotIn(ParserLoadLog.Source.UNFAIR_SUPPLIERS, items)
+
+    def test_dashboard_data_exposes_registry_enrichment_analytics(self):
+        rosatom_organization = RegisterOrganizationFactory(
+            mn_inn=7720699480,
+            mn_ogrn=1107746880031,
+        )
+        roscosmos_organization = RegisterOrganizationFactory(
+            mn_inn=7730239877,
+            mn_ogrn=1087302001797,
+        )
+        rosatom_membership = RegistryMembershipPeriodFactory(
+            organization=rosatom_organization
+        )
+        roscosmos_membership = RegistryMembershipPeriodFactory(
+            organization=roscosmos_organization
+        )
+        FinancialReport.objects.create(
+            external_id=_digits(5),
+            ogrn=str(rosatom_organization.mn_ogrn),
+            file_name=f"fin_{_digits(5)}_{rosatom_organization.mn_ogrn}.xlsx",
+            file_hash=fake.sha256(raw_output=False),
+            load_batch=1,
+            status=FinancialReport.Status.SUCCESS,
+            source=FinancialReport.SourceType.API,
+        )
+        IndustrialCertificateRecordFactory(
+            inn=str(rosatom_organization.mn_inn),
+            ogrn=str(rosatom_organization.mn_ogrn),
+        )
+        GenericParserRecord.objects.create(
+            load_batch=1,
+            source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
+            external_id="unfair-registry-analytics",
+            title="Risk signal",
+            inn=str(roscosmos_organization.mn_inn),
+            ogrn=str(roscosmos_organization.mn_ogrn),
+        )
+        GenericParserRecord.objects.create(
+            load_batch=1,
+            source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY,
+            external_id="bankruptcy-registry-analytics",
+            title="Bankruptcy risk signal",
+            inn=str(roscosmos_organization.mn_inn),
+            ogrn=str(roscosmos_organization.mn_ogrn),
+        )
+        InspectionRecordFactory(
+            inn=str(rosatom_organization.mn_inn),
+            ogrn=str(rosatom_organization.mn_ogrn),
+        )
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get("/api/v1/parsers/dashboard/")
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        analytics = response.data["data"]["registry_enrichment_analytics"]
+        self.assertEqual(
+            analytics["population"]["active_registry_organizations"],
+            2,
+        )
+        self.assertEqual(analytics["coverage_summary"]["with_any_enrichment"], 1)
+        self.assertEqual(analytics["coverage_summary"]["core_profile_complete"], 1)
+        self.assertEqual(analytics["coverage_summary"]["requires_attention"], 1)
+        source_coverage = {
+            item["source"]: item for item in analytics["source_coverage"]
+        }
+        self.assertEqual(
+            source_coverage[ParserLoadLog.Source.FNS_REPORTS][
+                "organizations_count"
+            ],
+            1,
+        )
+        self.assertEqual(
+            source_coverage[ParserLoadLog.Source.INDUSTRIAL][
+                "organizations_count"
+            ],
+            1,
+        )
+        self.assertNotIn(ParserLoadLog.Source.UNFAIR_SUPPLIERS, source_coverage)
+        self.assertNotIn(ParserLoadLog.Source.FEDRESURS_BANKRUPTCY, source_coverage)
+        self.assertNotIn(ParserLoadLog.Source.INSPECTIONS, source_coverage)
+        risk_signals = {item["source"]: item for item in analytics["risk_signals"]}
+        self.assertEqual(
+            risk_signals[ParserLoadLog.Source.UNFAIR_SUPPLIERS][
+                "organizations_count"
+            ],
+            1,
+        )
+        self.assertEqual(
+            risk_signals[ParserLoadLog.Source.UNFAIR_SUPPLIERS]["risk_severity"],
+            "critical",
+        )
+        self.assertEqual(
+            risk_signals[ParserLoadLog.Source.FEDRESURS_BANKRUPTCY][
+                "risk_severity"
+            ],
+            "critical",
+        )
+        self.assertEqual(
+            risk_signals[ParserLoadLog.Source.INSPECTIONS]["risk_severity"],
+            "soft",
+        )
+        self.assertEqual(
+            analytics["risk_summary"]["critical"]["organizations_count"],
+            1,
+        )
+        self.assertEqual(
+            analytics["risk_summary"]["soft"]["organizations_count"],
+            1,
+        )
+        matrix = {
+            item["registry_id"]: item
+            for item in analytics["registry_source_matrix"]
+        }
+        rosatom_matrix = matrix[str(rosatom_membership.registry_id)]
+        roscosmos_matrix = matrix[str(roscosmos_membership.registry_id)]
+        self.assertEqual(rosatom_matrix["active_organizations"], 1)
+        self.assertEqual(
+            rosatom_matrix["sources"][ParserLoadLog.Source.FNS_REPORTS][
+                "organizations_count"
+            ],
+            1,
+        )
+        self.assertEqual(
+            roscosmos_matrix["sources"][ParserLoadLog.Source.FNS_REPORTS][
+                "organizations_count"
+            ],
+            0,
+        )
+        total_matrix = analytics["registry_source_matrix"][-1]
+        self.assertEqual(total_matrix["registry_id"], "__all__")
+        self.assertEqual(total_matrix["registry_name"], "Все организации")
+        self.assertEqual(total_matrix["active_organizations"], 2)
+        self.assertEqual(
+            total_matrix["sources"][ParserLoadLog.Source.FNS_REPORTS][
+                "organizations_count"
+            ],
+            1,
+        )
+        self.assertEqual(
+            total_matrix["sources"][ParserLoadLog.Source.FNS_REPORTS][
+                "coverage_percent"
+            ],
+            50,
+        )
+        self.assertEqual(
+            total_matrix["sources"][ParserLoadLog.Source.INDUSTRIAL][
+                "organizations_count"
+            ],
+            1,
+        )
+        self.assertEqual(
+            total_matrix["sources"][ParserLoadLog.Source.INDUSTRIAL][
+                "coverage_percent"
+            ],
+            50,
+        )
+
     def test_financial_reports_list_and_retrieve(self):
         report = FinancialReport.objects.create(
             external_id=_digits(5),