feat: migrate parser data to source records

tests/apps/parsers/test_source_cards_views.py

@@ -6,19 +6,20 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from apps.core.models import BackgroundJob, JobStatus
-from apps.parsers.models import FinancialReport, FinancialReportLine, ParserLoadLog
+from apps.parsers.models import ParserLoadLog
+from apps.parsers.source_cards import SourceCardService
 from django.test import override_settings
 from django.urls import reverse
+from organizations.source_ingestion import (
+    OrganizationSourceIngestionService,
+    SourceRecordInput,
+)
+from registers.models import Register
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from tests.apps.parsers.factories import (
-    IndustrialCertificateRecordFactory,
-    IndustrialProductRecordFactory,
-    InspectionRecordFactory,
-    ManufacturerRecordFactory,
-    ParserLoadLogFactory,
-)
+from tests.apps.parsers.factories import ParserLoadLogFactory
+from tests.apps.registers.factories import RegistryMembershipPeriodFactory
 from tests.apps.user.factories import UserFactory
 from tests.utils.fixtures import fake
 
@@ -27,46 +28,58 @@ def _digits(length: int) -> str:
     return "".join(str(fake.random_int(0, 9)) for _ in range(length))
 
 
+def _save_source_record(
+    *,
+    source: str,
+    external_id: str,
+    inn: str = "",
+    ogrn: str = "",
+    organization_name: str = "",
+    title: str = "",
+) -> None:
+    OrganizationSourceIngestionService.save_records(
+        source=source,
+        load_batch=1,
+        records=[
+            SourceRecordInput(
+                external_id=external_id,
+                title=title,
+                organization_name=organization_name or title or external_id,
+                inn=inn,
+                ogrn=ogrn,
+            )
+        ],
+    )
+
+
 class SourceCardsApiTestCase(APITestCase):
     def setUp(self):
+        SourceCardService.clear_cache()
         self.user = UserFactory.create_user()
         self.admin = UserFactory.create_user(is_staff=True)
         self.client.force_authenticate(self.user)
 
     def test_source_cards_list_returns_aggregated_data(self):
-        report = FinancialReport.objects.create(
+        report_ogrn = _digits(13)
+        _save_source_record(
+            source=ParserLoadLog.Source.FNS_REPORTS,
             external_id=_digits(5),
-            ogrn=_digits(13),
-            file_name=f"fin_{_digits(5)}_{_digits(13)}.xlsx",
-            file_hash=fake.sha256(raw_output=False),
-            load_batch=1,
-            status=FinancialReport.Status.SUCCESS,
-            source=FinancialReport.SourceType.API,
+            ogrn=report_ogrn,
+            organization_name='ООО "Финансовая компания"',
+            title=f"fin_{_digits(5)}_{report_ogrn}.xlsx",
         )
-        FinancialReportLine.objects.create(
-            report=report,
-            form_code="1",
-            line_code="1100",
-            line_name="Активы",
-            year=2025,
-            period_start=100,
-            period_end=200,
-        )
-        FinancialReportLine.objects.create(
-            report=report,
-            form_code="2",
-            line_code="2110",
-            line_name="Выручка",
-            year=2025,
-            period_start=300,
-            period_end=400,
+        _save_source_record(
+            source=ParserLoadLog.Source.FNS_REPORTS,
+            external_id=_digits(5),
+            ogrn=report_ogrn,
+            organization_name='ООО "Финансовая компания"',
+            title=f"fin_{_digits(5)}_{report_ogrn}.xlsx",
         )
         ParserLoadLogFactory(
             source=ParserLoadLog.Source.FNS_REPORTS,
             status="success",
             records_count=2,
         )
-        InspectionRecordFactory()
         BackgroundJob.objects.create(
             task_id="job-inspections-active",
             task_name="apps.parsers.tasks.sync_inspections",
@@ -96,11 +109,73 @@ class SourceCardsApiTestCase(APITestCase):
         self.assertEqual(inspections_card["status"], "in_progress")
         self.assertEqual(inspections_card["progress"], 63)
 
+    def test_main_dashboard_returns_cached_source_cards_and_registry_stats(self):
+        registry, _created = Register.objects.get_or_create(
+            name="Реестр предприятий ОПК",
+        )
+        RegistryMembershipPeriodFactory(
+            registry=registry,
+        )
+        ParserLoadLogFactory(
+            source=ParserLoadLog.Source.FNS_REPORTS,
+            status="success",
+            records_count=1,
+        )
+        SourceCardService.clear_cache()
+
+        response = self.client.get(reverse("api_v1:stat:main-dashboard"))
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertTrue(response.data["success"])
+        self.assertEqual(response["X-Cache"], "MISS")
+
+        data = response.data["data"]
+        self.assertIn("source_cards", data)
+        self.assertIn("organization_stats", data)
+        self.assertGreaterEqual(len(data["source_cards"]), 1)
+        self.assertEqual(data["organization_stats"]["counts"]["opk"], 1)
+        self.assertEqual(data["cache_ttl_seconds"], 604800)
+
+        cached_response = self.client.get(reverse("api_v1:stat:main-dashboard"))
+        self.assertEqual(cached_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(cached_response["X-Cache"], "HIT")
+
+    def test_main_dashboard_cache_is_warmed_after_successful_parser_load(self):
+        with self.captureOnCommitCallbacks(execute=True):
+            ParserLoadLogFactory(
+                source=ParserLoadLog.Source.FNS_REPORTS,
+                status="success",
+                records_count=1,
+            )
+
+        response = self.client.get(reverse("api_v1:stat:main-dashboard"))
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response["X-Cache"], "HIT")
+
     def test_source_card_detail_returns_combined_minprom_stats(self):
         shared_inn = _digits(10)
-        IndustrialCertificateRecordFactory(inn=shared_inn)
-        IndustrialProductRecordFactory(inn=shared_inn)
-        ManufacturerRecordFactory(inn=shared_inn)
+        _save_source_record(
+            source=ParserLoadLog.Source.INDUSTRIAL,
+            external_id="industrial-1",
+            inn=shared_inn,
+            organization_name='ООО "Производитель"',
+            title="Сертификат промышленной продукции",
+        )
+        _save_source_record(
+            source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS,
+            external_id="product-1",
+            inn=shared_inn,
+            organization_name='ООО "Производитель"',
+            title="Промышленная продукция",
+        )
+        _save_source_record(
+            source=ParserLoadLog.Source.MANUFACTURES,
+            external_id="manufacturer-1",
+            inn=shared_inn,
+            organization_name='ООО "Производитель"',
+            title="Производитель",
+        )
         ParserLoadLogFactory(
             source=ParserLoadLog.Source.INDUSTRIAL,
             status="success",