feat: migrate parser data to source records

2026-05-19 20:21:31 +02:00
parent 1c7c7238be
commit b8a18d6da4
46 changed files with 2689 additions and 6179 deletions
--- a/tests/apps/parsers/test_procurement_service.py
+++ b/tests/apps/parsers/test_procurement_service.py
@@ -5,10 +5,11 @@ Unit-тесты для ProcurementService.
 """

 from apps.parsers.clients.zakupki.schemas import Procurement
-from apps.parsers.models import ProcurementRecord
+from apps.parsers.models import ParserLoadLog, ProcurementRecord
 from apps.parsers.services import ProcurementService
 from apps.registers.models import Organization
 from django.test import TestCase
+from organizations.models import OrganizationSourceRecord

 from tests.apps.parsers.factories import ProcurementRecordFactory, fake

@@ -21,6 +22,11 @@ def _region_code() -> str:
    return str(fake.random_int(min=1, max=99)).zfill(2)


+def _other_region_code(region_code: str) -> str:
+    next_region = int(region_code) % 99 + 1
+    return str(next_region).zfill(2)
+
+
 def _period() -> tuple[int, int]:
    return fake.random_int(min=2020, max=2025), fake.random_int(min=1, max=12)

@@ -61,6 +67,31 @@ def _build_procurement(**overrides) -> Procurement:
    return Procurement(**data)


+def _procurement_records():
+    return OrganizationSourceRecord.objects.filter(
+        source=ParserLoadLog.Source.PROCUREMENTS,
+    )
+
+
+def _save_procurement_source_record(
+    *,
+    batch_id: int = 1,
+    region_code: str | None = None,
+    data_year: int | None = None,
+    data_month: int | None = None,
+    **overrides,
+) -> Procurement:
+    procurement = _build_procurement(**overrides)
+    ProcurementService.save_procurements(
+        [procurement],
+        batch_id=batch_id,
+        region_code=region_code,
+        data_year=data_year,
+        data_month=data_month,
+    )
+    return procurement
+
+
 class ProcurementServiceSaveTestCase(TestCase):
    """Тесты метода save_procurements."""

@@ -90,14 +121,15 @@ class ProcurementServiceSaveTestCase(TestCase):
        )

        self.assertEqual(saved, 1)
-        self.assertEqual(ProcurementRecord.objects.count(), 1)
+        self.assertEqual(ProcurementRecord.objects.count(), 0)
+        self.assertEqual(_procurement_records().count(), 1)

-        record = ProcurementRecord.objects.first()
-        self.assertEqual(record.purchase_number, purchase_number)
-        self.assertEqual(record.customer_inn, customer_inn)
-        self.assertEqual(record.region_code, region_code)
-        self.assertEqual(record.data_year, year)
-        self.assertEqual(record.data_month, month)
+        record = _procurement_records().first()
+        self.assertEqual(record.external_id, purchase_number)
+        self.assertEqual(record.extension.organization.inn, customer_inn)
+        self.assertEqual(record.payload["region_code"], region_code)
+        self.assertEqual(record.payload["data_year"], year)
+        self.assertEqual(record.payload["data_month"], month)

    def test_save_multiple_procurements(self):
        """Сохранение нескольких закупок."""
@@ -106,7 +138,8 @@ class ProcurementServiceSaveTestCase(TestCase):
        saved = ProcurementService.save_procurements(procurements, batch_id=1)

        self.assertEqual(saved, 5)
-        self.assertEqual(ProcurementRecord.objects.count(), 5)
+        self.assertEqual(ProcurementRecord.objects.count(), 0)
+        self.assertEqual(_procurement_records().count(), 5)

    def test_save_links_registry_organization_when_exists(self):
        """При совпадении ИНН/ОГРН должна ставиться связь с registers.Organization."""
@@ -123,17 +156,15 @@ class ProcurementServiceSaveTestCase(TestCase):
        saved = ProcurementService.save_procurements([procurement], batch_id=1)

        self.assertEqual(saved, 1)
-        record = ProcurementRecord.objects.get(purchase_number=purchase_number)
+        record = _procurement_records().get(external_id=purchase_number)
        self.assertEqual(record.registry_organization_id, organization.id)

    def test_save_updates_duplicates(self):
        """Повторная синхронизация обновляет существующую закупку."""
-        # Создаём существующую запись
        purchase_number = _digits(19)
-        ProcurementRecordFactory(purchase_number=purchase_number)
-        original = ProcurementRecord.objects.get(purchase_number=purchase_number)
+        _save_procurement_source_record(purchase_number=purchase_number, batch_id=1)
+        original = _procurement_records().get(external_id=purchase_number)

-        # Пытаемся сохранить с тем же номером
        procurement = _build_procurement(
            purchase_number=purchase_number,
            customer_inn=_digits(10),
@@ -141,13 +172,13 @@ class ProcurementServiceSaveTestCase(TestCase):

        saved = ProcurementService.save_procurements([procurement], batch_id=2)

-        # Существующая запись обновляется в пределах той же строки
        self.assertEqual(saved, 1)
-        self.assertEqual(ProcurementRecord.objects.count(), 1)
-        refreshed = ProcurementRecord.objects.get(purchase_number=purchase_number)
-        self.assertEqual(refreshed.customer_inn, procurement.customer_inn)
+        self.assertEqual(ProcurementRecord.objects.count(), 0)
+        self.assertEqual(_procurement_records().count(), 1)
+        refreshed = _procurement_records().get(external_id=purchase_number)
+        self.assertEqual(refreshed.extension.organization.inn, procurement.customer_inn)
        self.assertEqual(refreshed.load_batch, 2)
-        self.assertEqual(refreshed.id, original.id)
+        self.assertEqual(refreshed.uid, original.uid)

    def test_save_with_chunking(self):
        """Сохранение большого количества записей чанками."""
@@ -158,7 +189,8 @@ class ProcurementServiceSaveTestCase(TestCase):
        )

        self.assertEqual(saved, 100)
-        self.assertEqual(ProcurementRecord.objects.count(), 100)
+        self.assertEqual(ProcurementRecord.objects.count(), 0)
+        self.assertEqual(_procurement_records().count(), 100)


 class ProcurementServiceFindTestCase(TestCase):
@@ -174,26 +206,26 @@ class ProcurementServiceFindTestCase(TestCase):
            self.region_b = _region_code()
        self.name_key = fake.word()
        self.unique_token = fake.word()
-        self.record1 = ProcurementRecordFactory(
+        self.record1 = _save_procurement_source_record(
            purchase_number=_digits(19),
            customer_inn=self.inn_target,
            customer_name=f"{self.unique_token} {self.name_key} {fake.company()}",
            region_code=self.region_a,
-            load_batch=1,
+            batch_id=1,
        )
-        self.record2 = ProcurementRecordFactory(
+        self.record2 = _save_procurement_source_record(
            purchase_number=_digits(19),
            customer_inn=self.inn_other,
            customer_name=f"{self.name_key} {fake.company()}",
            region_code=self.region_a,
-            load_batch=1,
+            batch_id=1,
        )
-        self.record3 = ProcurementRecordFactory(
+        self.record3 = _save_procurement_source_record(
            purchase_number=_digits(19),
            customer_inn=self.inn_target,  # Тот же ИНН что и у первого
            customer_name=f"{self.name_key} {fake.company()}",
            region_code=self.region_b,
-            load_batch=2,
+            batch_id=2,
        )

    def test_find_by_inn(self):
@@ -205,7 +237,7 @@ class ProcurementServiceFindTestCase(TestCase):
        """Поиск по ИНН с фильтром по batch."""
        results = ProcurementService.find_by_inn(self.inn_target, batch_id=1)
        self.assertEqual(results.count(), 1)
-        self.assertEqual(results.first().purchase_number, self.record1.purchase_number)
+        self.assertEqual(results.first().external_id, self.record1.purchase_number)

    def test_find_by_purchase_number(self):
        """Поиск по номеру закупки."""
@@ -213,7 +245,10 @@ class ProcurementServiceFindTestCase(TestCase):
            self.record2.purchase_number
        )
        self.assertEqual(results.count(), 1)
-        self.assertEqual(results.first().customer_inn, self.record2.customer_inn)
+        self.assertEqual(
+            results.first().extension.organization.inn,
+            self.record2.customer_inn,
+        )

    def test_find_by_region(self):
        """Поиск по региону."""
@@ -247,7 +282,7 @@ class ProcurementServicePeriodTestCase(TestCase):
        """Получение последнего загруженного периода."""
        periods = [_period() for _ in range(3)]
        for year, month in periods:
-            ProcurementRecordFactory(data_year=year, data_month=month)
+            _save_procurement_source_record(data_year=year, data_month=month)
        expected_year, expected_month = max(periods)

        year, month = ProcurementService.get_last_loaded_period()
@@ -263,10 +298,10 @@ class ProcurementServicePeriodTestCase(TestCase):
            region_b = _region_code()
        period_a = _period()
        period_b = _period()
-        ProcurementRecordFactory(
+        _save_procurement_source_record(
            data_year=period_a[0], data_month=period_a[1], region_code=region_a
        )
-        ProcurementRecordFactory(
+        _save_procurement_source_record(
            data_year=period_b[0], data_month=period_b[1], region_code=region_b
        )

@@ -281,10 +316,10 @@ class ProcurementServicePeriodTestCase(TestCase):
        law_type_b = _other_law(law_type_a)
        period_a = _period()
        period_b = _period()
-        ProcurementRecordFactory(
+        _save_procurement_source_record(
            data_year=period_a[0], data_month=period_a[1], law_type=law_type_a
        )
-        ProcurementRecordFactory(
+        _save_procurement_source_record(
            data_year=period_b[0], data_month=period_b[1], law_type=law_type_b
        )

@@ -296,7 +331,7 @@ class ProcurementServicePeriodTestCase(TestCase):
    def test_has_data_for_period_true(self):
        """Проверка наличия данных за период - есть данные."""
        year, month = _period()
-        ProcurementRecordFactory(data_year=year, data_month=month)
+        _save_procurement_source_record(data_year=year, data_month=month)

        result = ProcurementService.has_data_for_period(year, month)

@@ -305,7 +340,7 @@ class ProcurementServicePeriodTestCase(TestCase):
    def test_has_data_for_period_false(self):
        """Проверка наличия данных за период - нет данных."""
        year, month = _period()
-        ProcurementRecordFactory(data_year=year, data_month=month)
+        _save_procurement_source_record(data_year=year, data_month=month)
        other_month = month % 12 + 1

        result = ProcurementService.has_data_for_period(year, other_month)
@@ -317,7 +352,7 @@ class ProcurementServicePeriodTestCase(TestCase):
        year, month = _period()
        region_code = _region_code()
        law_type = fake.random_element(["44-FZ", "223-FZ"])
-        ProcurementRecordFactory(
+        _save_procurement_source_record(
            data_year=year,
            data_month=month,
            region_code=region_code,
@@ -334,7 +369,7 @@ class ProcurementServicePeriodTestCase(TestCase):
        # С неправильным регионом - нет
        self.assertFalse(
            ProcurementService.has_data_for_period(
-                year, month, region_code=_region_code()
+                year, month, region_code=_other_region_code(region_code)
            )
        )