feat: expand registry ingestion and demo exchange

src/organizations/test_companies.py

@@ -0,0 +1,539 @@
+"""Deterministic, realistic test-company dataset for frontend demonstrations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import date
+from decimal import Decimal
+from hashlib import sha256
+from uuid import UUID, uuid5
+
+from apps.parsers.models import ParserLoadLog
+from django.db.models import Count, Max, Min
+from django.utils import timezone
+
+from organizations.cache import invalidate_organization_api_cache
+from organizations.models import (
+    Organization,
+    OrganizationSourceFinancialLine,
+    OrganizationSourceRecord,
+    SourceExtensionStatus,
+)
+from organizations.source_cache import invalidate_source_data_cache
+from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS
+
+TEST_COMPANY_COUNT = 20
+TEST_COMPANY_NAMESPACE = UUID("59b36ae9-bcf8-4b7c-b77a-77578f485a01")
+TEST_RECORD_PREFIX = "mostovik-test-company"
+
+
+@dataclass(frozen=True)
+class TestCompanyDatasetResult:
+    """Counters returned by create and delete operations."""
+
+    organizations_created: int = 0
+    organizations_updated: int = 0
+    organizations_deleted: int = 0
+    extensions: int = 0
+    records: int = 0
+
+
+class TestCompanyDatasetService:
+    """Create, refresh, and remove the fixed frontend demonstration dataset."""
+
+    @classmethod
+    def company_uids(cls) -> list[UUID]:
+        return [
+            uuid5(TEST_COMPANY_NAMESPACE, f"company:{index}")
+            for index in range(1, TEST_COMPANY_COUNT + 1)
+        ]
+
+    @classmethod
+    def create_or_update(cls) -> TestCompanyDatasetResult:
+        now = timezone.now()
+        created_count = 0
+        updated_count = 0
+        total_extensions = 0
+        total_records = 0
+
+        descriptors_by_group = cls._descriptors_by_group()
+        for index, uid in enumerate(cls.company_uids(), start=1):
+            organization, created = Organization.objects.update_or_create(
+                uid=uid,
+                defaults=cls._organization_defaults(index=index, now=now),
+            )
+            created_count += int(created)
+            updated_count += int(not created)
+
+            extensions = {}
+            for source_group, descriptors in descriptors_by_group.items():
+                descriptor = descriptors[0]
+                extension, _ = descriptor.extension_model.objects.update_or_create(
+                    organization=organization,
+                    defaults={
+                        "title": descriptor.title,
+                        "status": SourceExtensionStatus.ACTIVE,
+                        "first_seen_at": now,
+                        "last_seen_at": now,
+                        "last_load_batch": 9_000_000 + index,
+                        "metadata": {
+                            "dataset": TEST_RECORD_PREFIX,
+                            "sources": [item.source for item in descriptors],
+                        },
+                    },
+                )
+                extensions[source_group] = extension
+
+            expected_external_ids = []
+            for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items():
+                if source not in ParserLoadLog.Source.values:
+                    continue
+                external_id = f"{TEST_RECORD_PREFIX}:{index:02d}:{source}"
+                expected_external_ids.append(external_id)
+                record_defaults = cls._record_defaults(
+                    source=source,
+                    index=index,
+                    organization=organization,
+                )
+                record, _ = OrganizationSourceRecord.objects.update_or_create(
+                    source=source,
+                    external_id=external_id,
+                    defaults={
+                        "extension": extensions[descriptor.source_group],
+                        "record_type": descriptor.record_type,
+                        "load_batch": 9_000_000 + index,
+                        **record_defaults,
+                    },
+                )
+                if source == ParserLoadLog.Source.FNS_REPORTS:
+                    cls._refresh_financial_lines(record=record, index=index)
+
+            OrganizationSourceRecord.objects.filter(
+                extension__organization=organization,
+                external_id__startswith=f"{TEST_RECORD_PREFIX}:",
+            ).exclude(external_id__in=expected_external_ids).delete()
+            cls._refresh_extension_counters(organization=organization)
+            total_extensions += len(extensions)
+            total_records += len(expected_external_ids)
+
+        cls._invalidate_caches()
+        return TestCompanyDatasetResult(
+            organizations_created=created_count,
+            organizations_updated=updated_count,
+            extensions=total_extensions,
+            records=total_records,
+        )
+
+    @classmethod
+    def delete(cls) -> TestCompanyDatasetResult:
+        company_uids = cls.company_uids()
+        queryset = Organization.objects.filter(uid__in=company_uids)
+        deleted_count = queryset.count()
+        extension_models = {
+            descriptor.extension_model
+            for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items()
+            if source in ParserLoadLog.Source.values
+        }
+        for extension_model in extension_models:
+            extension_model.objects.filter(organization_id__in=company_uids).delete()
+        queryset.delete()
+        cls._invalidate_caches()
+        return TestCompanyDatasetResult(organizations_deleted=deleted_count)
+
+    @staticmethod
+    def _descriptors_by_group():
+        descriptors = {}
+        for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items():
+            if source in ParserLoadLog.Source.values:
+                descriptors.setdefault(descriptor.source_group, []).append(descriptor)
+        return descriptors
+
+    @classmethod
+    def _organization_defaults(cls, *, index: int, now):
+        is_rosatom = index <= 10
+        corporation_name = (
+            'Госкорпорация "Росатом"' if is_rosatom else 'Госкорпорация "Роскосмос"'
+        )
+        corporation_ministry = (
+            'Государственная корпорация по атомной энергии "Росатом"'
+            if is_rosatom
+            else 'Государственная корпорация по космической деятельности "Роскосмос"'
+        )
+        industry = (
+            "Атомная промышленность"
+            if is_rosatom
+            else "Ракетно-космическая промышленность"
+        )
+        name = f"Тестовая компания {index}"
+        inn = cls._legal_entity_inn(index)
+        ogrn = cls._legal_entity_ogrn(index)
+        return {
+            "rn": 9_900_000 + index,
+            "gk_code": "2" if is_rosatom else "1",
+            "gk_name": corporation_name,
+            "in_korp_code": "1",
+            "in_korp_name": "Входит в состав",
+            "name": name,
+            "full_name": f'Акционерное общество "{name}"',
+            "short_name": name,
+            "pn_name": name,
+            "pn_name_en": f"Test Company {index}",
+            "inn": inn,
+            "kpp": f"7709{index:02d}001",
+            "ogrn": ogrn,
+            "okpo": f"90{index:06d}",
+            "filial": ".F.",
+            "is_branch": False,
+            "registration_date": date(2010 + index % 10, (index - 1) % 12 + 1, 15),
+            "create_date": str(2010 + index % 10),
+            "organizational_legal_form": "12267",
+            "organizational_legal_form1": "Непубличные акционерные общества",
+            "ownership_form": "61" if is_rosatom else "16",
+            "ownership_form1": (
+                "Собственность государственных корпораций"
+                if is_rosatom
+                else "Частная собственность"
+            ),
+            "authorized_capital": Decimal(1_000_000 + index * 100_000),
+            "legal_address": (
+                f"{101000 + index}, г. Москва, Тестовый проезд, д. {index}"
+            ),
+            "business_act_cod": "4" if is_rosatom else "2",
+            "business_activity": "Прочая" if is_rosatom else "Производственная",
+            "general_director": f"Иванов Иван Иванович {index}",
+            "general_director_tax_id": f"770100{index:06d}",
+            "appointment_date": date(2024, 1, min(index, 28)),
+            "akc_fs": "0",
+            "akc_sf": "0",
+            "re_za": False,
+            "re_zasf": False,
+            "goz_participation": True,
+            "opk_registry_membership": True,
+            "ropk_num": f"ТЕСТ-ОПК-{index:04d}",
+            "ropk_razdel_num": "3" if is_rosatom else "2",
+            "ropk_razdel_name": (
+                "Организации, подведомственные и находящиеся в сфере деятельности "
+                f"{corporation_name}"
+            ),
+            "min": corporation_ministry,
+            "dep": "Департамент тестовых данных",
+            "otr": industry,
+            "integrated_structure": f"Тестовая интегрированная структура {index}",
+            "state_sector_code": "2",
+            "state_sector_name": (
+                "контролируются РФ косвенно или совместно с организациями госсектора"
+            ),
+            "directory_source_file_hash": sha256(
+                f"{TEST_RECORD_PREFIX}:{index}".encode()
+            ).hexdigest(),
+            "directory_source_row_number": index,
+            "directory_imported_at": now,
+        }
+
+    @classmethod
+    def _record_defaults(cls, *, source: str, index: int, organization):
+        common = {
+            "inn": organization.inn,
+            "ogrn": organization.ogrn,
+            "organisation_name": organization.name,
+            "source": source,
+            "load_batch": 9_000_000 + index,
+        }
+        url = f"https://example.test/{source}/{index}"
+        values = {
+            ParserLoadLog.Source.FNS_REPORTS: {
+                "title": f"Бухгалтерская отчетность за 2025 год — {organization.name}",
+                "record_date": "2025",
+                "status": "processed",
+                "payload": {
+                    **common,
+                    "file_name": f"fin_test_{organization.ogrn}.xlsx",
+                    "file_hash": sha256(
+                        f"financial:{organization.ogrn}".encode()
+                    ).hexdigest(),
+                    "lines_count": 4,
+                },
+            },
+            ParserLoadLog.Source.PROCUREMENTS: cls._procurement_values(
+                common, index, organization, law="44-ФЗ/223-ФЗ", source=source
+            ),
+            ParserLoadLog.Source.PROCUREMENTS_44FZ: cls._procurement_values(
+                common, index, organization, law="44-ФЗ", source=source
+            ),
+            ParserLoadLog.Source.PROCUREMENTS_223FZ: cls._procurement_values(
+                common, index, organization, law="223-ФЗ", source=source
+            ),
+            ParserLoadLog.Source.CONTRACTS: {
+                "title": f"Контракт на поставку оборудования № {index}",
+                "record_date": "15.03.2026",
+                "amount": Decimal(3_000_000 + index * 25_000),
+                "status": "Исполнение",
+                "url": url,
+                "payload": {
+                    **common,
+                    "contract_number": f"КОНТРАКТ-{index:05d}",
+                    "subject": "Поставка испытательного оборудования",
+                    "customer": organization.name,
+                    "price": str(3_000_000 + index * 25_000),
+                    "law": "44-ФЗ",
+                    "status": "Исполнение",
+                    "url": url,
+                },
+            },
+            ParserLoadLog.Source.INDUSTRIAL: {
+                "title": f"Сертификат промышленного производства {index:05d}/26",
+                "record_date": "15.01.2026",
+                "url": url,
+                "payload": {
+                    **common,
+                    "certificate_number": f"{index:05d}/26",
+                    "issue_date": "15.01.2026",
+                    "issue_date_normalized": "2026-01-15",
+                    "expiry_date": "15.01.2029",
+                    "expiry_date_normalized": "2029-01-15",
+                    "certificate_file_url": url,
+                },
+            },
+            ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: {
+                "title": f"Испытательный комплекс ТК-{index}",
+                "payload": {
+                    **common,
+                    "full_organisation_name": organization.full_name,
+                    "registry_number": f"ПРОД-{index:06d}",
+                    "product_name": f"Испытательный комплекс ТК-{index}",
+                    "product_model": f"ТК-{index}.2026",
+                    "okpd2_code": "28.99.39.190",
+                    "tnved_code": "8479899707",
+                    "regulatory_document": "ТУ 28.99.39-001-2026",
+                },
+            },
+            ParserLoadLog.Source.MANUFACTURES: {
+                "title": organization.full_name,
+                "payload": {
+                    **common,
+                    "full_legal_name": organization.full_name,
+                    "address": organization.legal_address,
+                },
+            },
+            ParserLoadLog.Source.INSPECTIONS: {
+                "title": f"Плановая выездная проверка {organization.name}",
+                "record_date": "01.06.2026",
+                "status": "Запланирована",
+                "payload": {
+                    **common,
+                    "registration_number": f"24800000{index:04d}",
+                    "control_authority": "Ростехнадзор",
+                    "inspection_type": "scheduled",
+                    "inspection_form": "documentary_and_on_site",
+                    "legal_basis": "Федеральный закон № 248-ФЗ",
+                    "start_date": "01.06.2026",
+                    "start_date_normalized": "2026-06-01",
+                    "end_date": "15.06.2026",
+                    "end_date_normalized": "2026-06-15",
+                    "data_year": 2026,
+                    "data_month": 6,
+                    "status": "Запланирована",
+                },
+            },
+            ParserLoadLog.Source.FEDRESURS_BANKRUPTCY: {
+                "title": f"Сообщение по делу А40-{10000 + index}/2026",
+                "record_date": "01.05.2026",
+                "status": "Наблюдение",
+                "url": url,
+                "payload": {
+                    **common,
+                    "case_number": f"А40-{10000 + index}/2026",
+                    "message_type": "Введение наблюдения",
+                    "message_date": "01.05.2026",
+                    "messages_count": 3,
+                    "messages": [
+                        {
+                            "type": "Введение наблюдения",
+                            "date": "01.05.2026",
+                        }
+                    ],
+                    "url": url,
+                },
+            },
+            ParserLoadLog.Source.UNFAIR_SUPPLIERS: {
+                "title": f"Реестровая запись РНП-{index:05d}",
+                "record_date": "20.01.2026",
+                "status": "Включен",
+                "url": url,
+                "payload": {
+                    **common,
+                    "supplier": organization.name,
+                    "registry_number": f"РНП-{index:05d}",
+                    "included_at": "20.01.2026",
+                    "reason": "Нарушение условий закупки",
+                    "url": url,
+                },
+            },
+            ParserLoadLog.Source.FAS_GOZ: {
+                "title": f"Постановление ФАС по ГОЗ № {index}",
+                "record_date": "20.01.2026",
+                "status": "Исполнено",
+                "payload": {
+                    **common,
+                    "registry_number": str(index),
+                    "authority": "ФАС России",
+                    "decision": "Постановление по ГОЗ",
+                    "decision_date": "20.01.2026",
+                    "execution_status": "Исполнено",
+                },
+            },
+            ParserLoadLog.Source.ARBITRATION: {
+                "title": f"Дело А40-{20000 + index}/2026",
+                "record_date": "10.02.2026",
+                "amount": Decimal(500_000 + index * 10_000),
+                "status": "В производстве",
+                "url": url,
+                "payload": {
+                    **common,
+                    "case_number": f"А40-{20000 + index}/2026",
+                    "court": "Арбитражный суд города Москвы",
+                    "role": "ответчик",
+                    "claim_amount": str(500_000 + index * 10_000),
+                    "status": "В производстве",
+                    "url": url,
+                },
+            },
+            ParserLoadLog.Source.FSTEC: {
+                "title": f"Сертификат ФСТЭК № ТЕСТ-{index:04d}",
+                "record_date": "15.01.2026",
+                "status": "Действует",
+                "payload": {
+                    **common,
+                    "licensee": organization.name,
+                    "registry_number": f"ТЕСТ-{index:04d}",
+                    "issued_at": "15.01.2026",
+                    "status": "Действует",
+                    "№ сертификата": f"ТЕСТ-{index:04d}",
+                    "Заявитель": organization.name,
+                    "Дата внесения в реестр": "15.01.2026",
+                    "Срок действия сертификата": "15.01.2029",
+                },
+            },
+            ParserLoadLog.Source.TRUDVSEM: {
+                "title": f"Инженер-испытатель {index} категории",
+                "record_date": "20.05.2026",
+                "status": "Открыта",
+                "url": url,
+                "payload": {
+                    **common,
+                    "id": f"VAC-{index:05d}",
+                    "company": organization.name,
+                    "job-name": f"Инженер-испытатель {index} категории",
+                    "creation-date": "2026-05-20",
+                    "salary": "120000.00",
+                    "salary_min": 110000,
+                    "salary_max": 140000,
+                    "currency": "RUB",
+                    "employment": "Полная занятость",
+                    "schedule": "Полный день",
+                    "region": "Москва",
+                    "status": "Открыта",
+                    "vacancy_source": "trudvsem",
+                    "vac_url": url,
+                },
+            },
+        }
+        return values[source]
+
+    @staticmethod
+    def _procurement_values(common, index, organization, *, law: str, source: str):
+        purchase_number = f"03732000000{index:08d}"[:19]
+        amount = Decimal(1_250_000 + index * 50_000)
+        url = f"https://example.test/{source}/{index}"
+        return {
+            "title": f"Поставка оборудования для {organization.name}",
+            "record_date": "01.02.2026",
+            "amount": amount,
+            "status": "Размещено",
+            "url": url,
+            "payload": {
+                **common,
+                "purchase_number": purchase_number,
+                "subject": "Поставка испытательного оборудования",
+                "customer": organization.name,
+                "customer_inn": organization.inn,
+                "price": str(amount),
+                "law": law,
+                "published_at": "01.02.2026",
+                "status": "Размещено",
+                "region_code": "77",
+                "data_year": 2026,
+                "data_month": 2,
+                "url": url,
+            },
+        }
+
+    @staticmethod
+    def _refresh_financial_lines(*, record, index: int) -> None:
+        expected = {
+            ("1", "1600", "Баланс (актив)", 10_000_000 + index * 100_000),
+            ("1", "1300", "Капитал и резервы", 4_000_000 + index * 50_000),
+            ("2", "2110", "Выручка", 20_000_000 + index * 200_000),
+            ("2", "2400", "Чистая прибыль", 2_000_000 + index * 25_000),
+        }
+        expected_keys = []
+        for form_code, line_code, line_name, period_end in expected:
+            expected_keys.append((form_code, line_code, 2025))
+            OrganizationSourceFinancialLine.objects.update_or_create(
+                source_record=record,
+                form_code=form_code,
+                line_code=line_code,
+                year=2025,
+                defaults={
+                    "line_name": line_name,
+                    "period_start": period_end - 100_000,
+                    "period_end": period_end,
+                },
+            )
+        lines = OrganizationSourceFinancialLine.objects.filter(source_record=record)
+        for line in lines:
+            key = (line.form_code, line.line_code, line.year)
+            if key not in expected_keys:
+                line.delete()
+
+    @staticmethod
+    def _refresh_extension_counters(*, organization) -> None:
+        aggregates = {
+            item["extension_id"]: item
+            for item in OrganizationSourceRecord.objects.filter(
+                extension__organization=organization
+            )
+            .values("extension_id")
+            .annotate(
+                records_count=Count("uid"),
+                first_seen_at=Min("created_at"),
+                last_seen_at=Max("updated_at"),
+            )
+        }
+        for extension in organization.source_extensions.all():
+            aggregate = aggregates.get(extension.uid, {})
+            extension.records_count = aggregate.get("records_count", 0)
+            extension.first_seen_at = aggregate.get("first_seen_at")
+            extension.last_seen_at = aggregate.get("last_seen_at")
+            extension.save(
+                update_fields=["records_count", "first_seen_at", "last_seen_at"]
+            )
+
+    @staticmethod
+    def _legal_entity_inn(index: int) -> str:
+        base = f"770900{index:03d}"
+        weights = (2, 4, 10, 3, 5, 9, 4, 6, 8)
+        checksum = sum(
+            int(digit) * weight for digit, weight in zip(base, weights, strict=True)
+        )
+        return f"{base}{checksum % 11 % 10}"
+
+    @staticmethod
+    def _legal_entity_ogrn(index: int) -> str:
+        base = f"1267700{index:05d}"
+        return f"{base}{int(base) % 11 % 10}"
+
+    @staticmethod
+    def _invalidate_caches() -> None:
+        invalidate_organization_api_cache()
+        invalidate_source_data_cache()