"""Deterministic, realistic test-company dataset for frontend demonstrations.""" from __future__ import annotations from dataclasses import dataclass from datetime import date from decimal import Decimal from hashlib import sha256 from uuid import UUID, uuid5 from apps.parsers.models import ( FinancialReport, FinancialReportLine, GenericParserRecord, IndustrialCertificateRecord, IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, ProcurementRecord, ) from django.db.models import Count, Max, Min from django.utils import timezone from organizations.cache import invalidate_organization_api_cache from organizations.models import ( Organization, OrganizationSourceFinancialLine, OrganizationSourceRecord, SourceExtensionStatus, ) from organizations.source_cache import invalidate_source_data_cache from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS TEST_COMPANY_COUNT = 20 TEST_FINANCIAL_HISTORY_YEARS = 4 TEST_COMPANY_NAMESPACE = UUID("59b36ae9-bcf8-4b7c-b77a-77578f485a01") TEST_RECORD_PREFIX = "mostovik-test-company" @dataclass(frozen=True) class TestCompanyDatasetResult: """Counters returned by create and delete operations.""" organizations_created: int = 0 organizations_updated: int = 0 organizations_deleted: int = 0 extensions: int = 0 records: int = 0 class TestCompanyDatasetService: """Create, refresh, and remove the fixed frontend demonstration dataset.""" @classmethod def company_uids(cls) -> list[UUID]: return [ uuid5(TEST_COMPANY_NAMESPACE, f"company:{index}") for index in range(1, TEST_COMPANY_COUNT + 1) ] @classmethod def create_or_update(cls) -> TestCompanyDatasetResult: now = timezone.now() created_count = 0 updated_count = 0 total_extensions = 0 total_records = 0 descriptors_by_group = cls._descriptors_by_group() for index, uid in enumerate(cls.company_uids(), start=1): organization, created = Organization.objects.update_or_create( uid=uid, defaults=cls._organization_defaults(index=index, now=now), ) created_count += int(created) updated_count += int(not created) extensions = {} for source_group, descriptors in descriptors_by_group.items(): descriptor = descriptors[0] extension, _ = descriptor.extension_model.objects.update_or_create( organization=organization, defaults={ "title": descriptor.title, "status": SourceExtensionStatus.ACTIVE, "first_seen_at": now, "last_seen_at": now, "last_load_batch": 9_000_000 + index, "metadata": { "dataset": TEST_RECORD_PREFIX, "sources": [item.source for item in descriptors], }, }, ) extensions[source_group] = extension expected_external_ids = [] for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items(): if source not in ParserLoadLog.Source.values: continue external_id = f"{TEST_RECORD_PREFIX}:{index:02d}:{source}" expected_external_ids.append(external_id) record_defaults = cls._record_defaults( source=source, index=index, organization=organization, ) record, _ = OrganizationSourceRecord.objects.update_or_create( source=source, external_id=external_id, defaults={ "extension": extensions[descriptor.source_group], "record_type": descriptor.record_type, "load_batch": 9_000_000 + index, **record_defaults, }, ) legacy_record = cls._sync_parser_result_record( source=source, external_id=external_id, organization=organization, defaults=record_defaults, index=index, ) legacy_module = legacy_record.__class__.__module__.removesuffix( ".models" ) record.legacy_model = ( f"{legacy_module}.{legacy_record.__class__.__name__}" ) record.legacy_pk = str(legacy_record.pk) record.save(update_fields=["legacy_model", "legacy_pk", "updated_at"]) if source == ParserLoadLog.Source.FNS_REPORTS: cls._refresh_financial_lines( record=record, legacy_report=legacy_record, index=index, ) OrganizationSourceRecord.objects.filter( extension__organization=organization, external_id__startswith=f"{TEST_RECORD_PREFIX}:", ).exclude(external_id__in=expected_external_ids).delete() cls._refresh_extension_counters(organization=organization) total_extensions += len(extensions) total_records += len(expected_external_ids) cls._invalidate_caches() return TestCompanyDatasetResult( organizations_created=created_count, organizations_updated=updated_count, extensions=total_extensions, records=total_records, ) @classmethod def delete(cls) -> TestCompanyDatasetResult: company_uids = cls.company_uids() queryset = Organization.objects.filter(uid__in=company_uids) deleted_count = queryset.count() cls._delete_parser_result_records(company_uids) extension_models = { descriptor.extension_model for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items() if source in ParserLoadLog.Source.values } for extension_model in extension_models: extension_model.objects.filter(organization_id__in=company_uids).delete() queryset.delete() cls._invalidate_caches() return TestCompanyDatasetResult(organizations_deleted=deleted_count) @classmethod def _sync_parser_result_record( cls, *, source: str, external_id: str, organization: Organization, defaults: dict, index: int, ): """Mirror demo rows into the parser tables served by public source APIs.""" payload = defaults.get("payload", {}) load_batch = 9_000_000 + index common = { "load_batch": load_batch, "registry_organization": organization, } if source == ParserLoadLog.Source.INDUSTRIAL: legacy_record, _ = IndustrialCertificateRecord.objects.update_or_create( certificate_number=payload["certificate_number"], defaults={ **common, "issue_date": payload["issue_date"], "issue_date_normalized": date.fromisoformat( payload["issue_date_normalized"] ), "expiry_date": payload["expiry_date"], "expiry_date_normalized": date.fromisoformat( payload["expiry_date_normalized"] ), "certificate_file_url": payload["certificate_file_url"], "organisation_name": organization.name, "inn": organization.inn, "ogrn": organization.ogrn, }, ) return legacy_record if source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: legacy_record, _ = IndustrialProductRecord.objects.update_or_create( registry_number=payload["registry_number"], defaults={ **common, "full_organisation_name": payload["full_organisation_name"], "ogrn": organization.ogrn, "inn": organization.inn, "product_name": payload["product_name"], "product_model": payload["product_model"], "okpd2_code": payload["okpd2_code"], "tnved_code": payload["tnved_code"], "regulatory_document": payload["regulatory_document"], }, ) return legacy_record if source == ParserLoadLog.Source.MANUFACTURES: legacy_record, _ = ManufacturerRecord.objects.update_or_create( inn=organization.inn, defaults={ **common, "full_legal_name": payload["full_legal_name"], "ogrn": organization.ogrn, "address": payload["address"], }, ) return legacy_record if source == ParserLoadLog.Source.INSPECTIONS: legacy_record, _ = InspectionRecord.objects.update_or_create( registration_number=payload["registration_number"], defaults={ **common, "inn": organization.inn, "ogrn": organization.ogrn, "organisation_name": organization.name, "control_authority": payload["control_authority"], "inspection_type": payload["inspection_type"], "inspection_form": payload["inspection_form"], "start_date": payload["start_date"], "start_date_normalized": date.fromisoformat( payload["start_date_normalized"] ), "end_date": payload["end_date"], "end_date_normalized": date.fromisoformat( payload["end_date_normalized"] ), "status": payload["status"], "legal_basis": payload["legal_basis"], "is_federal_law_248": True, "data_year": payload["data_year"], "data_month": payload["data_month"], }, ) return legacy_record if source == ParserLoadLog.Source.PROCUREMENTS: legacy_record, _ = ProcurementRecord.objects.update_or_create( purchase_number=payload["purchase_number"], defaults={ **common, "purchase_name": payload["subject"], "customer_inn": organization.inn, "customer_kpp": organization.kpp, "customer_ogrn": organization.ogrn, "customer_name": organization.name, "max_price": payload["price"], "max_price_amount": defaults["amount"], "publish_date": payload["published_at"], "publish_date_normalized": date(2026, 2, 1), "status": payload["status"], "law_type": payload["law"], "purchase_object_info": payload["subject"], "href": payload["url"], "region_code": payload["region_code"], "data_year": payload["data_year"], "data_month": payload["data_month"], }, ) return legacy_record if source == ParserLoadLog.Source.FNS_REPORTS: legacy_record, _ = FinancialReport.objects.update_or_create( file_hash=payload["file_hash"], defaults={ **common, "external_id": external_id, "ogrn": organization.ogrn, "file_name": payload["file_name"], "status": FinancialReport.Status.SUCCESS, "source": FinancialReport.SourceType.API, }, ) return legacy_record legacy_record, _ = GenericParserRecord.objects.update_or_create( source=source, external_id=external_id, defaults={ **common, "inn": organization.inn, "ogrn": organization.ogrn, "organisation_name": organization.name, "title": defaults.get("title", ""), "record_date": defaults.get("record_date", ""), "amount": defaults.get("amount"), "status": defaults.get("status", ""), "url": defaults.get("url", ""), "payload": payload, }, ) return legacy_record @staticmethod def _delete_parser_result_records(company_uids: list[UUID]) -> None: for model in ( GenericParserRecord, IndustrialCertificateRecord, IndustrialProductRecord, ManufacturerRecord, InspectionRecord, ProcurementRecord, FinancialReport, ): model.objects.filter(registry_organization_id__in=company_uids).delete() @staticmethod def _descriptors_by_group(): descriptors = {} for source, descriptor in SOURCE_GROUP_DESCRIPTORS.items(): if source in ParserLoadLog.Source.values: descriptors.setdefault(descriptor.source_group, []).append(descriptor) return descriptors @classmethod def _organization_defaults(cls, *, index: int, now): is_rosatom = index <= 10 corporation_name = ( 'Госкорпорация "Росатом"' if is_rosatom else 'Госкорпорация "Роскосмос"' ) corporation_ministry = ( 'Государственная корпорация по атомной энергии "Росатом"' if is_rosatom else 'Государственная корпорация по космической деятельности "Роскосмос"' ) industry = ( "Атомная промышленность" if is_rosatom else "Ракетно-космическая промышленность" ) name = f"Тестовая компания {index}" inn = cls._legal_entity_inn(index) ogrn = cls._legal_entity_ogrn(index) return { "rn": 9_900_000 + index, "gk_code": "2" if is_rosatom else "1", "gk_name": corporation_name, "in_korp_code": "1", "in_korp_name": "Входит в состав", "name": name, "full_name": f'Акционерное общество "{name}"', "short_name": name, "pn_name": name, "pn_name_en": f"Test Company {index}", "inn": inn, "kpp": f"7709{index:02d}001", "ogrn": ogrn, "okpo": f"90{index:06d}", "filial": ".F.", "is_branch": False, "registration_date": date(2010 + index % 10, (index - 1) % 12 + 1, 15), "create_date": str(2010 + index % 10), "organizational_legal_form": "12267", "organizational_legal_form1": "Непубличные акционерные общества", "ownership_form": "61" if is_rosatom else "16", "ownership_form1": ( "Собственность государственных корпораций" if is_rosatom else "Частная собственность" ), "authorized_capital": Decimal(1_000_000 + index * 100_000), "legal_address": ( f"{101000 + index}, г. Москва, Тестовый проезд, д. {index}" ), "business_act_cod": "4" if is_rosatom else "2", "business_activity": "Прочая" if is_rosatom else "Производственная", "general_director": f"Иванов Иван Иванович {index}", "general_director_tax_id": f"770100{index:06d}", "appointment_date": date(2024, 1, min(index, 28)), "akc_fs": "0", "akc_sf": "0", "re_za": False, "re_zasf": False, "goz_participation": True, "opk_registry_membership": True, "ropk_num": f"ТЕСТ-ОПК-{index:04d}", "ropk_razdel_num": "3" if is_rosatom else "2", "ropk_razdel_name": ( "Организации, подведомственные и находящиеся в сфере деятельности " f"{corporation_name}" ), "min": corporation_ministry, "dep": "Департамент тестовых данных", "otr": industry, "integrated_structure": f"Тестовая интегрированная структура {index}", "state_sector_code": "2", "state_sector_name": ( "контролируются РФ косвенно или совместно с организациями госсектора" ), "directory_source_file_hash": sha256( f"{TEST_RECORD_PREFIX}:{index}".encode() ).hexdigest(), "directory_source_row_number": index, "directory_imported_at": now, } @classmethod def _record_defaults(cls, *, source: str, index: int, organization): report_year = timezone.localdate().year common = { "inn": organization.inn, "ogrn": organization.ogrn, "organisation_name": organization.name, "source": source, "load_batch": 9_000_000 + index, } url = f"https://example.test/{source}/{index}" values = { ParserLoadLog.Source.FNS_REPORTS: { "title": ( f"Бухгалтерская отчетность за {report_year} год — " f"{organization.name}" ), "record_date": str(report_year), "status": "processed", "payload": { **common, "file_name": f"fin_test_{organization.ogrn}.xlsx", "file_hash": sha256( f"financial:{organization.ogrn}".encode() ).hexdigest(), "lines_count": 4, }, }, ParserLoadLog.Source.PROCUREMENTS: cls._procurement_values( common, index, organization, law="44-ФЗ/223-ФЗ", source=source ), ParserLoadLog.Source.PROCUREMENTS_44FZ: cls._procurement_values( common, index, organization, law="44-ФЗ", source=source ), ParserLoadLog.Source.PROCUREMENTS_223FZ: cls._procurement_values( common, index, organization, law="223-ФЗ", source=source ), ParserLoadLog.Source.CONTRACTS: { "title": f"Контракт на поставку оборудования № {index}", "record_date": "15.03.2026", "amount": Decimal(3_000_000 + index * 25_000), "status": "Исполнение", "url": url, "payload": { **common, "contract_number": f"КОНТРАКТ-{index:05d}", "subject": "Поставка испытательного оборудования", "customer": organization.name, "price": str(3_000_000 + index * 25_000), "law": "44-ФЗ", "status": "Исполнение", "url": url, }, }, ParserLoadLog.Source.INDUSTRIAL: { "title": f"Сертификат промышленного производства {index:05d}/26", "record_date": "15.01.2026", "url": url, "payload": { **common, "certificate_number": f"{index:05d}/26", "issue_date": "15.01.2026", "issue_date_normalized": "2026-01-15", "expiry_date": "15.01.2029", "expiry_date_normalized": "2029-01-15", "certificate_file_url": url, }, }, ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: { "title": f"Испытательный комплекс ТК-{index}", "payload": { **common, "full_organisation_name": organization.full_name, "registry_number": f"ПРОД-{index:06d}", "product_name": f"Испытательный комплекс ТК-{index}", "product_model": f"ТК-{index}.2026", "okpd2_code": "28.99.39.190", "tnved_code": "8479899707", "regulatory_document": "ТУ 28.99.39-001-2026", }, }, ParserLoadLog.Source.MANUFACTURES: { "title": organization.full_name, "payload": { **common, "full_legal_name": organization.full_name, "address": organization.legal_address, }, }, ParserLoadLog.Source.INSPECTIONS: { "title": f"Плановая выездная проверка {organization.name}", "record_date": "01.06.2026", "status": "Запланирована", "payload": { **common, "registration_number": f"24800000{index:04d}", "control_authority": "Ростехнадзор", "inspection_type": "scheduled", "inspection_form": "documentary_and_on_site", "legal_basis": "Федеральный закон № 248-ФЗ", "start_date": "01.06.2026", "start_date_normalized": "2026-06-01", "end_date": "15.06.2026", "end_date_normalized": "2026-06-15", "data_year": 2026, "data_month": 6, "status": "Запланирована", }, }, ParserLoadLog.Source.FEDRESURS_BANKRUPTCY: { "title": f"Сообщение по делу А40-{10000 + index}/2026", "record_date": "01.05.2026", "status": "Наблюдение", "url": url, "payload": { **common, "case_number": f"А40-{10000 + index}/2026", "message_type": "Введение наблюдения", "message_date": "01.05.2026", "messages_count": 3, "messages": [ { "type": "Введение наблюдения", "date": "01.05.2026", } ], "url": url, }, }, ParserLoadLog.Source.UNFAIR_SUPPLIERS: { "title": f"Реестровая запись РНП-{index:05d}", "record_date": "20.01.2026", "status": "Включен", "url": url, "payload": { **common, "supplier": organization.name, "registry_number": f"РНП-{index:05d}", "included_at": "20.01.2026", "reason": "Нарушение условий закупки", "url": url, }, }, ParserLoadLog.Source.FAS_GOZ: { "title": f"Постановление ФАС по ГОЗ № {index}", "record_date": "20.01.2026", "status": "Исполнено", "payload": { **common, "registry_number": str(index), "authority": "ФАС России", "decision": "Постановление по ГОЗ", "decision_date": "20.01.2026", "execution_status": "Исполнено", }, }, ParserLoadLog.Source.ARBITRATION: { "title": f"Дело А40-{20000 + index}/2026", "record_date": "10.02.2026", "amount": Decimal(500_000 + index * 10_000), "status": "В производстве", "url": url, "payload": { **common, "case_number": f"А40-{20000 + index}/2026", "court": "Арбитражный суд города Москвы", "role": "ответчик", "claim_amount": str(500_000 + index * 10_000), "status": "В производстве", "url": url, }, }, ParserLoadLog.Source.FSTEC: { "title": f"Сертификат ФСТЭК № ТЕСТ-{index:04d}", "record_date": "15.01.2026", "status": "Действует", "payload": { **common, "licensee": organization.name, "registry_number": f"ТЕСТ-{index:04d}", "issued_at": "15.01.2026", "status": "Действует", "№ сертификата": f"ТЕСТ-{index:04d}", "Заявитель": organization.name, "Дата внесения в реестр": "15.01.2026", "Срок действия сертификата": "15.01.2029", }, }, ParserLoadLog.Source.TRUDVSEM: { "title": f"Инженер-испытатель {index} категории", "record_date": "20.05.2026", "status": "Открыта", "url": url, "payload": { **common, "id": f"VAC-{index:05d}", "company": organization.name, "job-name": f"Инженер-испытатель {index} категории", "creation-date": "2026-05-20", "salary": "120000.00", "salary_min": 110000, "salary_max": 140000, "currency": "RUB", "employment": "Полная занятость", "schedule": "Полный день", "region": "Москва", "status": "Открыта", "vacancy_source": "trudvsem", "vac_url": url, }, }, } return values[source] @staticmethod def _procurement_values(common, index, organization, *, law: str, source: str): purchase_number = f"03732000000{index:08d}"[:19] amount = Decimal(1_250_000 + index * 50_000) url = f"https://example.test/{source}/{index}" return { "title": f"Поставка оборудования для {organization.name}", "record_date": "01.02.2026", "amount": amount, "status": "Размещено", "url": url, "payload": { **common, "purchase_number": purchase_number, "subject": "Поставка испытательного оборудования", "customer": organization.name, "customer_inn": organization.inn, "price": str(amount), "law": law, "published_at": "01.02.2026", "status": "Размещено", "region_code": "77", "data_year": 2026, "data_month": 2, "url": url, }, } @staticmethod def _refresh_financial_lines(*, record, legacy_report, index: int) -> None: current_year = timezone.localdate().year report_years = range( current_year - TEST_FINANCIAL_HISTORY_YEARS + 1, current_year + 1, ) expected = ( ( "1", "1600", "Баланс (актив)", 10_000_000 + index * 100_000, 300_000 + index * 2_000, ), ( "1", "1300", "Капитал и резервы", 4_000_000 + index * 50_000, 120_000 + index * 1_000, ), ( "2", "2110", "Выручка", 20_000_000 + index * 200_000, 750_000 + index * 5_000, ), ( "2", "2400", "Чистая прибыль", 2_000_000 + index * 25_000, 80_000 + index * 500, ), ) expected_keys = set() for report_year in report_years: years_before_current = current_year - report_year for ( form_code, line_code, line_name, current_period_end, annual_change, ) in expected: period_end = current_period_end - annual_change * years_before_current expected_keys.add((form_code, line_code, report_year)) defaults = { "line_name": line_name, "period_start": period_end - annual_change, "period_end": period_end, } OrganizationSourceFinancialLine.objects.update_or_create( source_record=record, form_code=form_code, line_code=line_code, year=report_year, defaults=defaults, ) FinancialReportLine.objects.update_or_create( report=legacy_report, form_code=form_code, line_code=line_code, year=report_year, defaults=defaults, ) lines = OrganizationSourceFinancialLine.objects.filter(source_record=record) for line in lines: key = (line.form_code, line.line_code, line.year) if key not in expected_keys: line.delete() for line in legacy_report.lines.all(): key = (line.form_code, line.line_code, line.year) if key not in expected_keys: line.delete() @staticmethod def _refresh_extension_counters(*, organization) -> None: aggregates = { item["extension_id"]: item for item in OrganizationSourceRecord.objects.filter( extension__organization=organization ) .values("extension_id") .annotate( records_count=Count("uid"), first_seen_at=Min("created_at"), last_seen_at=Max("updated_at"), ) } for extension in organization.source_extensions.all(): aggregate = aggregates.get(extension.uid, {}) extension.records_count = aggregate.get("records_count", 0) extension.first_seen_at = aggregate.get("first_seen_at") extension.last_seen_at = aggregate.get("last_seen_at") extension.save( update_fields=["records_count", "first_seen_at", "last_seen_at"] ) @staticmethod def _legal_entity_inn(index: int) -> str: base = f"770900{index:03d}" weights = (2, 4, 10, 3, 5, 9, 4, 6, 8) checksum = sum( int(digit) * weight for digit, weight in zip(base, weights, strict=True) ) return f"{base}{checksum % 11 % 10}" @staticmethod def _legal_entity_ogrn(index: int) -> str: base = f"1267700{index:05d}" return f"{base}{int(base) % 11 % 10}" @staticmethod def _invalidate_caches() -> None: invalidate_organization_api_cache() invalidate_source_data_cache()