From 6ba8fa8d88a04e20661cff742be467e20cd46694 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Sun, 7 Jun 2026 16:04:43 +0200 Subject: [PATCH] feat: use authoritative organization directory --- src/apps/parsers/admin.py | 6 +- .../migrations/0024_auto_20260607_1017.py | 174 +++++++ src/apps/parsers/models.py | 42 +- src/apps/parsers/services.py | 92 +++- src/apps/parsers/tasks.py | 76 +-- src/apps/parsers/views.py | 121 +++-- src/core/api_v1_urls.py | 6 - src/core/api_v2_urls.py | 2 - src/organizations/api_enrichment.py | 79 ++-- src/organizations/directory_import.py | 447 ++++++++++++++++++ src/organizations/filters.py | 61 +-- .../commands/import_organization_directory.py | 35 ++ .../migrations/0007_auto_20260607_1017.py | 263 +++++++++++ src/organizations/models.py | 372 ++++++++++++--- src/organizations/resolver.py | 168 +++++++ src/organizations/serializers.py | 230 ++++----- src/organizations/services.py | 115 +---- src/organizations/signals.py | 68 +-- src/organizations/source_backfill.py | 33 +- src/organizations/source_ingestion.py | 131 +++-- src/organizations/views.py | 40 +- tests/apps/organizations/test_api_v2.py | 94 ++-- .../test_api_v2_source_extensions.py | 57 +-- .../organizations/test_directory_import.py | 218 +++++++++ .../organizations/test_populate_command.py | 116 +---- tests/apps/organizations/test_resolver.py | 102 ++++ .../organizations/test_source_backfill.py | 32 +- .../organizations/test_source_ingestion.py | 116 ++++- tests/apps/parsers/test_admin.py | 19 +- .../parsers/test_direct_ingestion_services.py | 20 + tests/apps/parsers/test_e2e.py | 10 +- tests/apps/parsers/test_fns_parser.py | 43 +- tests/apps/parsers/test_fns_upload.py | 24 +- .../apps/parsers/test_procurement_service.py | 32 +- tests/apps/parsers/test_service_helpers.py | 66 ++- tests/apps/parsers/test_services.py | 14 +- .../apps/parsers/test_source_cards_service.py | 19 +- tests/apps/parsers/test_source_cards_views.py | 69 ++- tests/apps/parsers/test_sources_api_e2e.py | 22 +- tests/apps/parsers/test_tasks.py | 137 +++++- tests/apps/parsers/test_views.py | 68 ++- 41 files changed, 2781 insertions(+), 1058 deletions(-) create mode 100644 src/apps/parsers/migrations/0024_auto_20260607_1017.py create mode 100644 src/organizations/directory_import.py create mode 100644 src/organizations/management/commands/import_organization_directory.py create mode 100644 src/organizations/migrations/0007_auto_20260607_1017.py create mode 100644 src/organizations/resolver.py create mode 100644 tests/apps/organizations/test_directory_import.py create mode 100644 tests/apps/organizations/test_resolver.py diff --git a/src/apps/parsers/admin.py b/src/apps/parsers/admin.py index 24353bc..a5c6fef 100644 --- a/src/apps/parsers/admin.py +++ b/src/apps/parsers/admin.py @@ -755,9 +755,9 @@ class FinancialReportAdmin(admin.ModelAdmin): "external_id", "ogrn", "file_name", - "registry_organization__pn_name", - "registry_organization__mn_ogrn", - "registry_organization__mn_inn", + "registry_organization__name", + "registry_organization__ogrn", + "registry_organization__inn", ] list_select_related = ["registry_organization"] readonly_fields = [ diff --git a/src/apps/parsers/migrations/0024_auto_20260607_1017.py b/src/apps/parsers/migrations/0024_auto_20260607_1017.py new file mode 100644 index 0000000..015b3fe --- /dev/null +++ b/src/apps/parsers/migrations/0024_auto_20260607_1017.py @@ -0,0 +1,174 @@ +# Generated by Django 3.2.25 on 2026-06-07 10:17 + +from django.db import migrations, models +import django.db.models.deletion + + +PARSER_ORGANIZATION_FK_TABLES = ( + "parsers_financial_report", + "parsers_generic_record", + "parsers_industrial_certificate", + "parsers_industrial_product", + "parsers_inspection", + "parsers_manufacturer", + "parsers_procurement", +) + + +def migrate_parser_organization_fks(apps, schema_editor): + """Move parser FK columns from registers.Organization ids to Organization UUIDs.""" + if schema_editor.connection.vendor != "postgresql": + return + + with schema_editor.connection.cursor() as cursor: + for table_name in PARSER_ORGANIZATION_FK_TABLES: + quoted_table = schema_editor.quote_name(table_name) + temp_column = "registry_organization_uuid" + quoted_temp_column = schema_editor.quote_name(temp_column) + quoted_fk_column = schema_editor.quote_name("registry_organization_id") + fk_name = f"{table_name}_registry_organization_id_fk" + index_name = f"{table_name}_registry_organization_id_idx" + + cursor.execute( + f""" + DO $$ + DECLARE constraint_name text; + BEGIN + FOR constraint_name IN + SELECT tc.constraint_name + FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = current_schema() + AND tc.table_name = '{table_name}' + AND kcu.column_name = 'registry_organization_id' + LOOP + EXECUTE format( + 'ALTER TABLE %I DROP CONSTRAINT %I', + '{table_name}', + constraint_name + ); + END LOOP; + END $$; + """ + ) + cursor.execute( + f"ALTER TABLE {quoted_table} " + f"ADD COLUMN IF NOT EXISTS {quoted_temp_column} uuid NULL" + ) + cursor.execute( + f""" + UPDATE {quoted_table} parser_row + SET {quoted_temp_column} = ( + SELECT org.uid AS organization_uid + FROM registers_organization registry_org + JOIN organizations_organization org + ON ( + (registry_org.mn_okpo <> '' AND org.okpo = registry_org.mn_okpo) + OR ( + org.inn = registry_org.mn_inn::text + AND registry_org.in_kpp IS NOT NULL + AND org.kpp = registry_org.in_kpp::text + ) + OR ( + org.ogrn = registry_org.mn_ogrn::text + AND registry_org.in_kpp IS NOT NULL + AND org.kpp = registry_org.in_kpp::text + ) + OR ( + org.inn = registry_org.mn_inn::text + AND org.ogrn = registry_org.mn_ogrn::text + ) + ) + WHERE registry_org.id = parser_row.{quoted_fk_column} + ORDER BY + CASE + WHEN registry_org.mn_okpo <> '' AND org.okpo = registry_org.mn_okpo THEN 0 + WHEN registry_org.in_kpp IS NOT NULL AND org.inn = registry_org.mn_inn::text AND org.kpp = registry_org.in_kpp::text THEN 1 + WHEN registry_org.in_kpp IS NOT NULL AND org.ogrn = registry_org.mn_ogrn::text AND org.kpp = registry_org.in_kpp::text THEN 2 + WHEN org.is_branch = false THEN 3 + ELSE 4 + END, + org.name, + org.uid + LIMIT 1 + ) + WHERE parser_row.{quoted_fk_column} IS NOT NULL + """ + ) + cursor.execute( + f"ALTER TABLE {quoted_table} DROP COLUMN {quoted_fk_column}" + ) + cursor.execute( + f"ALTER TABLE {quoted_table} " + f"RENAME COLUMN {quoted_temp_column} TO registry_organization_id" + ) + cursor.execute( + f"CREATE INDEX IF NOT EXISTS {schema_editor.quote_name(index_name)} " + f"ON {quoted_table} ({quoted_fk_column})" + ) + cursor.execute( + f"ALTER TABLE {quoted_table} " + f"ADD CONSTRAINT {schema_editor.quote_name(fk_name)} " + f"FOREIGN KEY ({quoted_fk_column}) " + f"REFERENCES organizations_organization (uid) " + f"DEFERRABLE INITIALLY DEFERRED" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('organizations', '0007_auto_20260607_1017'), + ('parsers', '0023_add_job_board_generic_record_sources'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunPython( + migrate_parser_organization_fks, + reverse_code=migrations.RunPython.noop, + ), + ], + state_operations=[ + migrations.AlterField( + model_name='financialreport', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='financial_reports', to='organizations.organization', verbose_name='организация'), + ), + migrations.AlterField( + model_name='genericparserrecord', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='generic_parser_records', to='organizations.organization', verbose_name='организация'), + ), + migrations.AlterField( + model_name='industrialcertificaterecord', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='industrial_certificate_records', to='organizations.organization', verbose_name='организация'), + ), + migrations.AlterField( + model_name='industrialproductrecord', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='industrial_product_records', to='organizations.organization', verbose_name='организация'), + ), + migrations.AlterField( + model_name='inspectionrecord', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='inspection_records', to='organizations.organization', verbose_name='организация'), + ), + migrations.AlterField( + model_name='manufacturerrecord', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='manufacturer_records', to='organizations.organization', verbose_name='организация'), + ), + migrations.AlterField( + model_name='procurementrecord', + name='registry_organization', + field=models.ForeignKey(blank=True, help_text='Связь с верхнеуровневой организацией', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='procurement_records', to='organizations.organization', verbose_name='организация'), + ), + ], + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 7968e19..70433ca 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -190,13 +190,13 @@ class IndustrialCertificateRecord(TimestampMixin, models.Model): help_text=_("ОГРН организации"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="industrial_certificate_records", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) class Meta: @@ -253,13 +253,13 @@ class ManufacturerRecord(TimestampMixin, models.Model): help_text=_("Юридический адрес организации"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="manufacturer_records", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) class Meta: @@ -344,13 +344,13 @@ class IndustrialProductRecord(TimestampMixin, models.Model): help_text=_("Наименование нормативного документа"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="industrial_product_records", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) class Meta: @@ -445,13 +445,13 @@ class GenericParserRecord(TimestampMixin, models.Model): help_text=_("Нормализованный исходный документ"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="generic_parser_records", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) class Meta: @@ -658,13 +658,13 @@ class InspectionRecord(TimestampMixin, models.Model): help_text=_("Месяц, за который загружены данные"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="inspection_records", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) class Meta: @@ -834,13 +834,13 @@ class ProcurementRecord(TimestampMixin, models.Model): help_text=_("Месяц, за который загружены данные"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="procurement_records", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) class Meta: @@ -897,13 +897,13 @@ class FinancialReport(TimestampMixin, models.Model): help_text=_("ОГРН организации"), ) registry_organization = models.ForeignKey( - "registers.Organization", + "organizations.Organization", on_delete=models.SET_NULL, null=True, blank=True, related_name="financial_reports", - verbose_name=_("организация из реестров"), - help_text=_("Связь с организацией из приложения реестров"), + verbose_name=_("организация"), + help_text=_("Связь с верхнеуровневой организацией"), ) file_name = models.CharField( _("имя файла"), diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index 67454db..56da735 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -41,13 +41,13 @@ from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import Q from django.utils import timezone -from organizations.models import OrganizationSourceRecord +from organizations.models import Organization, OrganizationSourceRecord from organizations.source_ingestion import ( + OrganizationSourceIngestionResult, OrganizationSourceIngestionService, SourceFinancialLineInput, SourceRecordInput, ) -from registers.models import Organization logger = logging.getLogger(__name__) @@ -141,16 +141,16 @@ def _optional_int(value: object) -> int | None: class RegistryOrganizationLookup: """Набор индексов для быстрого сопоставления parser-записей с организацией.""" - by_pair: dict[tuple[int, int], int] - by_inn: dict[int, int] - by_ogrn: dict[int, int] + by_pair: dict[tuple[str, str], str] + by_inn: dict[str, str] + by_ogrn: dict[str, str] class RegistryOrganizationResolver: """Разрешает связь parser-записей с канонической организацией из registers.""" @staticmethod - def normalize_identifier(value: str | int | None) -> int | None: + def normalize_identifier(value: str | int | None) -> str | None: if value is None: return None @@ -158,15 +158,15 @@ class RegistryOrganizationResolver: if not as_str or not as_str.isdigit(): return None - return int(as_str) + return as_str @classmethod - def build_lookup( + def build_lookup( # noqa: C901 cls, identifiers: list[tuple[str | int | None, str | int | None]], ) -> RegistryOrganizationLookup: - inn_values: set[int] = set() - ogrn_values: set[int] = set() + inn_values: set[str] = set() + ogrn_values: set[str] = set() for inn_raw, ogrn_raw in identifiers: inn = cls.normalize_identifier(inn_raw) @@ -181,29 +181,45 @@ class RegistryOrganizationResolver: filters = Q() if inn_values: - filters |= Q(mn_inn__in=inn_values) + filters |= Q(inn__in=inn_values) if ogrn_values: - filters |= Q(mn_ogrn__in=ogrn_values) + filters |= Q(ogrn__in=ogrn_values) organizations = Organization.objects.filter(filters).values( - "id", - "mn_inn", - "mn_ogrn", + "uid", + "inn", + "ogrn", + "is_branch", ) - by_pair: dict[tuple[int, int], int] = {} - by_inn_candidates: dict[int, list[int]] = defaultdict(list) - by_ogrn_candidates: dict[int, list[int]] = defaultdict(list) + by_pair_candidates: dict[tuple[str, str], list[dict]] = defaultdict(list) + by_inn_candidates: dict[str, list[str]] = defaultdict(list) + by_ogrn_candidates: dict[str, list[str]] = defaultdict(list) for organization in organizations: - organization_id = organization["id"] - inn = organization["mn_inn"] - ogrn = organization["mn_ogrn"] + organization_id = str(organization["uid"]) + inn = organization["inn"] + ogrn = organization["ogrn"] - by_pair[(inn, ogrn)] = organization_id + if inn and ogrn: + by_pair_candidates[(inn, ogrn)].append(organization) by_inn_candidates[inn].append(organization_id) by_ogrn_candidates[ogrn].append(organization_id) + by_pair = {} + for key, candidates in by_pair_candidates.items(): + unique_ids = {str(candidate["uid"]) for candidate in candidates} + if len(unique_ids) == 1: + by_pair[key] = next(iter(unique_ids)) + continue + head_ids = { + str(candidate["uid"]) + for candidate in candidates + if candidate["is_branch"] is False + } + if len(head_ids) == 1: + by_pair[key] = next(iter(head_ids)) + by_inn = { inn: ids[0] for inn, ids in by_inn_candidates.items() if len(set(ids)) == 1 } @@ -226,7 +242,7 @@ class RegistryOrganizationResolver: lookup: RegistryOrganizationLookup, inn: str | int | None, ogrn: str | int | None, - ) -> int | None: + ) -> str | None: normalized_inn = cls.normalize_identifier(inn) normalized_ogrn = cls.normalize_identifier(ogrn) @@ -1876,6 +1892,28 @@ class ProcurementService(BulkOperationsMixin, BaseService[ProcurementRecord]): return qs +class FNSReportOrganizationResolutionSkipped(ValueError): + """Raised when an FNS report is intentionally skipped by organization resolver.""" + + def __init__( + self, + *, + external_id: str, + ogrn: str, + ingestion_result: OrganizationSourceIngestionResult, + ) -> None: + self.external_id = external_id + self.ogrn = ogrn + self.ingestion_result = ingestion_result + self.reason = ( + "ambiguous" if ingestion_result.skipped_ambiguous else "unmatched" + ) + super().__init__( + "FNS report skipped because organization resolution is " + f"{self.reason}: external_id={external_id}, ogrn={ogrn}" + ) + + class FNSReportService(BulkOperationsMixin, BaseService[OrganizationSourceRecord]): """ Сервис для работы с бухгалтерской отчетностью ФНС. @@ -1934,7 +1972,7 @@ class FNSReportService(BulkOperationsMixin, BaseService[OrganizationSourceRecord ) for line in lines_data ] - OrganizationSourceIngestionService.save_records( + ingestion_result = OrganizationSourceIngestionService.save_records( source=ParserLoadLog.Source.FNS_REPORTS, load_batch=batch_id, records=[ @@ -1958,6 +1996,12 @@ class FNSReportService(BulkOperationsMixin, BaseService[OrganizationSourceRecord ) ], ) + if ingestion_result.skipped_unmatched or ingestion_result.skipped_ambiguous: + raise FNSReportOrganizationResolutionSkipped( + external_id=external_id, + ogrn=ogrn, + ingestion_result=ingestion_result, + ) report = OrganizationSourceRecord.objects.get( source=ParserLoadLog.Source.FNS_REPORTS, external_id=external_id, diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 7ffd30b..0934eb7 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -43,6 +43,7 @@ from apps.parsers.clients.vacancies import VacanciesClient from apps.parsers.clients.zakupki import ZakupkiClient from apps.parsers.models import ParserLoadLog from apps.parsers.services import ( + FNSReportOrganizationResolutionSkipped, FNSReportService, GenericParserRecordService, IndustrialCertificateService, @@ -61,8 +62,6 @@ from organizations.models import Organization as SourceOrganization from organizations.services import ( normalize_organization_name as normalize_identity_name, ) -from registers.models import Organization as RegistryOrganization -from registers.models import RegistryMembershipPeriod from requests.adapters import BaseAdapter logger = logging.getLogger(__name__) @@ -94,7 +93,7 @@ class ParserSourceSkipped(Exception): class RegistryLookupTarget: """Организация из активных реестров для lookup-загрузок.""" - organization_id: int + organization_id: str inn: str ogrn: str name: str @@ -158,13 +157,13 @@ def _active_registry_lookup_targets( ) -> list[RegistryLookupTarget]: """Вернуть организации, которые сейчас состоят хотя бы в одном реестре.""" queryset = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .order_by("organization_id") + SourceOrganization.objects.filter(opk_registry_membership=True) + .order_by("inn", "ogrn", "uid") .values( - "organization_id", - "organization__mn_inn", - "organization__mn_ogrn", - "organization__pn_name", + "uid", + "inn", + "ogrn", + "name", ) .distinct() ) @@ -173,16 +172,16 @@ def _active_registry_lookup_targets( targets: list[RegistryLookupTarget] = [] for row in queryset: - inn = str(row["organization__mn_inn"] or "").strip() - ogrn = str(row["organization__mn_ogrn"] or "").strip() + inn = str(row["inn"] or "").strip() + ogrn = str(row["ogrn"] or "").strip() if not inn and not ogrn: continue targets.append( RegistryLookupTarget( - organization_id=int(row["organization_id"]), + organization_id=str(row["uid"]), inn=inn, ogrn=ogrn, - name=str(row["organization__pn_name"] or "").strip(), + name=str(row["name"] or "").strip(), ) ) return targets @@ -574,24 +573,6 @@ def _fstec_local_identity_candidates( candidates_by_key: dict[str, dict[tuple[str, str], FstecIdentityCandidate]] = { key: {} for key in candidate_keys } - for organization in RegistryOrganization.objects.only( - "pn_name", - "mn_inn", - "mn_ogrn", - ).iterator(): - normalized_name = normalize_identity_name(organization.pn_name) - if normalized_name not in candidates_by_key: - continue - _add_fstec_identity_candidate( - candidates_by_key[normalized_name], - FstecIdentityCandidate( - inn=_normalize_fstec_identifier(organization.mn_inn, max_length=12), - ogrn=_normalize_fstec_identifier(organization.mn_ogrn, max_length=13), - name=organization.pn_name, - provider="local_registry", - ), - ) - for organization in ( SourceOrganization.objects.exclude(inn="") .exclude(ogrn="") @@ -932,7 +913,7 @@ class ArbitrationSubject: inn: str ogrn: str name: str - registry_organization_id: int | None = None + registry_organization_id: str | None = None def _normalize_identifier(value) -> str: @@ -954,7 +935,7 @@ def _add_arbitration_subject( inn, ogrn, name, - registry_organization_id: int | None = None, + registry_organization_id: str | None = None, limit: int, ) -> None: """Добавить subject, если по нему есть ИНН/ОГРН и лимит не исчерпан.""" @@ -1519,7 +1500,7 @@ def _move_to_dir( return target -def _process_fns_file_sync( +def _process_fns_file_sync( # noqa: C901 file_path: str | Path, *, task_id: str, @@ -1657,6 +1638,33 @@ def _process_fns_file_sync( raise return {"status": "failed", "error": str(e)} + except FNSReportOrganizationResolutionSkipped as e: + logger.info("FNS file skipped: %s - %s", file_path.name, e) + _move_to_dir( + file_path, + Path(settings.FNS_PROCESSED_DIRECTORY), + suffix=e.reason, + ) + ParserLoadLogService.update( + load_log, + status="skipped", + error_message=str(e), + ) + job.complete( + result={ + "status": "skipped", + "reason": e.reason, + "external_id": e.external_id, + "ogrn": e.ogrn, + } + ) + return { + "status": "skipped", + "reason": e.reason, + "external_id": e.external_id, + "ogrn": e.ogrn, + } + except Exception as e: logger.error( "FNS file processing error: %s - %s", diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 473e95e..fe4c453 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -66,7 +66,6 @@ from apps.parsers.serializers import ( ) from apps.parsers.source_cards import SourceCardService from apps.parsers.source_registry import PARSER_SOURCES -from apps.registers.models import RegistryMembershipPeriod from django.core.files.storage import default_storage from django.core.paginator import Paginator from django.db.models import CharField, Count, Q @@ -77,7 +76,7 @@ from django_celery_beat.models import CrontabSchedule, IntervalSchedule, Periodi from drf_yasg import openapi from drf_yasg.inspectors import SwaggerAutoSchema from drf_yasg.utils import no_body, swagger_auto_schema -from organizations.models import OrganizationSourceRecord +from organizations.models import Organization, OrganizationSourceRecord from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.parsers import FormParser, JSONParser, MultiPartParser @@ -188,14 +187,14 @@ CORE_PROFILE_INDUSTRIAL_SOURCES = { ParserLoadLog.Source.MANUFACTURES, } REGISTRY_ORGANIZATION_SEARCH_FIELDS = [ - "registry_organization__pn_name", + "registry_organization__name", "registry_organization_inn_text", "registry_organization_ogrn_text", - "registry_organization__mn_okpo", + "registry_organization__okpo", ] REGISTRY_ORGANIZATION_SEARCH_DESCRIPTION = ( "Поиск также включает название, ИНН, ОГРН и ОКПО связанной организации " - "из реестров." + "из справочника организаций." ) EXISTING_TASK_PARAMS = { "industrial": {"proxies", "requested_by_id"}, @@ -220,11 +219,11 @@ EXISTING_TASK_PARAMS = { def _with_registry_organization_search_annotations(queryset): return queryset.annotate( registry_organization_inn_text=Cast( - "registry_organization__mn_inn", + "registry_organization__inn", output_field=CharField(), ), registry_organization_ogrn_text=Cast( - "registry_organization__mn_ogrn", + "registry_organization__ogrn", output_field=CharField(), ), ) @@ -232,10 +231,10 @@ def _with_registry_organization_search_annotations(queryset): def _registry_organization_search_q(search: str) -> Q: return ( - Q(registry_organization__pn_name__icontains=search) + Q(registry_organization__name__icontains=search) | Q(registry_organization_inn_text__icontains=search) | Q(registry_organization_ogrn_text__icontains=search) - | Q(registry_organization__mn_okpo__icontains=search) + | Q(registry_organization__okpo__icontains=search) ) @@ -1580,15 +1579,15 @@ def _native_record_to_result( registry_organization = record.registry_organization external_id = record.external_id organisation_name = ( - registry_organization.pn_name if registry_organization else "" + registry_organization.name if registry_organization else "" ) title = record.file_name record_date = "" status_value = record.status url = "" inn = ( - str(registry_organization.mn_inn) - if registry_organization and registry_organization.mn_inn + str(registry_organization.inn) + if registry_organization and registry_organization.inn else "" ) ogrn = record.ogrn @@ -1740,25 +1739,21 @@ def _normalize_registry_identifier(value) -> str: def _active_registry_organization_indexes() -> ( tuple[ - set[int], - dict[str, set[int]], - dict[str, set[int]], + set[str], + dict[str, set[str]], + dict[str, set[str]], ] ): - organization_ids: set[int] = set() - by_inn: dict[str, set[int]] = defaultdict(set) - by_ogrn: dict[str, set[int]] = defaultdict(set) - rows = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .order_by() - .values_list( - "organization_id", - "organization__mn_inn", - "organization__mn_ogrn", - ) - .distinct() + organization_ids: set[str] = set() + by_inn: dict[str, set[str]] = defaultdict(set) + by_ogrn: dict[str, set[str]] = defaultdict(set) + rows = Organization.objects.filter(opk_registry_membership=True).values_list( + "uid", + "inn", + "ogrn", ) for organization_id, inn, ogrn in rows: + organization_id = str(organization_id) organization_ids.add(organization_id) inn_text = _normalize_registry_identifier(inn) ogrn_text = _normalize_registry_identifier(ogrn) @@ -1771,8 +1766,8 @@ def _active_registry_organization_indexes() -> ( def _matched_registry_organization_count( queryset, - by_inn: dict[str, set[int]], - by_ogrn: dict[str, set[int]], + by_inn: dict[str, set[str]], + by_ogrn: dict[str, set[str]], *, inn_field: str | None = "inn", ogrn_field: str | None = "ogrn", @@ -1786,7 +1781,7 @@ def _matched_registry_organization_count( return 0 fields = [field for field in (inn_field, ogrn_field) if field] - matched_ids: set[int] = set() + matched_ids: set[str] = set() for row in queryset.filter(filters).order_by().values_list(*fields).distinct(): values = dict(zip(fields, row, strict=True)) inn_text = _normalize_registry_identifier(values.get(inn_field)) @@ -1800,12 +1795,12 @@ def _matched_registry_organization_count( def _matched_registry_organization_ids( queryset, - by_inn: dict[str, set[int]], - by_ogrn: dict[str, set[int]], + by_inn: dict[str, set[str]], + by_ogrn: dict[str, set[str]], *, inn_field: str | None = "inn", ogrn_field: str | None = "ogrn", -) -> set[int]: +) -> set[str]: filters = Q() if inn_field and by_inn: filters |= Q(**{f"{inn_field}__in": list(by_inn)}) @@ -1815,7 +1810,7 @@ def _matched_registry_organization_ids( return set() fields = [field for field in (inn_field, ogrn_field) if field] - matched_ids: set[int] = set() + matched_ids: set[str] = set() for row in queryset.filter(filters).order_by().values_list(*fields).distinct(): values = dict(zip(fields, row, strict=True)) inn_text = _normalize_registry_identifier(values.get(inn_field)) @@ -1837,10 +1832,10 @@ def _source_record_queryset_for_parser_source(source: str): def _matched_source_record_registry_organization_ids( source: str, - by_inn: dict[str, set[int]], - by_ogrn: dict[str, set[int]], -) -> set[int]: - matched_ids: set[int] = set() + by_inn: dict[str, set[str]], + by_ogrn: dict[str, set[str]], +) -> set[str]: + matched_ids: set[str] = set() queryset = _source_record_queryset_for_parser_source(source) for inn, ogrn, ogrip in ( queryset.order_by() @@ -1861,10 +1856,10 @@ def _matched_source_record_registry_organization_ids( def _source_registry_matches( - by_inn: dict[str, set[int]], - by_ogrn: dict[str, set[int]], -) -> dict[str, set[int]]: - matches: dict[str, set[int]] = {} + by_inn: dict[str, set[str]], + by_ogrn: dict[str, set[str]], +) -> dict[str, set[str]]: + matches: dict[str, set[str]] = {} for source, _label in ParserLoadLog.Source.choices: matches[source] = _matched_source_record_registry_organization_ids( source, @@ -1880,25 +1875,24 @@ def _coverage_percent(count: int, total: int) -> float: def _registry_source_matrix_rows( *, - registry_organization_ids: set[int], - source_matches: dict[str, set[int]], + registry_organization_ids: set[str], + source_matches: dict[str, set[str]], enrichment_sources: list[str], ) -> list[dict]: total_organizations = len(registry_organization_ids) matrix_rows = [] registry_rows = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .select_related("registry") - .order_by("registry__name") - .values_list("registry_id", "registry__name", "organization_id") + Organization.objects.filter(opk_registry_membership=True) + .order_by("ropk_razdel_name", "uid") + .values_list("ropk_razdel_num", "ropk_razdel_name", "uid") .distinct() ) - registry_organizations: dict[tuple[str, str], set[int]] = {} + registry_organizations: dict[tuple[str, str], set[str]] = {} for registry_id, registry_name, organization_id in registry_rows: registry_organizations.setdefault( - (str(registry_id), registry_name), + (str(registry_id or "opk"), registry_name or "Реестр ОПК"), set(), - ).add(organization_id) + ).add(str(organization_id)) for ( registry_id, registry_name, @@ -2011,10 +2005,10 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict: else: source_coverage.append(item) - enriched_ids: set[int] = set() + enriched_ids: set[str] = set() for source in enrichment_sources: enriched_ids.update(source_matches.get(source, set())) - industrial_ids: set[int] = set() + industrial_ids: set[str] = set() for source in CORE_PROFILE_INDUSTRIAL_SOURCES: industrial_ids.update(source_matches.get(source, set())) core_profile_ids = ( @@ -2041,21 +2035,20 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict: } for severity, organization_ids in risk_ids_by_severity.items() } - active_memberships = RegistryMembershipPeriod.objects.filter( - ended_at__isnull=True - ).count() + active_memberships = total_organizations registries_total = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) + Organization.objects.filter(opk_registry_membership=True) + .exclude(ropk_razdel_num="") .order_by() - .values("registry_id") + .values("ropk_razdel_num") .distinct() .count() ) registries_with_data = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .filter(organization_id__in=enriched_ids) + Organization.objects.filter(uid__in=enriched_ids) + .exclude(ropk_razdel_num="") .order_by() - .values("registry_id") + .values("ropk_razdel_num") .distinct() .count() if enriched_ids @@ -2197,9 +2190,9 @@ def _native_field_map(source: str) -> dict[str, str]: "id": "id", "load_batch": "load_batch", "external_id": "external_id", - "inn": "registry_organization__mn_inn", + "inn": "registry_organization__inn", "ogrn": "ogrn", - "organisation_name": "registry_organization__pn_name", + "organisation_name": "registry_organization__name", "title": "file_name", "status": "status", "created_at": "created_at", @@ -2250,7 +2243,7 @@ def _native_search_q(source: str, search: str) -> Q: Q(file_name__icontains=search) | Q(external_id__icontains=search) | Q(ogrn__icontains=search) - | Q(registry_organization__pn_name__icontains=search) + | Q(registry_organization__name__icontains=search) | Q(status__icontains=search) ) return ( diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index 5ae9f5d..2e52601 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -11,7 +11,6 @@ API v1 URL configuration. - /api/v1/zakupki/ - Государственные закупки - /api/v1/fns/ - ФНС (бухгалтерская отчетность) - /api/v1/sources/ - Агрегированные карточки источников для фронтенда -- /api/v1/registers/ - Реестры организаций - /api/v1/exchange/ - Обмен с внешней БД - /api/v1/backups/ - Экспорт защищённых backup-архивов - /api/v1/system/ - Системные (логи, прокси) - только для админов @@ -35,7 +34,6 @@ from apps.parsers.urls import ( zakupki_urlpatterns, ) from django.urls import include, path -from registers.urls import registers_urlpatterns, stat_urlpatterns app_name = "api_v1" @@ -66,16 +64,12 @@ urlpatterns = [ path("fns/", include((fns_urlpatterns, "fns"))), # Результаты новых источников без перекрытия старых API выше path("", include("apps.parsers.api_result_urls", namespace="parser_results")), - # Сводные frontend-счетчики - path("stat/", include((stat_urlpatterns, "stat"))), # Управление parser Celery задачами и dashboard data path("parsers/", include("apps.parsers.urls")), # Агрегированные карточки источников для фронтенда path("sources/", include((sources_urlpatterns, "sources"))), # Настройки периодичности парсинга path("parsing/", include((parsing_urlpatterns, "parsing"))), - # Реестры организаций - path("registers/", include((registers_urlpatterns, "registers"))), # Обмен с внешней БД path("exchange/", include((exchange_urlpatterns, "exchange"))), # Backup архивы diff --git a/src/core/api_v2_urls.py b/src/core/api_v2_urls.py index ff3de9a..ec1431e 100644 --- a/src/core/api_v2_urls.py +++ b/src/core/api_v2_urls.py @@ -3,12 +3,10 @@ from apps.parsers.api_v2_urls import urlpatterns as parser_source_urlpatterns from django.urls import include, path from organizations.urls import organizations_urlpatterns -from registers.urls import registers_v2_urlpatterns app_name = "api_v2" urlpatterns = [ path("", include((organizations_urlpatterns, "organizations"))), - path("registers/", include((registers_v2_urlpatterns, "registers"))), path("sources/", include((parser_source_urlpatterns, "parser_sources"))), ] diff --git a/src/organizations/api_enrichment.py b/src/organizations/api_enrichment.py index e75542e..984afdc 100644 --- a/src/organizations/api_enrichment.py +++ b/src/organizations/api_enrichment.py @@ -19,7 +19,6 @@ from apps.parsers.models import ( ProcurementRecord, ) from django.db.models import Count, Prefetch, Q -from registers.models import RegistryMembershipPeriod from organizations.data_sources import to_api_data_source, to_internal_data_source from organizations.models import Organization @@ -71,20 +70,27 @@ def active_registry_identity_values( registry_name: str | None = None, ) -> tuple[set[str], set[str]]: """Return INN/OGRN values of organizations with active registry membership.""" - memberships = RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) + organizations = Organization.objects.filter( + Q(opk_registry_membership=True) | Q(goz_participation=True) | ~Q(ropk_num="") + ) if registry_id: - memberships = memberships.filter(registry_id=registry_id) + organizations = organizations.filter( + Q(ropk_razdel_num=str(registry_id)) | Q(gk_code=str(registry_id)) + ) if registry_name: - memberships = memberships.filter(registry__name__icontains=registry_name) + organizations = organizations.filter( + Q(ropk_razdel_name__icontains=registry_name) + | Q(gk_name__icontains=registry_name) + | Q(business_activity__icontains=registry_name) + ) inn_values: set[str] = set() ogrn_values: set[str] = set() - for inn, ogrn in memberships.values_list( - "organization__mn_inn", - "organization__mn_ogrn", - ): - inn_values.add(str(inn)) - ogrn_values.add(str(ogrn)) + for inn, ogrn in organizations.values_list("inn", "ogrn"): + if inn: + inn_values.add(str(inn)) + if ogrn: + ogrn_values.add(str(ogrn)) return inn_values, ogrn_values @@ -696,44 +702,27 @@ class OrganizationApiEnrichmentService: identifiers: dict[str, set[str]], ) -> dict[str, list[RegistrySummary]]: registries = {str(organization.uid): [] for organization in organizations} - if not identifiers["inn"] and not identifiers["ogrn"]: - return registries - - identity_filter = Q() - if identifiers["inn"]: - identity_filter |= Q(organization__mn_inn__in=identifiers["inn"]) - if identifiers["ogrn"]: - identity_filter |= Q(organization__mn_ogrn__in=identifiers["ogrn"]) - - memberships = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .filter(identity_filter) - .select_related("registry", "organization") - .order_by("registry__name") - ) - membership_by_inn: dict[str, list[RegistrySummary]] = {} - membership_by_ogrn: dict[str, list[RegistrySummary]] = {} - - for membership in memberships: - summary = RegistrySummary( - id=str(membership.registry_id), - name=membership.registry.name, - ) - membership_by_inn.setdefault( - str(membership.organization.mn_inn), - [], - ).append(summary) - membership_by_ogrn.setdefault( - str(membership.organization.mn_ogrn), - [], - ).append(summary) - for organization in organizations: seen: set[str] = set() summaries = [] - for summary in membership_by_inn.get( - organization.inn, [] - ) + membership_by_ogrn.get(organization.ogrn, []): + candidates = [] + if organization.opk_registry_membership: + candidates.append( + RegistrySummary( + id=organization.ropk_razdel_num or "opk", + name=organization.ropk_razdel_name or "Реестр ОПК", + ) + ) + if organization.goz_participation: + candidates.append(RegistrySummary(id="goz", name="Участие в ГОЗ")) + if organization.gk_name: + candidates.append( + RegistrySummary( + id=organization.gk_code or "gk", + name=organization.gk_name, + ) + ) + for summary in candidates: if summary.id in seen: continue seen.add(summary.id) diff --git a/src/organizations/directory_import.py b/src/organizations/directory_import.py new file mode 100644 index 0000000..be19e89 --- /dev/null +++ b/src/organizations/directory_import.py @@ -0,0 +1,447 @@ +"""Import the authoritative organization directory from XLSX.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any + +from django.db import transaction +from django.utils import timezone +from openpyxl import load_workbook + +from organizations.models import Organization + +DIRECTORY_SHEET = "Лист1" +GK_SHEET = "_gk" +IN_KORP_SHEET = "in_korp" +BUSINESS_ACTIVITY_SHEET = "business_act_cod" +STATE_SECTOR_SHEET = "_k - Госектор" + +SOURCE_HEADERS = [ + "rn", + "_gk", + "in_korp", + "full_name", + "short_name", + "pn_name", + "pn_name_en", + "inn", + "ogrn", + "okpo", + "filial", + "kpp", + "registration_date", + "create_date", + "organizational_legal_form", + "organizational_legal_form1", + "ownership_form", + "ownership_form1", + "authorized_capita", + "legal_address", + "business_act_cod", + "business_activity", + "general_director", + "general_director_tax_id", + "uk", + "inn_uk", + "appointment_date", + "cf_fl_rn", + "akc_fs", + "akc_sf", + "re_za", + "re_zasf", + "goz_participation", + "opk_registry_membership", + "ropk_num", + "ropk_razdel_num", + "ropk_razdel_name", + "min", + "dep", + "otr", + "is", + "_k", +] + + +class OrganizationDirectoryImportError(ValueError): + """Raised when an organization directory workbook is invalid.""" + + +@dataclass(frozen=True) +class OrganizationDirectoryImportResult: + """Counters returned by XLSX organization directory import.""" + + scanned: int = 0 + created: int = 0 + updated: int = 0 + skipped: int = 0 + + +class OrganizationDirectoryImportService: + """Load organizations from the authoritative XLSX directory.""" + + @classmethod + @transaction.atomic + def import_xlsx(cls, path: str | Path) -> OrganizationDirectoryImportResult: + file_path = Path(path) + if not file_path.exists(): + raise OrganizationDirectoryImportError(f"File does not exist: {file_path}") + if not file_path.is_file(): + raise OrganizationDirectoryImportError(f"Path is not a file: {file_path}") + + workbook = load_workbook(file_path, read_only=True, data_only=True) + if DIRECTORY_SHEET not in workbook.sheetnames: + raise OrganizationDirectoryImportError( + f"Sheet {DIRECTORY_SHEET!r} is missing" + ) + + references = cls._load_references(workbook) + file_hash = cls._sha256(file_path) + imported_at = timezone.now() + + scanned = 0 + created = 0 + updated = 0 + skipped = 0 + + worksheet = workbook[DIRECTORY_SHEET] + header_row = next(worksheet.iter_rows(min_row=1, max_row=1, values_only=True)) + headers = cls._headers(header_row) + missing = [header for header in SOURCE_HEADERS if header not in headers] + if missing: + raise OrganizationDirectoryImportError( + "Missing required columns: " + ", ".join(missing) + ) + + for row_number, row in enumerate( + worksheet.iter_rows(min_row=2, values_only=True), + start=2, + ): + scanned += 1 + for values in cls._source_value_dicts(headers, row, row_number=row_number): + rn = cls._required_positive_int(values.get("rn"), row_number=row_number) + defaults = cls._organization_defaults( + values, + references=references, + file_hash=file_hash, + row_number=row_number, + imported_at=imported_at, + ) + + _organization, was_created = Organization.objects.update_or_create( + rn=rn, + defaults=defaults, + ) + if was_created: + created += 1 + else: + updated += 1 + + return OrganizationDirectoryImportResult( + scanned=scanned, + created=created, + updated=updated, + skipped=skipped, + ) + + @classmethod + def _organization_defaults( + cls, + values: dict[str, Any], + *, + references: dict[str, dict[str, str]], + file_hash: str, + row_number: int, + imported_at, + ) -> dict[str, Any]: + full_name = cls._text(values.get("full_name")) + short_name = cls._text(values.get("short_name")) + pn_name = cls._text(values.get("pn_name")) + name = full_name or short_name or pn_name + if not name: + raise OrganizationDirectoryImportError( + f"Row {row_number}: one of full_name, short_name or pn_name is required" + ) + + gk_code = cls._text(values.get("_gk")) + in_korp_code = cls._text(values.get("in_korp")) + business_activity_code = cls._text(values.get("business_act_cod")) + state_sector_code = cls._text(values.get("_k")) + + return { + "name": name, + "gk_code": gk_code, + "gk_name": references[GK_SHEET].get(gk_code, ""), + "in_korp_code": in_korp_code, + "in_korp_name": references[IN_KORP_SHEET].get(in_korp_code, ""), + "full_name": full_name, + "short_name": short_name, + "pn_name": pn_name, + "pn_name_en": cls._text(values.get("pn_name_en")), + "inn": cls._digits(values.get("inn"), max_length=12), + "ogrn": cls._digits(values.get("ogrn"), max_length=13), + "okpo": cls._digits(values.get("okpo"), max_length=32), + "filial": cls._text(values.get("filial")), + "is_branch": cls._source_bool(values.get("filial")), + "kpp": cls._digits(values.get("kpp"), max_length=9), + "registration_date": cls._date(values.get("registration_date")), + "create_date": cls._text(values.get("create_date")), + "organizational_legal_form": cls._text( + values.get("organizational_legal_form") + ), + "organizational_legal_form1": cls._text( + values.get("organizational_legal_form1") + ), + "ownership_form": cls._text(values.get("ownership_form")), + "ownership_form1": cls._text(values.get("ownership_form1")), + "authorized_capital": cls._decimal(values.get("authorized_capita")), + "legal_address": cls._text(values.get("legal_address")), + "business_act_cod": business_activity_code, + "business_activity": cls._text(values.get("business_activity")) + or references[BUSINESS_ACTIVITY_SHEET].get(business_activity_code, ""), + "general_director": cls._text(values.get("general_director")), + "general_director_tax_id": cls._digits( + values.get("general_director_tax_id"), + max_length=32, + ), + "uk": cls._text(values.get("uk")), + "inn_uk": cls._digits(values.get("inn_uk"), max_length=12), + "appointment_date": cls._date(values.get("appointment_date")), + "cf_fl_rn": cls._text(values.get("cf_fl_rn")), + "akc_fs": cls._text(values.get("akc_fs")), + "akc_sf": cls._text(values.get("akc_sf")), + "re_za": cls._source_bool(values.get("re_za")), + "re_zasf": cls._source_bool(values.get("re_zasf")), + "goz_participation": cls._source_bool(values.get("goz_participation")), + "opk_registry_membership": cls._source_bool( + values.get("opk_registry_membership") + ), + "ropk_num": cls._text(values.get("ropk_num")), + "ropk_razdel_num": cls._text(values.get("ropk_razdel_num")), + "ropk_razdel_name": cls._text(values.get("ropk_razdel_name")), + "min": cls._text(values.get("min")), + "dep": cls._text(values.get("dep")), + "otr": cls._text(values.get("otr")), + "integrated_structure": cls._text(values.get("is")), + "state_sector_code": state_sector_code, + "state_sector_name": references[STATE_SECTOR_SHEET].get( + state_sector_code, + "", + ), + "directory_source_file_hash": file_hash, + "directory_source_row_number": row_number, + "directory_imported_at": imported_at, + } + + @classmethod + def _load_references(cls, workbook) -> dict[str, dict[str, str]]: + return { + GK_SHEET: cls._reference_sheet(workbook, GK_SHEET), + IN_KORP_SHEET: cls._reference_sheet(workbook, IN_KORP_SHEET), + BUSINESS_ACTIVITY_SHEET: cls._reference_sheet( + workbook, + BUSINESS_ACTIVITY_SHEET, + ), + STATE_SECTOR_SHEET: cls._reference_sheet(workbook, STATE_SECTOR_SHEET), + } + + @staticmethod + def _reference_sheet(workbook, sheet_name: str) -> dict[str, str]: + if sheet_name not in workbook.sheetnames: + return {} + worksheet = workbook[sheet_name] + references: dict[str, str] = {} + for row in worksheet.iter_rows(min_row=2, values_only=True): + if not row or row[0] is None: + continue + code = OrganizationDirectoryImportService._text(row[0]) + value = OrganizationDirectoryImportService._text(row[1] if len(row) > 1 else "") + if code: + references[code] = value + return references + + @staticmethod + def _headers(row: tuple[Any, ...]) -> dict[str, int]: + headers: dict[str, int] = {} + for index, value in enumerate(row): + header = OrganizationDirectoryImportService._text(value) + if header: + headers[header] = index + return headers + + @classmethod + def _source_value_dicts( + cls, + headers: dict[str, int], + row: tuple[Any, ...], + *, + row_number: int, + ) -> list[dict[str, Any]]: + values = { + header: row[index] if index < len(row) else None + for header, index in headers.items() + if header in SOURCE_HEADERS + } + pn_name_en = values.get("pn_name_en") + if not ( + isinstance(pn_name_en, str) + and "\t" in pn_name_en + and "_x000D_\n" in pn_name_en + ): + return [values] + + return cls._split_embedded_tsv_values( + values, + row, + headers, + row_number=row_number, + ) + + @classmethod + def _split_embedded_tsv_values( + cls, + values: dict[str, Any], + row: tuple[Any, ...], + headers: dict[str, int], + *, + row_number: int, + ) -> list[dict[str, Any]]: + embedded = values["pn_name_en"] + before_marker, second_prefix_raw = embedded.split("_x000D_\n", 1) + + first_tail = cls._drop_embedded_rn_noise( + before_marker.split("\t"), + rn=values.get("rn"), + expected_count=len(SOURCE_HEADERS) - 6, + ) + if len(first_tail) != len(SOURCE_HEADERS) - 6: + raise OrganizationDirectoryImportError( + f"Row {row_number}: invalid embedded first organization structure" + ) + + second_prefix = second_prefix_raw.split("\t") + if len(second_prefix) != 4: + raise OrganizationDirectoryImportError( + f"Row {row_number}: invalid embedded second organization structure" + ) + + second_tail_start = headers["inn"] + second_tail_count = len(SOURCE_HEADERS) - 4 + second_tail = list( + row[second_tail_start : second_tail_start + second_tail_count] + ) + if len(second_tail) != second_tail_count: + raise OrganizationDirectoryImportError( + f"Row {row_number}: incomplete embedded second organization structure" + ) + + first_values = dict( + zip( + SOURCE_HEADERS, + [values.get(header) for header in SOURCE_HEADERS[:6]] + first_tail, + strict=True, + ) + ) + second_values = dict(zip(SOURCE_HEADERS, second_prefix + second_tail, strict=True)) + if cls._text(second_values.get("is")) == cls._text(second_values.get("rn")): + second_values["is"] = "" + return [first_values, second_values] + + @classmethod + def _drop_embedded_rn_noise( + cls, + values: list[Any], + *, + rn: Any, + expected_count: int, + ) -> list[Any]: + if ( + len(values) == expected_count + 1 + and cls._text(values[-2]) == cls._text(rn) + ): + return [*values[:-2], values[-1]] + return values + + @staticmethod + def _required_positive_int(value: Any, *, row_number: int) -> int: + digits = OrganizationDirectoryImportService._digits(value, max_length=20) + if not digits: + raise OrganizationDirectoryImportError( + f"Row {row_number}: rn must be a positive integer" + ) + return int(digits) + + @staticmethod + def _text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str) and value.strip().replace(" ", "") == "..": + return "" + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value).strip() + + @staticmethod + def _digits(value: Any, *, max_length: int) -> str: + if value is None: + return "" + digits = re.sub(r"\D+", "", OrganizationDirectoryImportService._text(value)) + if not digits or len(digits) > max_length: + return "" + return digits + + @staticmethod + def _source_bool(value: Any) -> bool | None: + normalized = OrganizationDirectoryImportService._text(value).upper() + if normalized == ".T.": + return True + if normalized == ".F.": + return False + return None + + @staticmethod + def _date(value: Any) -> date | None: + if value is None: + return None + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + + raw = OrganizationDirectoryImportService._text(value) + if not raw: + return None + for date_format in ("%d.%m.%Y", "%Y-%m-%d", "%d/%m/%Y"): + try: + return datetime.strptime(raw, date_format).date() + except ValueError: + continue + raise OrganizationDirectoryImportError(f"Invalid date value: {raw}") + + @staticmethod + def _decimal(value: Any) -> Decimal | None: + if value is None: + return None + raw = OrganizationDirectoryImportService._text(value) + if not raw: + return None + normalized = raw.replace("\u00a0", "").replace(" ", "").replace(",", ".") + try: + return Decimal(normalized) + except (InvalidOperation, ValueError) as exc: + raise OrganizationDirectoryImportError( + f"Invalid decimal value: {raw}" + ) from exc + + @staticmethod + def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/src/organizations/filters.py b/src/organizations/filters.py index 96fa3d9..c69d0c3 100644 --- a/src/organizations/filters.py +++ b/src/organizations/filters.py @@ -1,9 +1,7 @@ """Filters for organization-centric API v2.""" -from django.db.models import CharField, Q -from django.db.models.functions import Cast +from django.db.models import Q from django_filters import rest_framework as filters -from registers.models import RegistryMembershipPeriod from organizations.models import Organization, OrganizationSourceExtension, SourceGroup @@ -45,7 +43,7 @@ class OrganizationFilter(filters.FilterSet): identity_status = filters.CharFilter( field_name="identity_status", lookup_expr="exact" ) - registry = filters.UUIDFilter(method="filter_registry") + registry = filters.CharFilter(method="filter_registry") registry_name = filters.CharFilter(method="filter_registry_name") has_registry = filters.BooleanFilter(method="filter_has_registry") source_group = filters.CharFilter(method="filter_source_group") @@ -127,7 +125,7 @@ class OrganizationFilter(filters.FilterSet): registry_name: str | None = None, has_registry: bool = True, ): - query = cls._registry_identity_query( + query = cls._registry_directory_query( registry_id=registry_id, registry_name=registry_name, ) @@ -135,20 +133,26 @@ class OrganizationFilter(filters.FilterSet): return queryset.filter(query) return queryset.exclude(query) - @classmethod - def _registry_identity_query( - cls, + @staticmethod + def _registry_directory_query( *, registry_id: str | None = None, registry_name: str | None = None, - ): - inn_values, ogrn_values = cls._registry_identity_value_querysets( - registry_id=registry_id, - registry_name=registry_name, - ) - return ( - Q(inn__in=inn_values) | Q(ogrn__in=ogrn_values) | Q(ogrip__in=ogrn_values) + ) -> Q: + query = ( + Q(opk_registry_membership=True) + | Q(goz_participation=True) + | ~Q(ropk_num="") ) + if registry_id: + query &= Q(ropk_razdel_num=str(registry_id)) | Q(gk_code=str(registry_id)) + if registry_name: + query &= ( + Q(ropk_razdel_name__icontains=registry_name) + | Q(gk_name__icontains=registry_name) + | Q(business_activity__icontains=registry_name) + ) + return query @staticmethod def _registry_identity_value_querysets( @@ -156,26 +160,13 @@ class OrganizationFilter(filters.FilterSet): registry_id: str | None = None, registry_name: str | None = None, ): - membership = RegistryMembershipPeriod.objects.filter( - ended_at__isnull=True, - ).order_by() - if registry_id: - membership = membership.filter(registry_id=registry_id) - if registry_name: - membership = membership.filter(registry__name__icontains=registry_name) - - membership = membership.annotate( - organization_inn_text=Cast( - "organization__mn_inn", - output_field=CharField(), - ), - organization_ogrn_text=Cast( - "organization__mn_ogrn", - output_field=CharField(), - ), + organizations = Organization.objects.filter( + OrganizationFilter._registry_directory_query( + registry_id=registry_id, + registry_name=registry_name, + ) ) - return ( - membership.values_list("organization_inn_text", flat=True), - membership.values_list("organization_ogrn_text", flat=True), + organizations.exclude(inn="").values_list("inn", flat=True), + organizations.exclude(ogrn="").values_list("ogrn", flat=True), ) diff --git a/src/organizations/management/commands/import_organization_directory.py b/src/organizations/management/commands/import_organization_directory.py new file mode 100644 index 0000000..b448f2e --- /dev/null +++ b/src/organizations/management/commands/import_organization_directory.py @@ -0,0 +1,35 @@ +"""Import the authoritative organizations directory.""" + +from __future__ import annotations + +import json + +from apps.core.management.commands.base import BaseAppCommand + +from organizations.directory_import import OrganizationDirectoryImportService + + +class Command(BaseAppCommand): + """Import organizations from the authoritative XLSX file.""" + + help = "Импортирует справочник organizations.Organization из перечня XLSX" + use_transaction = False + + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument("path", help="Путь к файлу перечня организаций XLSX.") + + def execute_command(self, *args, **options) -> str: + result = OrganizationDirectoryImportService.import_xlsx(options["path"]) + rendered = json.dumps( + { + "scanned": result.scanned, + "created": result.created, + "updated": result.updated, + "skipped": result.skipped, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/organizations/migrations/0007_auto_20260607_1017.py b/src/organizations/migrations/0007_auto_20260607_1017.py new file mode 100644 index 0000000..b28a7e3 --- /dev/null +++ b/src/organizations/migrations/0007_auto_20260607_1017.py @@ -0,0 +1,263 @@ +# Generated by Django 3.2.25 on 2026-06-07 10:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('organizations', '0006_polymorphic_source_extensions'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='organization', + name='unique_org_inn_kpp_not_blank', + ), + migrations.RemoveConstraint( + model_name='organization', + name='unique_org_inn_without_kpp', + ), + migrations.AddField( + model_name='organization', + name='akc_fs', + field=models.CharField(blank=True, max_length=64, verbose_name='акционерный капитал федеральная собственность'), + ), + migrations.AddField( + model_name='organization', + name='akc_sf', + field=models.CharField(blank=True, max_length=64, verbose_name='акционерный капитал собственность субъекта РФ'), + ), + migrations.AddField( + model_name='organization', + name='appointment_date', + field=models.DateField(blank=True, null=True, verbose_name='дата назначения руководителя'), + ), + migrations.AddField( + model_name='organization', + name='authorized_capital', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=24, null=True, verbose_name='уставный капитал'), + ), + migrations.AddField( + model_name='organization', + name='business_act_cod', + field=models.CharField(blank=True, db_index=True, max_length=32, verbose_name='код вида деятельности'), + ), + migrations.AddField( + model_name='organization', + name='business_activity', + field=models.CharField(blank=True, max_length=255, verbose_name='вид деятельности'), + ), + migrations.AddField( + model_name='organization', + name='cf_fl_rn', + field=models.CharField(blank=True, max_length=64, verbose_name='регистрационный номер руководителя'), + ), + migrations.AddField( + model_name='organization', + name='create_date', + field=models.CharField(blank=True, max_length=32, verbose_name='год или дата создания'), + ), + migrations.AddField( + model_name='organization', + name='dep', + field=models.TextField(blank=True, verbose_name='департамент'), + ), + migrations.AddField( + model_name='organization', + name='directory_imported_at', + field=models.DateTimeField(blank=True, null=True, verbose_name='дата импорта перечня'), + ), + migrations.AddField( + model_name='organization', + name='directory_source_file_hash', + field=models.CharField(blank=True, db_index=True, max_length=64, verbose_name='хеш файла перечня'), + ), + migrations.AddField( + model_name='organization', + name='directory_source_row_number', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='номер строки файла перечня'), + ), + migrations.AddField( + model_name='organization', + name='filial', + field=models.CharField(blank=True, db_index=True, max_length=64, verbose_name='признак филиала из источника'), + ), + migrations.AddField( + model_name='organization', + name='full_name', + field=models.TextField(blank=True, verbose_name='полное наименование'), + ), + migrations.AddField( + model_name='organization', + name='general_director', + field=models.TextField(blank=True, verbose_name='руководитель'), + ), + migrations.AddField( + model_name='organization', + name='general_director_tax_id', + field=models.CharField(blank=True, max_length=32, verbose_name='ИНН руководителя'), + ), + migrations.AddField( + model_name='organization', + name='gk_code', + field=models.CharField(blank=True, db_column='_gk', db_index=True, help_text='Отношение к ГК Росатом и ГК Роскосмос', max_length=16, verbose_name='код государственной корпорации'), + ), + migrations.AddField( + model_name='organization', + name='gk_name', + field=models.CharField(blank=True, help_text='Расшифровка кода государственной корпорации', max_length=255, verbose_name='государственная корпорация'), + ), + migrations.AddField( + model_name='organization', + name='goz_participation', + field=models.BooleanField(blank=True, db_index=True, null=True, verbose_name='участие в ГОЗ'), + ), + migrations.AddField( + model_name='organization', + name='in_korp_code', + field=models.CharField(blank=True, db_index=True, help_text='Положение внутри ГК Росатом и ГК Роскосмос', max_length=16, verbose_name='код положения внутри ГК'), + ), + migrations.AddField( + model_name='organization', + name='in_korp_name', + field=models.CharField(blank=True, help_text='Расшифровка положения внутри ГК', max_length=255, verbose_name='положение внутри ГК'), + ), + migrations.AddField( + model_name='organization', + name='inn_uk', + field=models.CharField(blank=True, db_index=True, max_length=12, verbose_name='ИНН управляющей компании'), + ), + migrations.AddField( + model_name='organization', + name='integrated_structure', + field=models.TextField(blank=True, db_column='is', verbose_name='интегрированная структура'), + ), + migrations.AddField( + model_name='organization', + name='is_branch', + field=models.BooleanField(blank=True, db_index=True, help_text='Нормализованный признак filial: .T. = филиал, .F. = головная', null=True, verbose_name='филиал'), + ), + migrations.AddField( + model_name='organization', + name='legal_address', + field=models.TextField(blank=True, verbose_name='юридический адрес'), + ), + migrations.AddField( + model_name='organization', + name='min', + field=models.TextField(blank=True, verbose_name='министерство'), + ), + migrations.AddField( + model_name='organization', + name='okpo', + field=models.CharField(blank=True, db_index=True, max_length=32, verbose_name='ОКПО'), + ), + migrations.AddField( + model_name='organization', + name='opk_registry_membership', + field=models.BooleanField(blank=True, db_index=True, null=True, verbose_name='членство в реестре ОПК'), + ), + migrations.AddField( + model_name='organization', + name='organizational_legal_form', + field=models.CharField(blank=True, db_index=True, max_length=32, verbose_name='код организационно-правовой формы'), + ), + migrations.AddField( + model_name='organization', + name='organizational_legal_form1', + field=models.CharField(blank=True, max_length=512, verbose_name='организационно-правовая форма'), + ), + migrations.AddField( + model_name='organization', + name='otr', + field=models.TextField(blank=True, verbose_name='отрасль'), + ), + migrations.AddField( + model_name='organization', + name='ownership_form', + field=models.CharField(blank=True, db_index=True, max_length=32, verbose_name='код формы собственности'), + ), + migrations.AddField( + model_name='organization', + name='ownership_form1', + field=models.CharField(blank=True, max_length=512, verbose_name='форма собственности'), + ), + migrations.AddField( + model_name='organization', + name='pn_name', + field=models.TextField(blank=True, verbose_name='приведенное наименование'), + ), + migrations.AddField( + model_name='organization', + name='pn_name_en', + field=models.TextField(blank=True, verbose_name='приведенное наименование на английском'), + ), + migrations.AddField( + model_name='organization', + name='re_za', + field=models.BooleanField(blank=True, db_index=True, null=True, verbose_name='реестр организаций-заказчиков'), + ), + migrations.AddField( + model_name='organization', + name='re_zasf', + field=models.BooleanField(blank=True, db_index=True, null=True, verbose_name='реестр организаций-заказчиков субъекта РФ'), + ), + migrations.AddField( + model_name='organization', + name='registration_date', + field=models.DateField(blank=True, db_index=True, null=True, verbose_name='дата регистрации'), + ), + migrations.AddField( + model_name='organization', + name='rn', + field=models.PositiveBigIntegerField(blank=True, db_index=True, help_text='Регистрационный номер организации из перечня', null=True, unique=True, verbose_name='регистрационный номер'), + ), + migrations.AddField( + model_name='organization', + name='ropk_num', + field=models.CharField(blank=True, db_index=True, max_length=64, verbose_name='номер в реестре ОПК'), + ), + migrations.AddField( + model_name='organization', + name='ropk_razdel_name', + field=models.TextField(blank=True, verbose_name='раздел реестра ОПК'), + ), + migrations.AddField( + model_name='organization', + name='ropk_razdel_num', + field=models.CharField(blank=True, max_length=64, verbose_name='номер раздела реестра ОПК'), + ), + migrations.AddField( + model_name='organization', + name='short_name', + field=models.TextField(blank=True, verbose_name='краткое наименование'), + ), + migrations.AddField( + model_name='organization', + name='state_sector_code', + field=models.CharField(blank=True, db_column='_k', db_index=True, max_length=16, verbose_name='код госсектора'), + ), + migrations.AddField( + model_name='organization', + name='state_sector_name', + field=models.CharField(blank=True, max_length=255, verbose_name='госсектор'), + ), + migrations.AddField( + model_name='organization', + name='uk', + field=models.TextField(blank=True, verbose_name='управляющая компания'), + ), + migrations.AddIndex( + model_name='organization', + index=models.Index(fields=['ogrn', 'kpp'], name='organizatio_ogrn_e69346_idx'), + ), + migrations.AddIndex( + model_name='organization', + index=models.Index(fields=['okpo'], name='organizatio_okpo_89a721_idx'), + ), + migrations.AddIndex( + model_name='organization', + index=models.Index(fields=['is_branch', 'inn', 'ogrn'], name='organizatio_is_bran_c94dbf_idx'), + ), + ] diff --git a/src/organizations/models.py b/src/organizations/models.py index 4443fc6..acfa798 100644 --- a/src/organizations/models.py +++ b/src/organizations/models.py @@ -58,12 +58,63 @@ class Organization(models.Model): default=uuid.uuid4, editable=False, ) + rn = models.PositiveBigIntegerField( + _("регистрационный номер"), + null=True, + blank=True, + unique=True, + db_index=True, + help_text=_("Регистрационный номер организации из перечня"), + ) + gk_code = models.CharField( + _("код государственной корпорации"), + max_length=16, + blank=True, + db_column="_gk", + db_index=True, + help_text=_("Отношение к ГК Росатом и ГК Роскосмос"), + ) + gk_name = models.CharField( + _("государственная корпорация"), + max_length=255, + blank=True, + help_text=_("Расшифровка кода государственной корпорации"), + ) + in_korp_code = models.CharField( + _("код положения внутри ГК"), + max_length=16, + blank=True, + db_index=True, + help_text=_("Положение внутри ГК Росатом и ГК Роскосмос"), + ) + in_korp_name = models.CharField( + _("положение внутри ГК"), + max_length=255, + blank=True, + help_text=_("Расшифровка положения внутри ГК"), + ) name = models.CharField( _("наименование"), max_length=1024, db_index=True, help_text=_("Наименование организации или ИП"), ) + full_name = models.TextField( + _("полное наименование"), + blank=True, + ) + short_name = models.TextField( + _("краткое наименование"), + blank=True, + ) + pn_name = models.TextField( + _("приведенное наименование"), + blank=True, + ) + pn_name_en = models.TextField( + _("приведенное наименование на английском"), + blank=True, + ) inn = models.CharField( _("ИНН"), max_length=12, @@ -85,6 +136,25 @@ class Organization(models.Model): db_index=True, help_text=_("ОГРН только для юридических лиц"), ) + okpo = models.CharField( + _("ОКПО"), + max_length=32, + blank=True, + db_index=True, + ) + filial = models.CharField( + _("признак филиала из источника"), + max_length=64, + blank=True, + db_index=True, + ) + is_branch = models.BooleanField( + _("филиал"), + null=True, + blank=True, + db_index=True, + help_text=_("Нормализованный признак filial: .T. = филиал, .F. = головная"), + ) ogrip = models.CharField( _("ОГРИП"), max_length=15, @@ -92,6 +162,184 @@ class Organization(models.Model): db_index=True, help_text=_("ОГРИП только для индивидуальных предпринимателей"), ) + registration_date = models.DateField( + _("дата регистрации"), + null=True, + blank=True, + db_index=True, + ) + create_date = models.CharField( + _("год или дата создания"), + max_length=32, + blank=True, + ) + organizational_legal_form = models.CharField( + _("код организационно-правовой формы"), + max_length=32, + blank=True, + db_index=True, + ) + organizational_legal_form1 = models.CharField( + _("организационно-правовая форма"), + max_length=512, + blank=True, + ) + ownership_form = models.CharField( + _("код формы собственности"), + max_length=32, + blank=True, + db_index=True, + ) + ownership_form1 = models.CharField( + _("форма собственности"), + max_length=512, + blank=True, + ) + authorized_capital = models.DecimalField( + _("уставный капитал"), + max_digits=24, + decimal_places=2, + null=True, + blank=True, + ) + legal_address = models.TextField( + _("юридический адрес"), + blank=True, + ) + business_act_cod = models.CharField( + _("код вида деятельности"), + max_length=32, + blank=True, + db_index=True, + ) + business_activity = models.CharField( + _("вид деятельности"), + max_length=255, + blank=True, + ) + general_director = models.TextField( + _("руководитель"), + blank=True, + ) + general_director_tax_id = models.CharField( + _("ИНН руководителя"), + max_length=32, + blank=True, + ) + uk = models.TextField( + _("управляющая компания"), + blank=True, + ) + inn_uk = models.CharField( + _("ИНН управляющей компании"), + max_length=12, + blank=True, + db_index=True, + ) + appointment_date = models.DateField( + _("дата назначения руководителя"), + null=True, + blank=True, + ) + cf_fl_rn = models.CharField( + _("регистрационный номер руководителя"), + max_length=64, + blank=True, + ) + akc_fs = models.CharField( + _("акционерный капитал федеральная собственность"), + max_length=64, + blank=True, + ) + akc_sf = models.CharField( + _("акционерный капитал собственность субъекта РФ"), + max_length=64, + blank=True, + ) + re_za = models.BooleanField( + _("реестр организаций-заказчиков"), + null=True, + blank=True, + db_index=True, + ) + re_zasf = models.BooleanField( + _("реестр организаций-заказчиков субъекта РФ"), + null=True, + blank=True, + db_index=True, + ) + goz_participation = models.BooleanField( + _("участие в ГОЗ"), + null=True, + blank=True, + db_index=True, + ) + opk_registry_membership = models.BooleanField( + _("членство в реестре ОПК"), + null=True, + blank=True, + db_index=True, + ) + ropk_num = models.CharField( + _("номер в реестре ОПК"), + max_length=64, + blank=True, + db_index=True, + ) + ropk_razdel_num = models.CharField( + _("номер раздела реестра ОПК"), + max_length=64, + blank=True, + ) + ropk_razdel_name = models.TextField( + _("раздел реестра ОПК"), + blank=True, + ) + min = models.TextField( + _("министерство"), + blank=True, + ) + dep = models.TextField( + _("департамент"), + blank=True, + ) + otr = models.TextField( + _("отрасль"), + blank=True, + ) + integrated_structure = models.TextField( + _("интегрированная структура"), + blank=True, + db_column="is", + ) + state_sector_code = models.CharField( + _("код госсектора"), + max_length=16, + blank=True, + db_column="_k", + db_index=True, + ) + state_sector_name = models.CharField( + _("госсектор"), + max_length=255, + blank=True, + ) + directory_source_file_hash = models.CharField( + _("хеш файла перечня"), + max_length=64, + blank=True, + db_index=True, + ) + directory_source_row_number = models.PositiveIntegerField( + _("номер строки файла перечня"), + null=True, + blank=True, + ) + directory_imported_at = models.DateTimeField( + _("дата импорта перечня"), + null=True, + blank=True, + ) identity_status = models.CharField( _("полнота реквизитов"), max_length=16, @@ -116,20 +364,13 @@ class Organization(models.Model): ordering = ["name"] indexes = [ models.Index(fields=["inn", "kpp"]), + models.Index(fields=["ogrn", "kpp"]), models.Index(fields=["inn", "ogrn"]), models.Index(fields=["inn", "ogrip"]), + models.Index(fields=["okpo"]), + models.Index(fields=["is_branch", "inn", "ogrn"]), ] constraints = [ - models.UniqueConstraint( - fields=["inn", "kpp"], - condition=~Q(inn="") & ~Q(kpp=""), - name="unique_org_inn_kpp_not_blank", - ), - models.UniqueConstraint( - fields=["inn"], - condition=~Q(inn="") & Q(kpp="") & Q(ogrip=""), - name="unique_org_inn_without_kpp", - ), models.UniqueConstraint( fields=["ogrip"], condition=~Q(ogrip=""), @@ -147,17 +388,69 @@ class Organization(models.Model): return f"{self.name} ({identifier})" return self.name + @property + def id(self): + """Compatibility alias for code paths that expect a model id attribute.""" + return self.pk + + @staticmethod + def _legacy_digits(value, *, max_length: int, zfill: int | None = None) -> str: + digits = "".join(char for char in str(value or "") if char.isdigit()) + if zfill and 0 < len(digits) < zfill: + digits = digits.zfill(zfill) + return digits[:max_length] + + @property + def mn_inn(self) -> str: + """Compatibility alias for the removed registers.Organization field.""" + return self.inn + + @mn_inn.setter + def mn_inn(self, value) -> None: + self.inn = self._legacy_digits(value, max_length=12, zfill=10) + + @property + def mn_ogrn(self) -> str: + """Compatibility alias for the removed registers.Organization field.""" + return self.ogrn + + @mn_ogrn.setter + def mn_ogrn(self, value) -> None: + self.ogrn = self._legacy_digits(value, max_length=13) + + @property + def in_kpp(self) -> str: + """Compatibility alias for the removed registers.Organization field.""" + return self.kpp + + @in_kpp.setter + def in_kpp(self, value) -> None: + self.kpp = self._legacy_digits(value, max_length=9, zfill=9) + + @property + def mn_okpo(self) -> str: + """Compatibility alias for the removed registers.Organization field.""" + return self.okpo + + @mn_okpo.setter + def mn_okpo(self, value) -> None: + self.okpo = self._legacy_digits(value, max_length=32) + @property def normalized_name(self) -> str: return normalize_organization_name(self.name) def save(self, *args, **kwargs) -> None: + if not self.name: + self.name = self.pn_name or self.short_name or self.full_name self.identity_status = self._resolve_identity_status() self.primary_identity = self._resolve_primary_identity() update_fields = kwargs.get("update_fields") if update_fields is not None: kwargs["update_fields"] = list( - dict.fromkeys([*update_fields, "identity_status", "primary_identity"]) + dict.fromkeys( + [*update_fields, "name", "identity_status", "primary_identity"] + ) ) super().save(*args, **kwargs) @@ -517,63 +810,12 @@ class OrganizationSourceRecord(models.Model): @property def registry_organization(self): - """Best-effort active registry organization matched by canonical identity.""" - from django.db.models import CharField - from django.db.models.functions import Cast - from registers.models import Organization as RegistryOrganization - from registers.models import RegistryMembershipPeriod - - def _registry_numeric_values(value: str) -> list[str]: - stripped = str(value or "").lstrip("0") - return [value, stripped] if stripped and stripped != value else [value] - - identity_filter = Q() - inn = self.inn - ogrn = self.ogrn - ogrip = self.ogrip - if inn: - identity_filter |= Q(registry_inn_text__in=_registry_numeric_values(inn)) - if ogrn: - identity_filter |= Q(registry_ogrn_text__in=_registry_numeric_values(ogrn)) - if ogrip: - identity_filter |= Q(registry_ogrn_text__in=_registry_numeric_values(ogrip)) - if not identity_filter: - return None - - membership = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .select_related("organization") - .annotate( - registry_inn_text=Cast( - "organization__mn_inn", - output_field=CharField(), - ), - registry_ogrn_text=Cast( - "organization__mn_ogrn", - output_field=CharField(), - ), - ) - .filter(identity_filter) - .order_by("organization__pn_name", "organization_id") - .first() - ) - if membership is not None: - return membership.organization - - return ( - RegistryOrganization.objects.annotate( - registry_inn_text=Cast("mn_inn", output_field=CharField()), - registry_ogrn_text=Cast("mn_ogrn", output_field=CharField()), - ) - .filter(identity_filter) - .order_by("pn_name", "id") - .first() - ) + """Compatibility alias for callers that still use registry wording.""" + return self.extension.organization @property def registry_organization_id(self): - registry_organization = self.registry_organization - return registry_organization.id if registry_organization is not None else None + return self.extension.organization_id class OrganizationSourceFinancialLine(models.Model): diff --git a/src/organizations/resolver.py b/src/organizations/resolver.py new file mode 100644 index 0000000..4fb2f56 --- /dev/null +++ b/src/organizations/resolver.py @@ -0,0 +1,168 @@ +"""Organization directory matching helpers.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Literal + +from organizations.models import Organization +from organizations.source_identity import normalize_identity_fields + +ResolveStatus = Literal["matched", "unmatched", "ambiguous"] + + +@dataclass(frozen=True) +class OrganizationIdentity: + """Normalized identity used to match parser rows to the directory.""" + + rn: str = "" + okpo: str = "" + inn: str = "" + kpp: str = "" + ogrn: str = "" + ogrip: str = "" + + +@dataclass(frozen=True) +class OrganizationResolveResult: + """Result of matching one parser row to an existing organization.""" + + status: ResolveStatus + organization: Organization | None = None + + +class OrganizationDirectoryResolver: + """Resolve existing organizations from the authoritative directory only.""" + + @classmethod + def identity( + cls, + *, + rn: object = None, + okpo: object = None, + inn: object = None, + kpp: object = None, + ogrn: object = None, + ogrip: object = None, + ) -> OrganizationIdentity: + normalized_inn, normalized_kpp, normalized_ogrn, normalized_ogrip = ( + normalize_identity_fields( + inn=inn, + kpp=kpp, + ogrn=ogrn, + ogrip=ogrip, + ) + ) + return OrganizationIdentity( + rn=cls._digits(rn, max_length=20), + okpo=cls._digits(okpo, max_length=32), + inn=normalized_inn, + kpp=normalized_kpp, + ogrn=normalized_ogrn, + ogrip=normalized_ogrip, + ) + + @classmethod + def from_payload( + cls, + *, + payload: dict | None, + inn: object = None, + kpp: object = None, + ogrn: object = None, + ogrip: object = None, + ) -> OrganizationIdentity: + payload = payload or {} + return cls.identity( + rn=payload.get("rn") or payload.get("organization_rn"), + okpo=payload.get("okpo") or payload.get("mn_okpo"), + inn=inn or payload.get("inn"), + kpp=kpp or payload.get("kpp"), + ogrn=ogrn or payload.get("ogrn"), + ogrip=ogrip or payload.get("ogrip"), + ) + + @classmethod + def resolve(cls, identity: OrganizationIdentity) -> OrganizationResolveResult: # noqa: C901 + if identity.rn: + organization = cls._single(Organization.objects.filter(rn=identity.rn)) + if organization is not None: + return OrganizationResolveResult("matched", organization) + + if identity.okpo: + result = cls._resolve_unique(Organization.objects.filter(okpo=identity.okpo)) + if result.status != "unmatched": + return result + + if identity.inn and identity.kpp: + result = cls._resolve_unique( + Organization.objects.filter(inn=identity.inn, kpp=identity.kpp), + ) + if result.status != "unmatched": + return result + + if identity.ogrn and identity.kpp: + result = cls._resolve_unique( + Organization.objects.filter(ogrn=identity.ogrn, kpp=identity.kpp), + ) + if result.status != "unmatched": + return result + + if identity.ogrip: + result = cls._resolve_unique(Organization.objects.filter(ogrip=identity.ogrip)) + if result.status != "unmatched": + return result + + if identity.inn and identity.ogrn: + return cls._resolve_by_inn_ogrn(identity.inn, identity.ogrn) + + if identity.inn: + result = cls._resolve_unique(Organization.objects.filter(inn=identity.inn)) + if result.status != "unmatched": + return result + + if identity.ogrn: + result = cls._resolve_unique(Organization.objects.filter(ogrn=identity.ogrn)) + if result.status != "unmatched": + return result + + return OrganizationResolveResult("unmatched") + + @classmethod + def _resolve_by_inn_ogrn( + cls, + inn: str, + ogrn: str, + ) -> OrganizationResolveResult: + queryset = Organization.objects.filter(inn=inn, ogrn=ogrn) + result = cls._resolve_unique(queryset) + if result.status != "ambiguous": + return result + + head_organizations = list(queryset.filter(is_branch=False)[:2]) + if len(head_organizations) == 1: + return OrganizationResolveResult("matched", head_organizations[0]) + return OrganizationResolveResult("ambiguous") + + @staticmethod + def _resolve_unique(queryset) -> OrganizationResolveResult: + organizations = list(queryset[:2]) + if not organizations: + return OrganizationResolveResult("unmatched") + if len(organizations) > 1: + return OrganizationResolveResult("ambiguous") + return OrganizationResolveResult("matched", organizations[0]) + + @staticmethod + def _single(queryset) -> Organization | None: + return queryset.first() + + @staticmethod + def _digits(value: object, *, max_length: int) -> str: + if value is None: + return "" + digits = re.sub(r"\D+", "", str(value)) + if not digits or len(digits) > max_length: + return "" + return digits diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index d413b86..6ea14c7 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -1,9 +1,6 @@ """Serializers for organization-centric API v2.""" -from django.db.models import CharField, Q -from django.db.models.functions import Cast from drf_yasg.utils import swagger_serializer_method -from registers.models import RegistryMembershipPeriod from rest_framework import serializers from organizations.models import ( @@ -13,8 +10,6 @@ from organizations.models import ( OrganizationSourceRecord, ) -REGISTRY_ORGANIZATION_CACHE_KEY = "_registry_organization_by_canonical_uid" - class OrganizationSourceFinancialLineSerializer(serializers.ModelSerializer): """Structured financial line under a source record.""" @@ -38,9 +33,22 @@ class OrganizationSourceRecordOrganizationSerializer(serializers.Serializer): uid = serializers.UUIDField(read_only=True) name = serializers.CharField(read_only=True, allow_blank=True) + full_name = serializers.CharField(read_only=True, allow_blank=True) + short_name = serializers.CharField(read_only=True, allow_blank=True) + normalized_name = serializers.CharField(read_only=True, allow_blank=True) inn = serializers.CharField(read_only=True, allow_blank=True) kpp = serializers.CharField(read_only=True, allow_blank=True) ogrn = serializers.CharField(read_only=True, allow_blank=True) + okpo = serializers.CharField(read_only=True, allow_blank=True) + legal_address = serializers.CharField(read_only=True, allow_blank=True) + business_activity = serializers.CharField(read_only=True, allow_blank=True) + dep = serializers.CharField(read_only=True, allow_blank=True) + integrated_structure = serializers.CharField(read_only=True, allow_blank=True) + gk_name = serializers.CharField(read_only=True, allow_blank=True) + in_korp_name = serializers.CharField(read_only=True, allow_blank=True) + state_sector_name = serializers.CharField(read_only=True, allow_blank=True) + filial = serializers.CharField(read_only=True, allow_blank=True) + is_branch = serializers.BooleanField(read_only=True, allow_null=True) ogrip = serializers.CharField(read_only=True, allow_blank=True) @@ -84,133 +92,31 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): @swagger_serializer_method( serializer_or_field=OrganizationSourceRecordOrganizationSerializer, ) - def get_organization(self, obj) -> dict[str, str]: + def get_organization(self, obj) -> dict[str, str | bool | None]: organization = obj.extension.organization - registry_organization = self._get_registry_organization(organization) - - name = organization.name - inn = organization.inn - kpp = organization.kpp - ogrn = organization.ogrn - if registry_organization is not None: - name = registry_organization.pn_name or name - inn = inn or self._value_to_inn_string(registry_organization.mn_inn) - kpp = kpp or self._value_to_string(registry_organization.in_kpp) - ogrn = ogrn or self._value_to_string(registry_organization.mn_ogrn) return { "uid": str(organization.uid), - "name": name, - "inn": inn, - "kpp": kpp, - "ogrn": ogrn, + "name": organization.name, + "full_name": organization.full_name, + "short_name": organization.short_name, + "normalized_name": organization.normalized_name, + "inn": organization.inn, + "kpp": organization.kpp, + "ogrn": organization.ogrn, + "okpo": organization.okpo, + "legal_address": organization.legal_address, + "business_activity": organization.business_activity, + "dep": organization.dep, + "integrated_structure": organization.integrated_structure, + "gk_name": organization.gk_name, + "in_korp_name": organization.in_korp_name, + "state_sector_name": organization.state_sector_name, + "filial": organization.filial, + "is_branch": organization.is_branch, "ogrip": organization.ogrip, } - def _get_registry_organization(self, organization: Organization): - cache = self.context.get(REGISTRY_ORGANIZATION_CACHE_KEY) - if cache is None: - cache = self._build_registry_organization_cache() - self.context[REGISTRY_ORGANIZATION_CACHE_KEY] = cache - return cache.get(organization.uid) - - def _build_registry_organization_cache(self) -> dict: - records = self._get_source_records_for_cache() - organizations_by_uid = { - record.extension.organization.uid: record.extension.organization - for record in records - } - if not organizations_by_uid: - return {} - - inn_values = { - organization.inn - for organization in organizations_by_uid.values() - if organization.inn - } - ogrn_values = { - organization.ogrn - for organization in organizations_by_uid.values() - if organization.ogrn - } - ogrn_values.update( - organization.ogrip - for organization in organizations_by_uid.values() - if organization.ogrip - ) - - identity_filter = Q() - if inn_values: - identity_filter |= Q(registry_inn_text__in=inn_values) - if ogrn_values: - identity_filter |= Q(registry_ogrn_text__in=ogrn_values) - if not identity_filter: - return {uid: None for uid in organizations_by_uid} - - memberships = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - .select_related("organization") - .annotate( - registry_inn_text=Cast( - "organization__mn_inn", - output_field=CharField(), - ), - registry_ogrn_text=Cast( - "organization__mn_ogrn", - output_field=CharField(), - ), - ) - .filter(identity_filter) - .order_by("organization__pn_name", "organization_id") - ) - - registry_by_inn = {} - registry_by_ogrn = {} - for membership in memberships: - registry_by_inn.setdefault( - membership.registry_inn_text, - membership.organization, - ) - registry_by_ogrn.setdefault( - membership.registry_ogrn_text, - membership.organization, - ) - - cache = {} - for uid, organization in organizations_by_uid.items(): - registry_organization = None - if organization.inn: - registry_organization = registry_by_inn.get(organization.inn) - if registry_organization is None and organization.ogrn: - registry_organization = registry_by_ogrn.get(organization.ogrn) - if registry_organization is None and organization.ogrip: - registry_organization = registry_by_ogrn.get(organization.ogrip) - cache[uid] = registry_organization - return cache - - def _get_source_records_for_cache(self) -> list[OrganizationSourceRecord]: - instance = getattr(self.root, "instance", None) - if instance is None: - instance = getattr(self, "instance", None) - if instance is None: - return [] - if isinstance(instance, OrganizationSourceRecord): - return [instance] - return list(instance) - - @staticmethod - def _value_to_string(value) -> str: - if value is None: - return "" - return str(value) - - @classmethod - def _value_to_inn_string(cls, value) -> str: - inn = cls._value_to_string(value) - if len(inn) in {9, 11} and inn.isdigit(): - return inn.zfill(len(inn) + 1) - return inn - class OrganizationSourceRecordListResponseSerializer(serializers.Serializer): """Paginated source-record list response in unified API format.""" @@ -253,14 +159,60 @@ class OrganizationSerializer(serializers.ModelSerializer): model = Organization fields = [ "uid", + "rn", "name", + "full_name", + "short_name", + "pn_name", + "pn_name_en", "normalized_name", + "gk_code", + "gk_name", + "in_korp_code", + "in_korp_name", "inn", "kpp", "ogrn", + "okpo", + "filial", + "is_branch", "ogrip", + "registration_date", + "create_date", + "organizational_legal_form", + "organizational_legal_form1", + "ownership_form", + "ownership_form1", + "authorized_capital", + "legal_address", + "business_act_cod", + "business_activity", + "general_director", + "general_director_tax_id", + "uk", + "inn_uk", + "appointment_date", + "cf_fl_rn", + "akc_fs", + "akc_sf", + "re_za", + "re_zasf", + "goz_participation", + "opk_registry_membership", + "ropk_num", + "ropk_razdel_num", + "ropk_razdel_name", + "min", + "dep", + "otr", + "integrated_structure", + "state_sector_code", + "state_sector_name", "identity_status", "primary_identity", + "directory_source_file_hash", + "directory_source_row_number", + "directory_imported_at", "sources", "registries", ] @@ -271,20 +223,16 @@ class OrganizationSerializer(serializers.ModelSerializer): return OrganizationSourceExtensionSerializer(extensions, many=True).data def get_registries(self, obj) -> list[dict[str, str]]: - query = RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) - if obj.inn: - query = query.filter(organization__mn_inn=obj.inn) - elif obj.ogrn: - query = query.filter(organization__mn_ogrn=obj.ogrn) - else: - return [] - - return [ - { - "id": str(membership.registry_id), - "name": membership.registry.name, - } - for membership in query.select_related("registry").order_by( - "registry__name" + registries = [] + if obj.opk_registry_membership: + registries.append( + { + "id": obj.ropk_razdel_num or "opk", + "name": obj.ropk_razdel_name or "Реестр ОПК", + } ) - ] + if obj.goz_participation: + registries.append({"id": "goz", "name": "Участие в ГОЗ"}) + if obj.gk_name: + registries.append({"id": obj.gk_code or "gk", "name": obj.gk_name}) + return registries diff --git a/src/organizations/services.py b/src/organizations/services.py index d6c581e..d2fa06d 100644 --- a/src/organizations/services.py +++ b/src/organizations/services.py @@ -17,10 +17,8 @@ from apps.parsers.models import ( ParserLoadLog, ProcurementRecord, ) -from django.db import transaction from django.db.models import Q from django.utils import timezone -from registers.models import Organization as RegisterOrganization from organizations.api_enrichment import OrganizationApiEnrichmentService from organizations.data_sources import data_source_summary @@ -351,118 +349,19 @@ def normalize_organization_name(value: str | None) -> str: class OrganizationPopulationService: - """Builds organizations from currently available source tables.""" + """Deprecated parser-based population service. + + Organizations are imported from the authoritative XLSX directory. + """ @classmethod def populate(cls) -> PopulateOrganizationsResult: - scanned = 0 - created = 0 - updated = 0 - skipped = 0 - - with transaction.atomic(): - existing = list(Organization.objects.all()) - lookup = cls._build_lookup(existing) - create_instances: list[Organization] = [] - update_instances_by_uid: dict[str, Organization] = {} - - for candidate in cls.iter_candidates(): - scanned += 1 - if not normalize_organization_name(candidate.name): - skipped += 1 - continue - - organization = cls._find_existing(lookup, candidate) - if organization is None: - organization = Organization( - name=candidate.name.strip(), - inn=candidate.inn, - kpp=candidate.kpp, - ogrn=candidate.ogrn, - ogrip=candidate.ogrip, - ) - existing.append(organization) - create_instances.append(organization) - cls._index_organization(lookup, organization) - created += 1 - continue - - if cls._assign_existing_fields(organization, candidate): - cls._index_organization(lookup, organization) - update_instances_by_uid[str(organization.uid)] = organization - updated += 1 - - if create_instances: - Organization.objects.bulk_create(create_instances, batch_size=1000) - update_instances = list(update_instances_by_uid.values()) - if update_instances: - Organization.objects.bulk_update( - update_instances, - fields=["name", "inn", "kpp", "ogrn", "ogrip"], - batch_size=1000, - ) - - return PopulateOrganizationsResult( - scanned=scanned, - created=created, - updated=updated, - skipped=skipped, - ) + return PopulateOrganizationsResult(scanned=0, created=0, updated=0, skipped=0) @classmethod def iter_candidates(cls) -> Iterable[OrganizationCandidate]: - """Yield organization candidates from all current source tables.""" - for row in RegisterOrganization.objects.iterator(): - yield cls._candidate( - name=row.pn_name, - inn=row.mn_inn, - kpp=row.in_kpp, - ogrn=row.mn_ogrn, - ) - - for row in IndustrialCertificateRecord.objects.iterator(): - yield cls._candidate( - name=row.organisation_name, - inn=row.inn, - ogrn=row.ogrn, - ) - - for row in ManufacturerRecord.objects.iterator(): - yield cls._candidate( - name=row.full_legal_name, - inn=row.inn, - ogrn=row.ogrn, - ) - - for row in IndustrialProductRecord.objects.iterator(): - yield cls._candidate( - name=row.full_organisation_name, - inn=row.inn, - ogrn=row.ogrn, - ) - - for row in GenericParserRecord.objects.iterator(): - yield cls._candidate( - name=row.organisation_name or row.title, - inn=row.inn, - kpp=cls._payload_kpp(row.payload), - ogrn=row.ogrn, - ) - - for row in InspectionRecord.objects.iterator(): - yield cls._candidate( - name=row.organisation_name, - inn=row.inn, - ogrn=row.ogrn, - ) - - for row in ProcurementRecord.objects.iterator(): - yield cls._candidate( - name=row.customer_name, - inn=row.customer_inn, - kpp=row.customer_kpp, - ogrn=row.customer_ogrn, - ) + """Parser-derived organization population is no longer supported.""" + return iter(()) @staticmethod def _candidate( diff --git a/src/organizations/signals.py b/src/organizations/signals.py index ebb344a..776ea9e 100644 --- a/src/organizations/signals.py +++ b/src/organizations/signals.py @@ -2,27 +2,14 @@ from __future__ import annotations -import logging - from apps.parsers.models import ParserLoadLog -from django.conf import settings from django.db import transaction from django.db.models.signals import post_delete, post_save from django.dispatch import receiver -from registers.models import ( - Organization as RegistryOrganization, -) -from registers.models import ( - Register, - RegisterUpload, - RegistryMembershipPeriod, -) from organizations.cache import invalidate_organization_api_cache from organizations.models import OrganizationDataSnapshot -logger = logging.getLogger(__name__) - SOURCE_UPDATE_STATUSES = { ParserLoadLog.Status.SUCCESS, ParserLoadLog.Status.SKIPPED, @@ -33,21 +20,6 @@ def _invalidate_on_commit() -> None: transaction.on_commit(invalidate_organization_api_cache) -def _warm_main_dashboard_cache_on_commit() -> None: - def enqueue_warm_task() -> None: - try: - from registers.tasks import warm_main_dashboard_cache_task - - if getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False): - warm_main_dashboard_cache_task.apply() - else: - warm_main_dashboard_cache_task.delay() - except Exception: - logger.exception("Failed to enqueue main dashboard cache warm task") - - transaction.on_commit(enqueue_warm_task) - - @receiver( post_save, sender=ParserLoadLog, dispatch_uid="organizations_parser_load_save" ) @@ -55,31 +27,8 @@ def invalidate_for_parser_load(sender, instance: ParserLoadLog, **kwargs) -> Non """Invalidate when a parser source reaches a visible terminal state.""" if instance.status in SOURCE_UPDATE_STATUSES: _invalidate_on_commit() - _warm_main_dashboard_cache_on_commit() -@receiver(post_save, sender=Register, dispatch_uid="organizations_register_save") -@receiver(post_delete, sender=Register, dispatch_uid="organizations_register_delete") -@receiver( - post_save, - sender=RegistryOrganization, - dispatch_uid="organizations_registry_organization_save", -) -@receiver( - post_delete, - sender=RegistryOrganization, - dispatch_uid="organizations_registry_organization_delete", -) -@receiver( - post_save, - sender=RegistryMembershipPeriod, - dispatch_uid="organizations_registry_membership_save", -) -@receiver( - post_delete, - sender=RegistryMembershipPeriod, - dispatch_uid="organizations_registry_membership_delete", -) @receiver( post_save, sender=OrganizationDataSnapshot, @@ -91,20 +40,5 @@ def invalidate_for_parser_load(sender, instance: ParserLoadLog, **kwargs) -> Non dispatch_uid="organizations_data_snapshot_delete", ) def invalidate_for_registry_or_snapshot_change(sender, **kwargs) -> None: - """Invalidate for direct registry and snapshot writes.""" - _invalidate_on_commit() - - -@receiver( - post_save, sender=RegisterUpload, dispatch_uid="organizations_register_upload_save" -) -def invalidate_for_successful_register_upload( - sender, - instance: RegisterUpload, - created: bool, - **kwargs, -) -> None: - """Invalidate once a registry import has completed successfully.""" - if created or instance.import_status != RegisterUpload.ImportStatus.SUCCESS: - return + """Invalidate for direct snapshot writes.""" _invalidate_on_commit() diff --git a/src/organizations/source_backfill.py b/src/organizations/source_backfill.py index b45dfff..c6d514c 100644 --- a/src/organizations/source_backfill.py +++ b/src/organizations/source_backfill.py @@ -30,6 +30,7 @@ from organizations.models import ( OrganizationSourceRecord, ) from organizations.name_normalization import normalize_organization_name +from organizations.resolver import OrganizationDirectoryResolver from organizations.source_groups import ( SOURCE_GROUP_DESCRIPTORS, SourceGroupDescriptor, @@ -139,7 +140,7 @@ class OrganizationSourceBackfillService: ( organization, organization_created, - ) = cls._resolve_or_create_organization(adapter) + ) = cls._resolve_existing_organization(adapter) if organization is None: unresolved += 1 continue @@ -289,22 +290,24 @@ class OrganizationSourceBackfillService: cls, adapter: LegacyRecordAdapter, ) -> tuple[Organization | None, bool]: - organization = cls._resolve_organization(adapter) - if organization is not None: - return organization, False + """Compatibility wrapper; backfill must not create organizations.""" + return cls._resolve_existing_organization(adapter) - name = adapter.organization_name or adapter.title or adapter.external_id - if not name: - return None, False - - organization = Organization.objects.create( - name=name, - inn=adapter.inn, - kpp=adapter.kpp, - ogrn=adapter.ogrn if len(adapter.ogrn) == 13 else "", - ogrip=adapter.ogrip, + @classmethod + def _resolve_existing_organization( + cls, + adapter: LegacyRecordAdapter, + ) -> tuple[Organization | None, bool]: + result = OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.from_payload( + payload=adapter.payload, + inn=adapter.inn, + kpp=adapter.kpp, + ogrn=adapter.ogrn, + ogrip=adapter.ogrip, + ) ) - return organization, True + return result.organization, False @classmethod def _resolve_organization(cls, adapter: LegacyRecordAdapter) -> Organization | None: diff --git a/src/organizations/source_ingestion.py b/src/organizations/source_ingestion.py index 0143599..d7b0ab5 100644 --- a/src/organizations/source_ingestion.py +++ b/src/organizations/source_ingestion.py @@ -20,6 +20,7 @@ from organizations.models import ( OrganizationSourceRecord, ) from organizations.name_normalization import normalize_organization_name +from organizations.resolver import OrganizationDirectoryResolver from organizations.source_cache import invalidate_source_data_cache from organizations.source_groups import ( SourceGroupDescriptor, @@ -72,12 +73,16 @@ class OrganizationSourceIngestionResult: created_financial_lines: int = 0 updated_financial_lines: int = 0 unresolved: int = 0 + skipped_unmatched: int = 0 + skipped_ambiguous: int = 0 @dataclass(frozen=True) class _NormalizedRecordInput: index: int record: SourceRecordInput + rn: str + okpo: str inn: str kpp: str ogrn: str @@ -131,8 +136,9 @@ class OrganizationSourceIngestionService: normalized_records = cls._normalize_records(records) ( organizations_by_index, - created_organizations, - ) = cls._resolve_or_create_organizations(normalized_records) + skipped_unmatched, + skipped_ambiguous, + ) = cls._resolve_existing_organizations(normalized_records) del normalized_records unresolved = scanned - len(organizations_by_index) @@ -174,7 +180,7 @@ class OrganizationSourceIngestionService: return OrganizationSourceIngestionResult( scanned=scanned, - created_organizations=created_organizations, + created_organizations=0, created_extensions=created_extensions, updated_extensions=updated_extensions, created_records=created_records, @@ -182,6 +188,8 @@ class OrganizationSourceIngestionService: created_financial_lines=created_financial_lines, updated_financial_lines=updated_financial_lines, unresolved=unresolved, + skipped_unmatched=skipped_unmatched, + skipped_ambiguous=skipped_ambiguous, ) @staticmethod @@ -229,6 +237,16 @@ class OrganizationSourceIngestionService: _NormalizedRecordInput( index=index, record=record_input, + rn=OrganizationDirectoryResolver._digits( + (record_input.payload or {}).get("rn") + or (record_input.payload or {}).get("organization_rn"), + max_length=20, + ), + okpo=OrganizationDirectoryResolver._digits( + (record_input.payload or {}).get("okpo") + or (record_input.payload or {}).get("mn_okpo"), + max_length=32, + ), inn=inn, kpp=kpp, ogrn=ogrn, @@ -238,42 +256,46 @@ class OrganizationSourceIngestionService: ) return normalized_records + @classmethod + def _resolve_existing_organizations( + cls, + normalized_records: list[_NormalizedRecordInput], + ) -> tuple[dict[int, Organization], int, int]: + organizations_by_index: dict[int, Organization] = {} + skipped_unmatched = 0 + skipped_ambiguous = 0 + + for record in normalized_records: + result = OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity( + rn=record.rn, + okpo=record.okpo, + inn=record.inn, + kpp=record.kpp, + ogrn=record.ogrn, + ogrip=record.ogrip, + ) + ) + if result.organization is not None: + organizations_by_index[record.index] = result.organization + continue + if result.status == "ambiguous": + skipped_ambiguous += 1 + else: + skipped_unmatched += 1 + + return organizations_by_index, skipped_unmatched, skipped_ambiguous + @classmethod def _resolve_or_create_organizations( cls, normalized_records: list[_NormalizedRecordInput], ) -> tuple[dict[int, Organization], int]: - organizations_by_index: dict[int, Organization] = {} - - cls._resolve_organizations_by_inn_kpp( - normalized_records, - organizations_by_index, + """Compatibility wrapper; parser ingestion must not create organizations.""" + organizations, _skipped_unmatched, _skipped_ambiguous = ( + cls._resolve_existing_organizations(normalized_records) ) - cls._resolve_organizations_by_ogrn_or_ogrip( - normalized_records, - organizations_by_index, - ) - cls._resolve_organizations_by_unique_inn( - normalized_records, - organizations_by_index, - ) - cls._resolve_organizations_by_exact_name( - normalized_records, - organizations_by_index, - ) - - ( - organizations_by_index, - created_organizations, - ) = cls._create_missing_organizations( - normalized_records, - organizations_by_index, - ) - cls._update_resolved_organization_identities( - normalized_records, - organizations_by_index, - ) - return organizations_by_index, created_organizations + return organizations, 0 @classmethod def _resolve_organizations_by_inn_kpp( @@ -875,34 +897,16 @@ class OrganizationSourceIngestionService: ogrn=record_input.ogrn, ogrip=record_input.ogrip, ) - organization = cls._resolve_organization( - inn=inn, - kpp=kpp, - ogrn=ogrn, - ogrip=ogrip, - organization_name=record_input.organization_name, - ) - if organization is not None: - return organization, False - - name = ( - str(record_input.organization_name or "").strip() - or str(record_input.title or "").strip() - or str(record_input.external_id or "").strip() - ) - if not name: - return None, False - - return ( - Organization.objects.create( - name=name, + result = OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.from_payload( + payload=record_input.payload, inn=inn, kpp=kpp, ogrn=ogrn, ogrip=ogrip, - ), - True, + ) ) + return result.organization, False @classmethod def _resolve_organization( @@ -914,23 +918,16 @@ class OrganizationSourceIngestionService: ogrip: str, organization_name: str, ) -> Organization | None: - for resolver in ( - cls._resolve_by_inn_kpp, - cls._resolve_by_ogrn_or_ogrip, - cls._resolve_by_ogrip, - cls._resolve_by_unique_inn, - cls._resolve_by_exact_normalized_name, - ): - organization = resolver( + result = OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity( inn=inn, kpp=kpp, ogrn=ogrn, ogrip=ogrip, organization_name=organization_name, ) - if organization is not None: - return organization - return None + ) + return result.organization @staticmethod def _resolve_by_inn_kpp( diff --git a/src/organizations/views.py b/src/organizations/views.py index f1e36f0..2bd2383 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -14,7 +14,6 @@ from django.db.models.functions import Cast from django_filters import rest_framework as filters from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema -from registers.models import RegistryMembershipPeriod from rest_framework.decorators import action from rest_framework.filters import OrderingFilter, SearchFilter from rest_framework.permissions import AllowAny, IsAuthenticated @@ -531,39 +530,14 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): @staticmethod def _registry_search_query(search_term: str) -> Q: - registry_membership = ( - RegistryMembershipPeriod.objects.filter( - ended_at__isnull=True, - ) - .order_by() - .annotate( - registry_inn_text=Cast( - "organization__mn_inn", - output_field=CharField(), - ), - registry_kpp_text=Cast( - "organization__in_kpp", - output_field=CharField(), - ), - registry_ogrn_text=Cast( - "organization__mn_ogrn", - output_field=CharField(), - ), - ) - .filter( - Q(organization__pn_name__icontains=search_term) - | Q(registry_inn_text__icontains=search_term) - | Q(registry_kpp_text__icontains=search_term) - | Q(registry_ogrn_text__icontains=search_term), - ) - ) - inn_values = registry_membership.values_list("registry_inn_text", flat=True) - ogrn_values = registry_membership.values_list("registry_ogrn_text", flat=True) - return ( - Q(extension__organization__inn__in=inn_values) - | Q(extension__organization__ogrn__in=ogrn_values) - | Q(extension__organization__ogrip__in=ogrn_values) + Q(extension__organization__name__icontains=search_term) + | Q(extension__organization__full_name__icontains=search_term) + | Q(extension__organization__short_name__icontains=search_term) + | Q(extension__organization__inn__icontains=search_term) + | Q(extension__organization__kpp__icontains=search_term) + | Q(extension__organization__ogrn__icontains=search_term) + | Q(extension__organization__okpo__icontains=search_term) ) @swagger_auto_schema( diff --git a/tests/apps/organizations/test_api_v2.py b/tests/apps/organizations/test_api_v2.py index af6642c..ec7b9b2 100644 --- a/tests/apps/organizations/test_api_v2.py +++ b/tests/apps/organizations/test_api_v2.py @@ -22,13 +22,6 @@ from organizations.models import ( from rest_framework import status from rest_framework.test import APITestCase -from tests.apps.registers.factories import ( - OrganizationFactory as RegistryOrganizationFactory, -) -from tests.apps.registers.factories import ( - RegisterFactory, - RegistryMembershipPeriodFactory, -) from tests.apps.user.factories import UserFactory @@ -241,9 +234,21 @@ class OrganizationsApiV2Test(APITestCase): def test_source_records_endpoint_returns_requested_source_payload(self): organization = Organization.objects.create( name='ООО "Явные данные"', + full_name='Общество с ограниченной ответственностью "Явные данные"', + short_name='ООО "Явные данные"', + gk_name="ГК Роскосмос", + in_korp_name="Внутри корпорации", inn="7712345683", kpp="771201006", ogrn="1027700132201", + okpo="12345678", + filial=".F.", + is_branch=False, + legal_address="127000, Москва, ул. Тестовая, д. 1", + business_activity="Производство испытательного оборудования", + dep="Департамент промышленности", + integrated_structure="Интегрированная структура", + state_sector_name="Госсектор", ) extension = IndustrialProductionExtension.objects.create( organization=organization, @@ -270,6 +275,32 @@ class OrganizationsApiV2Test(APITestCase): response.data["data"][0]["payload"], {"certificate_number": "CERT-SOURCE-RECORDS"}, ) + embedded_organization = response.data["data"][0]["organization"] + self.assertEqual( + embedded_organization["full_name"], + 'Общество с ограниченной ответственностью "Явные данные"', + ) + self.assertEqual(embedded_organization["short_name"], 'ООО "Явные данные"') + self.assertEqual(embedded_organization["normalized_name"], 'ООО "Явные данные"') + self.assertEqual(embedded_organization["okpo"], "12345678") + self.assertEqual( + embedded_organization["legal_address"], + "127000, Москва, ул. Тестовая, д. 1", + ) + self.assertEqual( + embedded_organization["business_activity"], + "Производство испытательного оборудования", + ) + self.assertEqual(embedded_organization["dep"], "Департамент промышленности") + self.assertEqual( + embedded_organization["integrated_structure"], + "Интегрированная структура", + ) + self.assertEqual(embedded_organization["gk_name"], "ГК Роскосмос") + self.assertEqual(embedded_organization["in_korp_name"], "Внутри корпорации") + self.assertEqual(embedded_organization["state_sector_name"], "Госсектор") + self.assertEqual(embedded_organization["filial"], ".F.") + self.assertEqual(embedded_organization["is_branch"], False) def test_detail_returns_compact_source_summaries_by_default(self): organization = Organization.objects.create( @@ -483,7 +514,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(second_response["X-Cache"], "HIT") self.assertEqual(third_response["X-Cache"], "MISS") - def test_registry_update_invalidates_organization_cache(self): + def test_explicit_invalidation_refreshes_organization_cache(self): Organization.objects.create( name='ООО "Реестр сброса"', inn="7744444447", @@ -495,8 +526,7 @@ class OrganizationsApiV2Test(APITestCase): first_response = self.client.get(url, params) second_response = self.client.get(url, params) - with self.captureOnCommitCallbacks(execute=True): - RegisterFactory(name="Реестр для сброса кеша") + invalidate_organization_api_cache() third_response = self.client.get(url, params) self.assertEqual(first_response["X-Cache"], "MISS") @@ -530,15 +560,9 @@ class OrganizationsApiV2Test(APITestCase): inn="7777777777", kpp="777701001", ogrn="1027700132777", - ) - registry = RegisterFactory(name="Росатом ГОЗ") - registry_organization = RegistryOrganizationFactory( - mn_inn=int(organization.inn), - mn_ogrn=int(organization.ogrn), - ) - RegistryMembershipPeriodFactory( - registry=registry, - organization=registry_organization, + opk_registry_membership=True, + ropk_razdel_num="opk-1", + ropk_razdel_name="Росатом ГОЗ", ) industrial = IndustrialProductionExtension.objects.create( organization=organization, @@ -647,7 +671,7 @@ class OrganizationsApiV2Test(APITestCase): item = response.data["data"][0] self.assertEqual( item["registries"], - [{"id": str(registry.id), "name": "Росатом ГОЗ"}], + [{"id": "opk-1", "name": "Росатом ГОЗ"}], ) self.assertNotIn("data_presence", item) self.assertNotIn("data", item) @@ -681,6 +705,9 @@ class OrganizationsApiV2Test(APITestCase): inn="7788888888", kpp="778801001", ogrn="1027700132888", + opk_registry_membership=True, + ropk_razdel_num="opk-space", + ropk_razdel_name="Роскосмос ОПК", ) without_registry = Organization.objects.create( name='ООО "Без реестра"', @@ -688,19 +715,9 @@ class OrganizationsApiV2Test(APITestCase): kpp="779901001", ogrn="1027700132999", ) - registry = RegisterFactory(name="Роскосмос ОПК") - registry_organization = RegistryOrganizationFactory( - mn_inn=int(with_registry.inn), - mn_ogrn=int(with_registry.ogrn), - ) - RegistryMembershipPeriodFactory( - registry=registry, - organization=registry_organization, - ) - by_registry = self.client.get( reverse("api_v2:organizations:organizations-list"), - {"registry": str(registry.id)}, + {"registry": "opk-space"}, ) has_registry = self.client.get( reverse("api_v2:organizations:organizations-list"), @@ -730,7 +747,7 @@ class OrganizationsApiV2Test(APITestCase): self.assertTrue(filterset.is_valid(), filterset.errors) sql = str(filterset.qs.query).upper() - self.assertIn(" IN ", sql) + self.assertIn("OPK_REGISTRY_MEMBERSHIP", sql) self.assertNotIn("EXISTS", sql) def test_list_defaults_to_has_registry_true(self): @@ -739,6 +756,9 @@ class OrganizationsApiV2Test(APITestCase): inn="7800000101", kpp="780001101", ogrn="1027700133101", + opk_registry_membership=True, + ropk_razdel_num="default", + ropk_razdel_name="Дефолтный реестр", ) Organization.objects.create( name='ООО "Скрыто по дефолту"', @@ -746,16 +766,6 @@ class OrganizationsApiV2Test(APITestCase): kpp="780001102", ogrn="1027700133102", ) - registry = RegisterFactory(name="Дефолтный реестр") - registry_organization = RegistryOrganizationFactory( - mn_inn=int(with_registry.inn), - mn_ogrn=int(with_registry.ogrn), - ) - RegistryMembershipPeriodFactory( - registry=registry, - organization=registry_organization, - ) - default_response = self.client.get( reverse("api_v2:organizations:organizations-list") ) diff --git a/tests/apps/organizations/test_api_v2_source_extensions.py b/tests/apps/organizations/test_api_v2_source_extensions.py index 9fee7fa..97ed2ab 100644 --- a/tests/apps/organizations/test_api_v2_source_extensions.py +++ b/tests/apps/organizations/test_api_v2_source_extensions.py @@ -10,13 +10,6 @@ from organizations.models import ( from rest_framework import status from rest_framework.test import APITestCase -from tests.apps.registers.factories import ( - OrganizationFactory as RegistryOrganizationFactory, -) -from tests.apps.registers.factories import ( - RegisterFactory, - RegistryMembershipPeriodFactory, -) from tests.apps.user.factories import UserFactory @@ -172,6 +165,8 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): name='ООО "With Registry Source"', inn="7707083815", ogrn="1027700132015", + opk_registry_membership=True, + ropk_razdel_num="source-records", ) without_registry = Organization.objects.create( name='ООО "Without Registry Source"', @@ -200,16 +195,6 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): external_id="INSP-WITHOUT-REGISTRY", title="Проверка организации без реестра", ) - registry = RegisterFactory(name="Реестр source records") - registry_organization = RegistryOrganizationFactory( - mn_inn=int(with_registry.inn), - mn_ogrn=int(with_registry.ogrn), - ) - RegistryMembershipPeriodFactory( - registry=registry, - organization=registry_organization, - ) - only_registry = self.client.get( reverse("api_v2:organizations:organization-source-records-list"), { @@ -245,8 +230,12 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): def test_source_record_organization_uses_active_registry_identity(self): organization = Organization.objects.create( - name="1241800009703", + name='ООО "Реестровое имя"', + inn="1800020960", + kpp="180001001", ogrn="1241800009703", + opk_registry_membership=True, + ropk_razdel_num="identity", ) extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -259,14 +248,6 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): external_id="INSP-REGISTRY-IDENTITY", title="Проверка организации из реестра", ) - registry_organization = RegistryOrganizationFactory( - pn_name='ООО "Реестровое имя"', - mn_inn=1800020960, - mn_ogrn=int(organization.ogrn), - in_kpp=180001001, - ) - RegistryMembershipPeriodFactory(organization=registry_organization) - response = self.client.get( reverse("api_v2:organizations:organization-source-records-list"), { @@ -289,6 +270,8 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): name='ООО "Башнефть-Добыча"', inn="0277106840", ogrn="1090280032699", + opk_registry_membership=True, + ropk_razdel_num="leading-zero", ) extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -301,14 +284,6 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): external_id="INSP-LEADING-ZERO-INN", title="Проверка организации с ведущим нулем", ) - registry_organization = RegistryOrganizationFactory( - pn_name='ООО "Башнефть-Добыча"', - mn_inn=277106840, - mn_ogrn=int(organization.ogrn), - in_kpp=27701001, - ) - RegistryMembershipPeriodFactory(organization=registry_organization) - response = self.client.get( reverse("api_v2:organizations:organization-source-records-list"), { @@ -389,8 +364,12 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): def test_flat_source_records_searches_registry_identity_displayed_in_tables(self): organization = Organization.objects.create( - name="1241800009704", + name='АО "Реестровый поиск"', + inn="1800020961", + kpp="180001002", ogrn="1241800009704", + opk_registry_membership=True, + ropk_razdel_num="search", ) extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -403,14 +382,6 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): external_id="INSP-REGISTRY-SEARCH", title="Проверка поиска по реестру", ) - registry_organization = RegistryOrganizationFactory( - pn_name='АО "Реестровый поиск"', - mn_inn=1800020961, - mn_ogrn=int(organization.ogrn), - in_kpp=180001002, - ) - RegistryMembershipPeriodFactory(organization=registry_organization) - response = self.client.get( reverse("api_v2:organizations:organization-source-records-list"), { diff --git a/tests/apps/organizations/test_directory_import.py b/tests/apps/organizations/test_directory_import.py new file mode 100644 index 0000000..739abb3 --- /dev/null +++ b/tests/apps/organizations/test_directory_import.py @@ -0,0 +1,218 @@ +"""Tests for authoritative organization directory import.""" + +from decimal import Decimal +from tempfile import NamedTemporaryFile + +from django.test import TestCase +from openpyxl import Workbook +from organizations.directory_import import ( + DIRECTORY_SHEET, + SOURCE_HEADERS, + OrganizationDirectoryImportError, + OrganizationDirectoryImportService, +) +from organizations.models import Organization + + +class OrganizationDirectoryImportServiceTest(TestCase): + """Checks importing the authoritative XLSX organization directory.""" + + def test_import_xlsx_creates_and_updates_organizations_by_rn(self): + path = self._workbook_path( + [ + self._row( + rn="10", + full_name='Акционерное общество "Тест"', + short_name='АО "Тест"', + inn="7701001001", + ogrn="1027700100001", + okpo="00123456", + filial=".F.", + kpp="770101001", + registration_date="01.02.2020", + authorized_capita=1000, + business_act_cod="2", + goz_participation=".T.", + opk_registry_membership=".F.", + _gk="1", + in_korp="1", + _k="2", + ) + ] + ) + + first = OrganizationDirectoryImportService.import_xlsx(path) + second = OrganizationDirectoryImportService.import_xlsx(path) + + self.assertEqual(first.scanned, 1) + self.assertEqual(first.created, 1) + self.assertEqual(second.updated, 1) + self.assertEqual(Organization.objects.count(), 1) + + organization = Organization.objects.get(rn=10) + self.assertEqual(organization.name, 'Акционерное общество "Тест"') + self.assertEqual(organization.okpo, "00123456") + self.assertFalse(organization.is_branch) + self.assertEqual(organization.authorized_capital, Decimal("1000.00")) + self.assertEqual(organization.business_activity, "Производственная") + self.assertEqual(organization.gk_name, "Роскосмос") + self.assertEqual(organization.in_korp_name, "Входит в состав") + self.assertEqual(organization.state_sector_name, "Головные исполнители") + self.assertTrue(organization.goz_participation) + self.assertFalse(organization.opk_registry_membership) + + def test_import_xlsx_rejects_missing_required_column(self): + path = self._workbook_path([], headers=SOURCE_HEADERS[:-1]) + + with self.assertRaisesMessage( + OrganizationDirectoryImportError, + "Missing required columns", + ): + OrganizationDirectoryImportService.import_xlsx(path) + + def test_import_xlsx_treats_spaced_dot_date_marker_as_empty(self): + path = self._workbook_path( + [ + self._row( + rn="11", + registration_date=". .", + appointment_date=". .", + filial=".T.", + ) + ] + ) + + result = OrganizationDirectoryImportService.import_xlsx(path) + + self.assertEqual(result.created, 1) + organization = Organization.objects.get(rn=11) + self.assertIsNone(organization.registration_date) + self.assertIsNone(organization.appointment_date) + self.assertTrue(organization.is_branch) + + def test_import_xlsx_recovers_embedded_tsv_organization_row(self): + first_tail = [ + *[ + self._row( + rn="20", + pn_name_en="First EN", + inn="7701001020", + ogrn="1027700100020", + okpo="00000020", + filial=".F.", + kpp="770101020", + registration_date="01.01.2020", + _k="2", + )[header] + for header in SOURCE_HEADERS[6:-1] + ], + "20", + "2", + ] + embedded_value = "\t".join(str(value or "") for value in first_tail) + embedded_value = ( + embedded_value + + "_x000D_\n" + + "\t".join(["21", "0", "0", 'ООО "Вторая"']) + ) + + second_row = self._row( + rn="21", + full_name='ООО "Вторая"', + short_name='ООО "Вторая"', + pn_name='ООО "Вторая"', + inn="7701001021", + ogrn="1027700100021", + okpo="00000021", + filial=".F.", + kpp="770101021", + registration_date="02.01.2020", + _k="2", + ) + second_row["is"] = "21" + path = self._workbook_path( + [ + { + "__raw_row__": [ + "20", + "0", + "0", + 'ООО "Первая"', + 'ООО "Первая"', + 'ООО "Первая"', + embedded_value, + *[second_row[header] for header in SOURCE_HEADERS[4:]], + ] + } + ] + ) + + result = OrganizationDirectoryImportService.import_xlsx(path) + + self.assertEqual(result.scanned, 1) + self.assertEqual(result.created, 2) + self.assertEqual(Organization.objects.count(), 2) + self.assertEqual(Organization.objects.get(rn=20).inn, "7701001020") + second = Organization.objects.get(rn=21) + self.assertEqual(second.inn, "7701001021") + self.assertEqual(second.integrated_structure, "") + + def _workbook_path( + self, + rows: list[dict[str, object]], + *, + headers: list[str] | None = None, + ) -> str: + workbook = Workbook() + worksheet = workbook.active + worksheet.title = DIRECTORY_SHEET + worksheet.append(headers or SOURCE_HEADERS) + for values in rows: + raw_row = values.get("__raw_row__") + if raw_row is not None: + worksheet.append(raw_row) + else: + worksheet.append([values.get(header, "") for header in SOURCE_HEADERS]) + + self._append_reference_sheet(workbook, "_gk", {"1": "Роскосмос"}) + self._append_reference_sheet(workbook, "in_korp", {"1": "Входит в состав"}) + self._append_reference_sheet( + workbook, + "business_act_cod", + {"2": "Производственная"}, + ) + self._append_reference_sheet( + workbook, + "_k - Госектор", + {"2": "Головные исполнители"}, + ) + + temp_file = NamedTemporaryFile(suffix=".xlsx", delete=False) + temp_file.close() + workbook.save(temp_file.name) + return temp_file.name + + @staticmethod + def _append_reference_sheet( + workbook: Workbook, + title: str, + values: dict[str, str], + ) -> None: + worksheet = workbook.create_sheet(title=title) + worksheet.append(["kod", "value"]) + for key, value in values.items(): + worksheet.append([key, value]) + + @staticmethod + def _row(**overrides) -> dict[str, object]: + row = {header: "" for header in SOURCE_HEADERS} + row.update( + { + "rn": "1", + "full_name": 'ООО "По умолчанию"', + "inn": "7701001000", + "ogrn": "1027700100000", + } + ) + row.update(overrides) + return row diff --git a/tests/apps/organizations/test_populate_command.py b/tests/apps/organizations/test_populate_command.py index 230da58..f7ea92c 100644 --- a/tests/apps/organizations/test_populate_command.py +++ b/tests/apps/organizations/test_populate_command.py @@ -1,128 +1,30 @@ -"""Tests for populate_organizations command.""" +"""Tests for deprecated populate_organizations command.""" -from apps.parsers.models import GenericParserRecord, ProcurementRecord +from apps.parsers.models import GenericParserRecord from django.core.management import call_command from django.test import TestCase from organizations.models import Organization -from registers.models import Organization as RegisterOrganization class PopulateOrganizationsCommandTest(TestCase): - """Checks idempotent organization population from existing DB records.""" + """Checks parser-derived organization population is disabled.""" - def test_command_populates_from_registers_and_parser_records(self): - RegisterOrganization.objects.create( - pn_name='Общество с ограниченной ответственностью "Ромашка"', - mn_inn=7707083893, - mn_ogrn=1027700132195, - in_kpp=770701001, - mn_okpo="12345678", - ) - ProcurementRecord.objects.create( - load_batch=1, - purchase_number="001", - purchase_name="Поставка", - customer_inn="7707083893", - customer_kpp="770701001", - customer_ogrn="1027700132195", - customer_name='OOO "Ромашка"', - ) + def test_command_does_not_create_organizations_from_parser_records(self): GenericParserRecord.objects.create( load_batch=1, source="arbitration", external_id="name-only-1", organisation_name='ООО "Ромашка"', - ) - GenericParserRecord.objects.create( - load_batch=1, - source="trudvsem", - external_id="name-only-2", - organisation_name="ИП Иванов Иван Иванович", - inn="500100732259", - ogrn="304500116000157", - ) - - call_command("populate_organizations", silent=True, verbosity=0) - call_command("populate_organizations", silent=True, verbosity=0) - - self.assertEqual(Organization.objects.count(), 2) - - legal_entity = Organization.objects.get(inn="7707083893") - self.assertEqual(legal_entity.name, 'Общество с ограниченной ответственностью "Ромашка"') - self.assertEqual(legal_entity.kpp, "770701001") - self.assertEqual(legal_entity.ogrn, "1027700132195") - self.assertEqual(legal_entity.ogrip, "") - - entrepreneur = Organization.objects.get(inn="500100732259") - self.assertEqual(entrepreneur.name, "ИП Иванов Иван Иванович") - self.assertEqual(entrepreneur.kpp, "") - self.assertEqual(entrepreneur.ogrn, "") - self.assertEqual(entrepreneur.ogrip, "304500116000157") - - def test_command_merges_name_only_ooo_variants(self): - GenericParserRecord.objects.create( - load_batch=1, - source="arbitration", - external_id="name-only-1", - organisation_name='OOO "Ромашка"', - ) - GenericParserRecord.objects.create( - load_batch=1, - source="trudvsem", - external_id="name-only-2", - organisation_name="Общество с ограниченной ответственностью Ромашка", - ) - - call_command("populate_organizations", silent=True, verbosity=0) - - self.assertEqual(Organization.objects.count(), 1) - self.assertEqual(Organization.objects.get().name, "Общество с ограниченной ответственностью Ромашка") - - def test_command_populates_kpp_from_generic_payload_company(self): - GenericParserRecord.objects.create( - load_batch=1, - source="trudvsem", - external_id="generic-with-kpp", - organisation_name='ООО "Источник"', inn="7707083893", ogrn="1027700132195", - payload={"company": {"kpp": "770701001"}}, ) call_command("populate_organizations", silent=True, verbosity=0) - organization = Organization.objects.get(inn="7707083893") - self.assertEqual(organization.kpp, "770701001") + self.assertEqual(Organization.objects.count(), 0) - def test_command_keeps_same_inn_different_kpp_as_distinct_branches(self): - GenericParserRecord.objects.create( - load_batch=1, - source="trudvsem", - external_id="branch-1", - organisation_name='ООО "Источник"', - inn="7707083893", - ogrn="1027700132195", - payload={"company": {"kpp": "770701001"}}, - ) - GenericParserRecord.objects.create( - load_batch=1, - source="trudvsem", - external_id="branch-2", - organisation_name='ООО "Источник"', - inn="7707083893", - ogrn="1027700132195", - payload={"company": {"kpp": "780101001"}}, - ) - - call_command("populate_organizations", silent=True, verbosity=0) - - self.assertEqual( - set(Organization.objects.filter(inn="7707083893").values_list("kpp", flat=True)), - {"770701001", "780101001"}, - ) - - def test_command_updates_existing_blank_kpp_organization_from_branch_candidate(self): - Organization.objects.create( + def test_command_does_not_update_existing_organization_identity(self): + organization = Organization.objects.create( name='ООО "Источник"', inn="7707083893", ogrn="1027700132195", @@ -139,5 +41,5 @@ class PopulateOrganizationsCommandTest(TestCase): call_command("populate_organizations", silent=True, verbosity=0) - self.assertEqual(Organization.objects.filter(inn="7707083893").count(), 1) - self.assertEqual(Organization.objects.get(inn="7707083893").kpp, "770701001") + organization.refresh_from_db() + self.assertEqual(organization.kpp, "") diff --git a/tests/apps/organizations/test_resolver.py b/tests/apps/organizations/test_resolver.py new file mode 100644 index 0000000..0df8591 --- /dev/null +++ b/tests/apps/organizations/test_resolver.py @@ -0,0 +1,102 @@ +"""Tests for organization directory resolver.""" + +from django.test import TestCase +from organizations.models import Organization +from organizations.resolver import OrganizationDirectoryResolver + + +class OrganizationDirectoryResolverTest(TestCase): + """Checks matching parser rows to the organization directory.""" + + def test_resolves_by_rn_okpo_and_exact_pairs(self): + by_rn = Organization.objects.create(name="RN", rn=100) + by_okpo = Organization.objects.create(name="OKPO", okpo="001") + by_inn_kpp = Organization.objects.create( + name="INN KPP", + inn="7701001001", + kpp="770101001", + ) + by_ogrn_kpp = Organization.objects.create( + name="OGRN KPP", + ogrn="1027700100001", + kpp="770101002", + ) + + self.assertEqual( + OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity(rn="100") + ).organization, + by_rn, + ) + self.assertEqual( + OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity(okpo="001") + ).organization, + by_okpo, + ) + self.assertEqual( + OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity( + inn="7701001001", + kpp="770101001", + ) + ).organization, + by_inn_kpp, + ) + self.assertEqual( + OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity( + ogrn="1027700100001", + kpp="770101002", + ) + ).organization, + by_ogrn_kpp, + ) + + def test_duplicate_inn_ogrn_uses_single_head(self): + head = Organization.objects.create( + name="Head", + inn="7701001001", + ogrn="1027700100001", + is_branch=False, + ) + Organization.objects.create( + name="Branch", + inn="7701001001", + ogrn="1027700100001", + is_branch=True, + ) + + result = OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity( + inn="7701001001", + ogrn="1027700100001", + ) + ) + + self.assertEqual(result.status, "matched") + self.assertEqual(result.organization, head) + + def test_duplicate_inn_ogrn_without_single_head_is_ambiguous(self): + Organization.objects.create( + name="Head 1", + inn="7701001001", + ogrn="1027700100001", + is_branch=False, + ) + Organization.objects.create( + name="Head 2", + inn="7701001001", + ogrn="1027700100001", + is_branch=False, + ) + + result = OrganizationDirectoryResolver.resolve( + OrganizationDirectoryResolver.identity( + inn="7701001001", + ogrn="1027700100001", + ) + ) + + self.assertEqual(result.status, "ambiguous") + self.assertIsNone(result.organization) diff --git a/tests/apps/organizations/test_source_backfill.py b/tests/apps/organizations/test_source_backfill.py index 17627fd..943b3b1 100644 --- a/tests/apps/organizations/test_source_backfill.py +++ b/tests/apps/organizations/test_source_backfill.py @@ -23,9 +23,6 @@ from tests.apps.parsers.factories import ( InspectionRecordFactory, ManufacturerRecordFactory, ) -from tests.apps.registers.factories import ( - OrganizationFactory as RegistryOrganizationFactory, -) class OrganizationSourceBackfillServiceTest(TestCase): @@ -79,17 +76,13 @@ class OrganizationSourceBackfillServiceTest(TestCase): inn="7707083815", ogrn="1027700132015", ) - registry_organization = RegistryOrganizationFactory( - mn_inn=int(organization.inn), - mn_ogrn=int(organization.ogrn), - ) InspectionRecordFactory( load_batch=78, inn=organization.inn, ogrn=organization.ogrn, organisation_name=organization.name, registration_number="INSP-FK", - registry_organization=registry_organization, + registry_organization=organization, ) OrganizationSourceBackfillService.backfill( @@ -98,7 +91,7 @@ class OrganizationSourceBackfillServiceTest(TestCase): ) record = OrganizationSourceRecord.objects.get(external_id="INSP-FK") - self.assertEqual(record.payload["registry_organization"], registry_organization.pk) + self.assertEqual(record.payload["registry_organization"], str(organization.pk)) def test_backfills_financial_report_lines(self): organization = Organization.objects.create( @@ -169,7 +162,7 @@ class OrganizationSourceBackfillServiceTest(TestCase): self.assertEqual(record.external_id, generic_record.external_id) self.assertEqual(record.payload["reason"], "test") - def test_backfill_drops_invalid_identity_values_when_creating_organization(self): + def test_backfill_skips_invalid_identity_without_directory_organization(self): manufacturer = ManufacturerRecordFactory( load_batch=100, full_legal_name='ИП "Грязный ИНН"', @@ -183,14 +176,17 @@ class OrganizationSourceBackfillServiceTest(TestCase): ) self.assertEqual(result.scanned, 1) - self.assertEqual(result.created_organizations, 1) - self.assertEqual(result.created_records, 1) - self.assertEqual(result.unresolved, 0) - organization = Organization.objects.get(name=manufacturer.full_legal_name) - self.assertEqual(organization.inn, "") - self.assertEqual(organization.ogrn, manufacturer.ogrn) - record = OrganizationSourceRecord.objects.get(legacy_pk=str(manufacturer.pk)) - self.assertEqual(record.payload["inn"], "396440000001000") + self.assertEqual(result.created_organizations, 0) + self.assertEqual(result.created_records, 0) + self.assertEqual(result.unresolved, 1) + self.assertFalse( + Organization.objects.filter(name=manufacturer.full_legal_name).exists() + ) + self.assertFalse( + OrganizationSourceRecord.objects.filter( + legacy_pk=str(manufacturer.pk) + ).exists() + ) def test_management_command_runs_source_backfill(self): organization = Organization.objects.create( diff --git a/tests/apps/organizations/test_source_ingestion.py b/tests/apps/organizations/test_source_ingestion.py index 2e1631c..a8e9e04 100644 --- a/tests/apps/organizations/test_source_ingestion.py +++ b/tests/apps/organizations/test_source_ingestion.py @@ -23,6 +23,11 @@ class OrganizationSourceIngestionServiceTest(TestCase): """Checks runtime parser writes bypass legacy parser record tables.""" def test_save_generic_records_writes_source_records_without_legacy_rows(self): + Organization.objects.create( + name='ООО "ГОЗ"', + inn="7707083803", + ) + result = OrganizationSourceIngestionService.save_records( source=ParserLoadLog.Source.UNFAIR_SUPPLIERS, load_batch=42, @@ -47,6 +52,7 @@ class OrganizationSourceIngestionServiceTest(TestCase): self.assertEqual(result.updated_records, 0) self.assertEqual(result.created_extensions, 1) self.assertEqual(result.unresolved, 0) + self.assertEqual(result.created_organizations, 0) self.assertEqual(GenericParserRecord.objects.count(), 0) organization = Organization.objects.get(inn="7707083803") @@ -64,6 +70,11 @@ class OrganizationSourceIngestionServiceTest(TestCase): self.assertEqual(record.legacy_pk, "") def test_save_records_is_idempotent_by_source_external_id(self): + Organization.objects.create( + name='ООО "Идемпотентность"', + inn="7707083810", + ) + first = SourceRecordInput( external_id="unfair-idempotent", title="Old title", @@ -101,7 +112,7 @@ class OrganizationSourceIngestionServiceTest(TestCase): self.assertEqual(record.payload["version"], 2) self.assertEqual(record.load_batch, 44) - def test_save_records_does_not_attach_identified_record_to_name_only_org(self): + def test_save_records_skips_identified_record_without_directory_match(self): name_only = Organization.objects.create(name="Acme Security") result = OrganizationSourceIngestionService.save_records( @@ -119,17 +130,19 @@ class OrganizationSourceIngestionServiceTest(TestCase): ], ) - self.assertEqual(result.created_records, 1) - organization = Organization.objects.get(inn="7713497980") - self.assertNotEqual(organization.uid, name_only.uid) - self.assertEqual(organization.ogrn, "1237700253306") - record = OrganizationSourceRecord.objects.get(external_id="fstec-5066") - self.assertEqual(record.extension.organization, organization) + self.assertEqual(result.created_records, 0) + self.assertEqual(result.unresolved, 1) + self.assertEqual(result.skipped_unmatched, 1) + self.assertFalse(Organization.objects.filter(inn="7713497980").exists()) + self.assertFalse( + OrganizationSourceRecord.objects.filter(external_id="fstec-5066").exists() + ) self.assertFalse(name_only.source_extensions.exists()) - def test_save_records_fills_missing_identity_on_resolved_organization(self): + def test_save_records_does_not_fill_missing_identity_on_resolved_organization(self): partial = Organization.objects.create( - name="1237700253306", + name="Acme Security", + inn="7713497980", ogrn="1237700253306", ) @@ -154,6 +167,78 @@ class OrganizationSourceIngestionServiceTest(TestCase): record = OrganizationSourceRecord.objects.get(external_id="fstec-5067") self.assertEqual(record.extension.organization, partial) + def test_save_records_uses_single_head_organization_for_duplicate_inn_ogrn(self): + head = Organization.objects.create( + name="Head", + inn="7713497980", + ogrn="1237700253306", + filial=".F.", + is_branch=False, + ) + Organization.objects.create( + name="Branch", + inn="7713497980", + ogrn="1237700253306", + filial=".T.", + is_branch=True, + ) + + result = OrganizationSourceIngestionService.save_records( + source=ParserLoadLog.Source.FSTEC, + load_batch=49, + records=[ + SourceRecordInput( + external_id="fstec-head", + title="Weblock", + organization_name="Head", + inn="7713497980", + ogrn="1237700253306", + ) + ], + ) + + self.assertEqual(result.created_records, 1) + record = OrganizationSourceRecord.objects.get(external_id="fstec-head") + self.assertEqual(record.extension.organization, head) + + def test_save_records_skips_ambiguous_duplicate_inn_ogrn(self): + Organization.objects.create( + name="Head 1", + inn="7713497980", + ogrn="1237700253306", + filial=".F.", + is_branch=False, + ) + Organization.objects.create( + name="Head 2", + inn="7713497980", + ogrn="1237700253306", + filial=".F.", + is_branch=False, + ) + + result = OrganizationSourceIngestionService.save_records( + source=ParserLoadLog.Source.FSTEC, + load_batch=50, + records=[ + SourceRecordInput( + external_id="fstec-ambiguous", + title="Weblock", + organization_name="Head", + inn="7713497980", + ogrn="1237700253306", + ) + ], + ) + + self.assertEqual(result.created_records, 0) + self.assertEqual(result.skipped_ambiguous, 1) + self.assertFalse( + OrganizationSourceRecord.objects.filter( + external_id="fstec-ambiguous" + ).exists() + ) + def test_save_records_ignores_ogrip_like_ogrn_for_legal_entity_inn(self): result = OrganizationSourceIngestionService.save_records( source=ParserLoadLog.Source.INDUSTRIAL, @@ -169,10 +254,9 @@ class OrganizationSourceIngestionServiceTest(TestCase): ], ) - self.assertEqual(result.created_records, 1) - organization = Organization.objects.get(inn="7720525156") - self.assertEqual(organization.ogrn, "") - self.assertEqual(organization.ogrip, "") + self.assertEqual(result.created_records, 0) + self.assertEqual(result.unresolved, 1) + self.assertFalse(Organization.objects.filter(inn="7720525156").exists()) def test_save_records_does_not_add_ogrip_to_existing_legal_entity(self): organization = Organization.objects.create( @@ -196,11 +280,17 @@ class OrganizationSourceIngestionServiceTest(TestCase): ) self.assertEqual(result.created_records, 1) + self.assertEqual(result.unresolved, 0) organization.refresh_from_db() self.assertEqual(organization.ogrn, "1057746645395") self.assertEqual(organization.ogrip, "") def test_save_financial_report_writes_financial_lines_without_legacy_report(self): + Organization.objects.create( + name='ООО "Финансовый отчет"', + ogrn="1027700132002", + ) + result = OrganizationSourceIngestionService.save_records( source=ParserLoadLog.Source.FNS_REPORTS, load_batch=88, diff --git a/tests/apps/parsers/test_admin.py b/tests/apps/parsers/test_admin.py index 36b5654..3f7495e 100644 --- a/tests/apps/parsers/test_admin.py +++ b/tests/apps/parsers/test_admin.py @@ -31,11 +31,12 @@ from django.contrib.messages.storage.fallback import FallbackStorage from django.core.files.uploadedfile import SimpleUploadedFile from django.test import RequestFactory, TestCase, override_settings from django.urls import reverse +from openpyxl import Workbook from organizations.models import ( + Organization, OrganizationSourceFinancialLine, OrganizationSourceRecord, ) -from openpyxl import Workbook from tests.apps.parsers.factories import ( IndustrialCertificateRecordFactory, @@ -78,8 +79,13 @@ def _build_fns_excel_bytes() -> bytes: def _build_fns_zip_upload() -> SimpleUploadedFile: buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + ogrn = _digits(13) + Organization.objects.create( + name=f"ФНС {ogrn}", + ogrn=ogrn, + ) archive.writestr( - f"fin_{_digits(5)}_{_digits(13)}.xlsx", + f"fin_{_digits(5)}_{ogrn}.xlsx", _build_fns_excel_bytes(), ) archive.writestr("ignored.txt", b"invalid") @@ -91,8 +97,13 @@ def _build_fns_zip_upload() -> SimpleUploadedFile: def _build_fns_excel_upload() -> SimpleUploadedFile: + ogrn = _digits(13) + Organization.objects.create( + name=f"ФНС {ogrn}", + ogrn=ogrn, + ) return SimpleUploadedFile( - f"fin_{_digits(5)}_{_digits(13)}.xlsx", + f"fin_{_digits(5)}_{ogrn}.xlsx", _build_fns_excel_bytes(), content_type=( "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" @@ -366,7 +377,7 @@ class ParsersAdminTest(TestCase): self.assertEqual(admin.lines_count(report), 1) self.assertIn("span", str(admin.status_badge(report))) self.assertIn("registry_organization", admin.list_display) - self.assertIn("registry_organization__pn_name", admin.search_fields) + self.assertIn("registry_organization__name", admin.search_fields) route_names = [route.name for route in admin.get_urls()] self.assertIn("parsers_financialreport_upload_excel", route_names) self.assertIn("parsers_financialreport_upload_zip", route_names) diff --git a/tests/apps/parsers/test_direct_ingestion_services.py b/tests/apps/parsers/test_direct_ingestion_services.py index bea67b4..6d20b19 100644 --- a/tests/apps/parsers/test_direct_ingestion_services.py +++ b/tests/apps/parsers/test_direct_ingestion_services.py @@ -32,6 +32,7 @@ from apps.parsers.services import ( ) from django.test import TestCase from organizations.models import ( + Organization, OrganizationSourceFinancialLine, OrganizationSourceRecord, ) @@ -40,6 +41,25 @@ from organizations.models import ( class DirectIngestionParserServicesTest(TestCase): """Parser save services should not write legacy parser record rows.""" + def setUp(self): + for inn, ogrn in ( + ("7707083801", "1027700132001"), + ("7707083802", "1027700132002"), + ("7707083803", ""), + ("7707083804", "1027700132004"), + ("", "1027700132005"), + ("", "1027700132006"), + ("7707083809", "1027700132009"), + ("7707083810", "1027700132010"), + ): + Organization.objects.create( + name=f"Справочник {inn or ogrn}", + inn=inn, + ogrn=ogrn, + kpp="770701001" if inn == "7707083810" else "", + okpo="", + ) + def test_industrial_certificate_save_records_writes_organization_source_records(self): saved = IndustrialCertificateService.save_certificates( [ diff --git a/tests/apps/parsers/test_e2e.py b/tests/apps/parsers/test_e2e.py index 07e0ee8..23a2828 100644 --- a/tests/apps/parsers/test_e2e.py +++ b/tests/apps/parsers/test_e2e.py @@ -12,7 +12,7 @@ from apps.parsers.clients.zakupki import ZakupkiClient from apps.parsers.models import ParserLoadLog, ProcurementRecord from apps.parsers.services import ParserLoadLogService, ProcurementService from django.test import TestCase, override_settings -from organizations.models import OrganizationSourceRecord +from organizations.models import Organization, OrganizationSourceRecord from tests.utils import TestHTTPServer from tests.utils.fixtures import build_zakupki_xml, build_zip, fake @@ -43,6 +43,14 @@ def _add_zakupki_zip( count: int = 3, ) -> int: xml_bytes, rows = build_zakupki_xml(count=count) + for row in rows: + Organization.objects.create( + name=row.customer_name, + inn=row.customer_inn, + kpp=row.customer_kpp, + ogrn=row.customer_ogrn, + okpo=_digits(8), + ) zip_bytes = build_zip([(f"data_{region_code}_{year}_{month}.xml", xml_bytes)]) fz_suffix = f"fz{law_type}" path = ( diff --git a/tests/apps/parsers/test_fns_parser.py b/tests/apps/parsers/test_fns_parser.py index cb02838..9ea0ce8 100644 --- a/tests/apps/parsers/test_fns_parser.py +++ b/tests/apps/parsers/test_fns_parser.py @@ -5,9 +5,12 @@ from apps.parsers.clients.fns.parser import FNSExcelParser, FNSParserError from apps.parsers.clients.fns.schemas import ParsedReport, ReportLine from apps.parsers.models import FinancialReport -from apps.parsers.services import FNSReportService -from apps.registers.models import Organization +from apps.parsers.services import ( + FNSReportOrganizationResolutionSkipped, + FNSReportService, +) from django.test import TestCase +from organizations.models import Organization from tests.utils.fixtures import fake @@ -30,11 +33,11 @@ def _form_code() -> str: def _create_registry_organization(*, inn: str, ogrn: str) -> Organization: return Organization.objects.create( - pn_name=fake.company(), - mn_ogrn=int(ogrn), - mn_inn=int(inn), - in_kpp=int(_digits(9)), - mn_okpo=_digits(8), + name=fake.company(), + ogrn=ogrn, + inn=inn, + kpp=_digits(9), + okpo=_digits(8), ) @@ -277,6 +280,7 @@ class TestFNSReportServiceIntegration(TestCase): """Сохранение отчета.""" external_id = _digits(6) ogrn = _digits(13) + _create_registry_organization(inn=_digits(10), ogrn=ogrn) file_hash = fake.sha1(raw_output=False) lines_data = [ { @@ -314,7 +318,7 @@ class TestFNSReportServiceIntegration(TestCase): self.assertEqual(report.lines.count(), 2) def test_save_report_links_registry_organization_when_exists(self): - """Отчет ФНС должен связываться с организацией из registers по ОГРН.""" + """Отчет ФНС должен связываться с организацией из справочника по ОГРН.""" ogrn = _digits(13) organization = _create_registry_organization(inn=_digits(10), ogrn=ogrn) external_id = _digits(6) @@ -331,12 +335,32 @@ class TestFNSReportServiceIntegration(TestCase): self.assertEqual(report.registry_organization_id, organization.id) + def test_save_report_raises_skipped_when_organization_is_ambiguous(self): + ogrn = _digits(13) + _create_registry_organization(inn=_digits(10), ogrn=ogrn) + _create_registry_organization(inn=_digits(10), ogrn=ogrn) + + with self.assertRaises(FNSReportOrganizationResolutionSkipped) as exc: + FNSReportService.save_report( + external_id=_digits(6), + ogrn=ogrn, + file_name=f"fin_{_digits(6)}_{ogrn}.xlsx", + file_hash=fake.sha1(raw_output=False), + source=FinancialReport.SourceType.API, + batch_id=1, + lines_data=[], + ) + + self.assertEqual(exc.exception.reason, "ambiguous") + def test_exists_by_hash(self): """Проверка существования по хешу.""" unique_hash = fake.sha1(raw_output=False) + ogrn = _digits(13) + _create_registry_organization(inn=_digits(10), ogrn=ogrn) FNSReportService.save_report( external_id=_digits(6), - ogrn=_digits(13), + ogrn=ogrn, file_name=fake.file_name(extension="xlsx"), file_hash=unique_hash, source=FinancialReport.SourceType.API, @@ -350,6 +374,7 @@ class TestFNSReportServiceIntegration(TestCase): def test_find_by_ogrn(self): """Поиск по ОГРН.""" ogrn = _digits(13) + _create_registry_organization(inn=_digits(10), ogrn=ogrn) FNSReportService.save_report( external_id=_digits(6), ogrn=ogrn, diff --git a/tests/apps/parsers/test_fns_upload.py b/tests/apps/parsers/test_fns_upload.py index 5ea0dde..9d18a30 100644 --- a/tests/apps/parsers/test_fns_upload.py +++ b/tests/apps/parsers/test_fns_upload.py @@ -14,11 +14,12 @@ from apps.parsers.models import FinancialReport from django.core.files.uploadedfile import SimpleUploadedFile from django.test import override_settings from django.urls import reverse +from openpyxl import Workbook from organizations.models import ( + Organization, OrganizationSourceFinancialLine, OrganizationSourceRecord, ) -from openpyxl import Workbook from rest_framework import status from rest_framework.test import APITestCase @@ -53,6 +54,13 @@ def _build_fns_zip_bytes(file_map: dict[str, bytes]) -> bytes: return buffer.getvalue() +def _ensure_directory_organization(*, ogrn: str) -> Organization: + return Organization.objects.create( + name=f"ФНС {ogrn}", + ogrn=ogrn, + ) + + class FNSUploadIntegrationTest(APITestCase): """Tests real upload + processing of FNS files.""" @@ -73,6 +81,7 @@ class FNSUploadIntegrationTest(APITestCase): content = _build_fns_excel_bytes() external_id = _digits(5) ogrn = _digits(13) + _ensure_directory_organization(ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload = SimpleUploadedFile( filename, @@ -119,6 +128,7 @@ class FNSUploadIntegrationTest(APITestCase): content = _build_fns_excel_bytes() external_id = _digits(3) ogrn = _digits(13) + _ensure_directory_organization(ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload1 = SimpleUploadedFile( filename, @@ -165,6 +175,7 @@ class FNSUploadIntegrationTest(APITestCase): content = _build_fns_excel_bytes() external_id = _digits(5) ogrn = _digits(13) + _ensure_directory_organization(ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload = SimpleUploadedFile( filename, @@ -230,6 +241,7 @@ class FNSUploadIntegrationTest(APITestCase): content = _build_fns_excel_bytes() external_id = _digits(5) ogrn = _digits(13) + _ensure_directory_organization(ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload = SimpleUploadedFile( filename, @@ -337,6 +349,7 @@ class FNSUploadIntegrationTest(APITestCase): content = _build_fns_excel_bytes() external_id = _digits(5) ogrn = _digits(13) + _ensure_directory_organization(ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload = SimpleUploadedFile( filename, @@ -370,6 +383,7 @@ class FNSUploadIntegrationTest(APITestCase): content = _build_fns_excel_bytes() external_id = _digits(5) ogrn = _digits(13) + _ensure_directory_organization(ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload = SimpleUploadedFile( filename, @@ -500,8 +514,12 @@ class FNSUploadIntegrationTest(APITestCase): ) def test_queue_zip_archive_processes_valid_files_and_skips_invalid(self): - first_name = f"fin_{_digits(5)}_{_digits(13)}.xlsx" - second_name = f"fin_{_digits(5)}_{_digits(13)}.xlsx" + first_ogrn = _digits(13) + second_ogrn = _digits(13) + _ensure_directory_organization(ogrn=first_ogrn) + _ensure_directory_organization(ogrn=second_ogrn) + first_name = f"fin_{_digits(5)}_{first_ogrn}.xlsx" + second_name = f"fin_{_digits(5)}_{second_ogrn}.xlsx" zip_content = _build_fns_zip_bytes( { first_name: _build_fns_excel_bytes(), diff --git a/tests/apps/parsers/test_procurement_service.py b/tests/apps/parsers/test_procurement_service.py index 23c4b6c..38c6678 100644 --- a/tests/apps/parsers/test_procurement_service.py +++ b/tests/apps/parsers/test_procurement_service.py @@ -7,9 +7,8 @@ Unit-тесты для ProcurementService. from apps.parsers.clients.zakupki.schemas import Procurement from apps.parsers.models import ParserLoadLog, ProcurementRecord from apps.parsers.services import ProcurementService -from apps.registers.models import Organization from django.test import TestCase -from organizations.models import OrganizationSourceRecord +from organizations.models import Organization, OrganizationSourceRecord from tests.apps.parsers.factories import ProcurementRecordFactory, fake @@ -37,11 +36,23 @@ def _other_law(law_type: str) -> str: def _create_registry_organization(*, inn: str, ogrn: str) -> Organization: return Organization.objects.create( - pn_name=fake.company(), - mn_ogrn=int(ogrn), - mn_inn=int(inn), - in_kpp=int(_digits(9)), - mn_okpo=_digits(8), + name=fake.company(), + ogrn=ogrn, + inn=inn, + kpp=_digits(9), + okpo=_digits(8), + ) + + +def _ensure_directory_organization_for_procurement( + procurement: Procurement, +) -> Organization: + return Organization.objects.create( + name=procurement.customer_name, + ogrn=procurement.customer_ogrn, + inn=procurement.customer_inn, + kpp=procurement.customer_kpp, + okpo=_digits(8), ) @@ -82,6 +93,7 @@ def _save_procurement_source_record( **overrides, ) -> Procurement: procurement = _build_procurement(**overrides) + _ensure_directory_organization_for_procurement(procurement) ProcurementService.save_procurements( [procurement], batch_id=batch_id, @@ -111,6 +123,7 @@ class ProcurementServiceSaveTestCase(TestCase): purchase_number=purchase_number, customer_inn=customer_inn, ) + _ensure_directory_organization_for_procurement(procurement) saved = ProcurementService.save_procurements( [procurement], @@ -134,6 +147,8 @@ class ProcurementServiceSaveTestCase(TestCase): def test_save_multiple_procurements(self): """Сохранение нескольких закупок.""" procurements = [_build_procurement() for _ in range(5)] + for procurement in procurements: + _ensure_directory_organization_for_procurement(procurement) saved = ProcurementService.save_procurements(procurements, batch_id=1) @@ -169,6 +184,7 @@ class ProcurementServiceSaveTestCase(TestCase): purchase_number=purchase_number, customer_inn=_digits(10), ) + _ensure_directory_organization_for_procurement(procurement) saved = ProcurementService.save_procurements([procurement], batch_id=2) @@ -183,6 +199,8 @@ class ProcurementServiceSaveTestCase(TestCase): def test_save_with_chunking(self): """Сохранение большого количества записей чанками.""" procurements = [_build_procurement() for _ in range(100)] + for procurement in procurements: + _ensure_directory_organization_for_procurement(procurement) saved = ProcurementService.save_procurements( procurements, batch_id=1, chunk_size=25 diff --git a/tests/apps/parsers/test_service_helpers.py b/tests/apps/parsers/test_service_helpers.py index aa97bbd..4002d22 100644 --- a/tests/apps/parsers/test_service_helpers.py +++ b/tests/apps/parsers/test_service_helpers.py @@ -22,9 +22,9 @@ from apps.parsers.services import ( ) from django.db import IntegrityError from django.test import TestCase +from organizations.models import Organization from tests.apps.parsers.factories import ParserLoadLogFactory -from tests.apps.registers.factories import OrganizationFactory class NormalizeHelpersTest(TestCase): @@ -67,34 +67,38 @@ class RegistryOrganizationResolverTest(TestCase): self.assertEqual(lookup.by_ogrn, {}) def test_resolve_organization_id_by_unique_inn_and_ogrn(self): - org_by_inn = OrganizationFactory( - mn_inn=7_701_001_001, mn_ogrn=10_277_001_000_001 + org_by_inn = Organization.objects.create( + name="By INN", + inn="7701001001", + ogrn="1027700100001", ) - org_by_ogrn = OrganizationFactory( - mn_inn=7_701_001_002, mn_ogrn=10_277_001_000_002 + org_by_ogrn = Organization.objects.create( + name="By OGRN", + inn="7701001002", + ogrn="1027700100002", ) lookup = RegistryOrganizationResolver.build_lookup( [ - (org_by_inn.mn_inn, None), - (None, org_by_ogrn.mn_ogrn), + (org_by_inn.inn, None), + (None, org_by_ogrn.ogrn), ] ) self.assertEqual( RegistryOrganizationResolver.resolve_organization_id( lookup=lookup, - inn=str(org_by_inn.mn_inn), + inn=org_by_inn.inn, ogrn=None, ), - org_by_inn.id, + str(org_by_inn.uid), ) self.assertEqual( RegistryOrganizationResolver.resolve_organization_id( lookup=lookup, inn=None, - ogrn=str(org_by_ogrn.mn_ogrn), + ogrn=org_by_ogrn.ogrn, ), - org_by_ogrn.id, + str(org_by_ogrn.uid), ) @@ -142,6 +146,17 @@ class ParserLoadLogServiceRetryTest(TestCase): class SmallParserServiceQueryTest(TestCase): def test_industrial_product_service_query_helpers(self): + Organization.objects.create( + name='ООО "Продукт 1"', + inn="7701001001", + ogrn="1027700100001", + ) + Organization.objects.create( + name='ООО "Продукт 2"', + inn="7701001001", + ogrn="1027700100002", + ) + IndustrialProductService.save_products( [ IndustrialProduct( @@ -186,6 +201,17 @@ class SmallParserServiceQueryTest(TestCase): ) def test_inspection_service_has_data_for_period(self): + Organization.objects.create( + name='ООО "Проверка 1"', + inn="7701002001", + ogrn="1027700200001", + ) + Organization.objects.create( + name='ООО "Проверка 2"', + inn="7701002002", + ogrn="1027700200002", + ) + InspectionService.save_inspections( [ Inspection( @@ -237,6 +263,19 @@ class SmallParserServiceQueryTest(TestCase): self.assertTrue(InspectionService.has_data_for_period(2026, 4, True)) def test_procurement_service_find_by_customer_name_with_batch(self): + Organization.objects.create( + name="АО Тестовый заказчик 1", + inn="7701003001", + kpp="770101001", + ogrn="1027700300001", + ) + Organization.objects.create( + name="АО Тестовый заказчик 2", + inn="7701003002", + kpp="770101002", + ogrn="1027700300002", + ) + ProcurementService.save_procurements( [ Procurement( @@ -293,6 +332,11 @@ class SmallParserServiceQueryTest(TestCase): class FNSReportServiceHelpersTest(TestCase): def test_exists_find_and_status_helpers(self): + Organization.objects.create( + name='ООО "ФНС"', + ogrn="1027700111111", + ) + report = FNSReportService.save_report( external_id="EXT-100", ogrn="1027700111111", diff --git a/tests/apps/parsers/test_services.py b/tests/apps/parsers/test_services.py index 73c2f79..73d41f1 100644 --- a/tests/apps/parsers/test_services.py +++ b/tests/apps/parsers/test_services.py @@ -1,7 +1,7 @@ """Tests for parsers services.""" -from datetime import timedelta import unittest +from datetime import timedelta from unittest.mock import patch from urllib.parse import urlparse @@ -35,9 +35,9 @@ from apps.parsers.services import ( ProxyService, ProxyToolsSyncService, ) -from apps.registers.models import Organization from django.test import TestCase, override_settings, tag from django.utils import timezone +from organizations.models import Organization from tests.utils import TestHTTPServer from tests.utils.fixtures import build_minpromtorg_certificates_excel, fake @@ -62,11 +62,11 @@ def _proxy_address() -> str: def _create_registry_organization(*, inn: str, ogrn: str) -> Organization: return Organization.objects.create( - pn_name=fake.company(), - mn_ogrn=int(ogrn), - mn_inn=int(inn), - in_kpp=int(_digits(9)), - mn_okpo=_digits(8), + name=fake.company(), + ogrn=ogrn, + inn=inn, + kpp=_digits(9), + okpo=_digits(8), ) diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index 92b8565..f1d00f2 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib from datetime import timedelta from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -17,6 +18,7 @@ from django.http import Http404 from django.test import SimpleTestCase, TestCase, override_settings from django.test.utils import CaptureQueriesContext from django.utils import timezone +from organizations.models import Organization from organizations.source_ingestion import ( OrganizationSourceIngestionService, SourceRecordInput, @@ -33,6 +35,21 @@ def _save_source_record( title: str = "", payload: dict | None = None, ) -> None: + payload = dict(payload or {}) + if inn: + if not Organization.objects.filter(inn=inn).exists(): + Organization.objects.create( + name=organization_name or title or external_id, + inn=inn, + ) + else: + rn_key = organization_name or title or external_id + rn = str(int(hashlib.sha256(rn_key.encode()).hexdigest()[:12], 16) % 10**12) + payload.setdefault("rn", rn) + Organization.objects.get_or_create( + rn=rn, + defaults={"name": organization_name or title or external_id}, + ) OrganizationSourceIngestionService.save_records( source=source, load_batch=1, @@ -42,7 +59,7 @@ def _save_source_record( title=title, organization_name=organization_name or title or external_id, inn=inn, - payload=payload or {}, + payload=payload, ) ], ) diff --git a/tests/apps/parsers/test_source_cards_views.py b/tests/apps/parsers/test_source_cards_views.py index 023f140..b2d2362 100644 --- a/tests/apps/parsers/test_source_cards_views.py +++ b/tests/apps/parsers/test_source_cards_views.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib from pathlib import Path from tempfile import TemporaryDirectory @@ -9,17 +10,16 @@ from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import ParserLoadLog from apps.parsers.source_cards import SourceCardService from django.test import override_settings -from django.urls import reverse +from django.urls import NoReverseMatch, reverse +from organizations.models import Organization from organizations.source_ingestion import ( OrganizationSourceIngestionService, SourceRecordInput, ) -from registers.models import Register from rest_framework import status from rest_framework.test import APITestCase from tests.apps.parsers.factories import ParserLoadLogFactory -from tests.apps.registers.factories import RegistryMembershipPeriodFactory from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake @@ -37,6 +37,27 @@ def _save_source_record( organization_name: str = "", title: str = "", ) -> None: + payload: dict = {} + if inn or ogrn: + query = {} + if inn: + query["inn"] = inn + if ogrn: + query["ogrn"] = ogrn + if not Organization.objects.filter(**query).exists(): + Organization.objects.create( + name=organization_name or title or external_id, + inn=inn, + ogrn=ogrn, + ) + else: + rn_key = organization_name or title or external_id + rn = str(int(hashlib.sha256(rn_key.encode()).hexdigest()[:12], 16) % 10**12) + payload["rn"] = rn + Organization.objects.get_or_create( + rn=rn, + defaults={"name": organization_name or title or external_id}, + ) OrganizationSourceIngestionService.save_records( source=source, load_batch=1, @@ -47,6 +68,7 @@ def _save_source_record( organization_name=organization_name or title or external_id, inn=inn, ogrn=ogrn, + payload=payload, ) ], ) @@ -109,38 +131,11 @@ class SourceCardsApiTestCase(APITestCase): self.assertEqual(inspections_card["status"], "in_progress") self.assertEqual(inspections_card["progress"], 63) - def test_main_dashboard_returns_cached_source_cards_and_registry_stats(self): - registry, _created = Register.objects.get_or_create( - name="Реестр предприятий ОПК", - ) - RegistryMembershipPeriodFactory( - registry=registry, - ) - ParserLoadLogFactory( - source=ParserLoadLog.Source.FNS_REPORTS, - status="success", - records_count=1, - ) - SourceCardService.clear_cache() + def test_main_dashboard_route_is_removed_with_registers_api(self): + with self.assertRaises(NoReverseMatch): + reverse("api_v1:stat:main-dashboard") - response = self.client.get(reverse("api_v1:stat:main-dashboard")) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertTrue(response.data["success"]) - self.assertEqual(response["X-Cache"], "MISS") - - data = response.data["data"] - self.assertIn("source_cards", data) - self.assertIn("organization_stats", data) - self.assertGreaterEqual(len(data["source_cards"]), 1) - self.assertEqual(data["organization_stats"]["counts"]["opk"], 1) - self.assertEqual(data["cache_ttl_seconds"], 604800) - - cached_response = self.client.get(reverse("api_v1:stat:main-dashboard")) - self.assertEqual(cached_response.status_code, status.HTTP_200_OK) - self.assertEqual(cached_response["X-Cache"], "HIT") - - def test_main_dashboard_cache_is_warmed_after_successful_parser_load(self): + def test_successful_parser_load_does_not_restore_removed_stat_route(self): with self.captureOnCommitCallbacks(execute=True): ParserLoadLogFactory( source=ParserLoadLog.Source.FNS_REPORTS, @@ -148,10 +143,8 @@ class SourceCardsApiTestCase(APITestCase): records_count=1, ) - response = self.client.get(reverse("api_v1:stat:main-dashboard")) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response["X-Cache"], "HIT") + with self.assertRaises(NoReverseMatch): + reverse("api_v1:stat:main-dashboard") def test_source_card_detail_returns_combined_minprom_stats(self): shared_inn = _digits(10) diff --git a/tests/apps/parsers/test_sources_api_e2e.py b/tests/apps/parsers/test_sources_api_e2e.py index 229d5ee..95a5393 100644 --- a/tests/apps/parsers/test_sources_api_e2e.py +++ b/tests/apps/parsers/test_sources_api_e2e.py @@ -6,6 +6,7 @@ from unittest.mock import patch from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import FinancialReport, FinancialReportLine, ParserLoadLog from django.urls import reverse +from organizations.models import Organization from organizations.source_backfill import OrganizationSourceBackfillService from rest_framework import status from rest_framework.test import APITestCase @@ -33,9 +34,20 @@ class SourcesApiE2ETest(APITestCase): def test_source_cards_and_statuses_flow(self): shared_inn = _digits(10) + shared_ogrn = _digits(13) + report_ogrn = _digits(13) + Organization.objects.create( + name="Shared Organization", + inn=shared_inn, + ogrn=shared_ogrn, + ) + Organization.objects.create( + name="Financial Organization", + ogrn=report_ogrn, + ) report = FinancialReport.objects.create( external_id=_digits(5), - ogrn=_digits(13), + ogrn=report_ogrn, file_name=f"fin_{_digits(5)}_{_digits(13)}.xlsx", file_hash=fake.sha256(raw_output=False), load_batch=1, @@ -51,11 +63,11 @@ class SourcesApiE2ETest(APITestCase): period_start=100, period_end=200, ) - IndustrialCertificateRecordFactory(inn=shared_inn) - IndustrialProductRecordFactory(inn=shared_inn) - ManufacturerRecordFactory(inn=shared_inn) + IndustrialCertificateRecordFactory(inn=shared_inn, ogrn=shared_ogrn) + IndustrialProductRecordFactory(inn=shared_inn, ogrn=shared_ogrn) + ManufacturerRecordFactory(inn=shared_inn, ogrn=shared_ogrn) InspectionRecordFactory() - ProcurementRecordFactory(customer_inn=shared_inn) + ProcurementRecordFactory(customer_inn=shared_inn, customer_ogrn=shared_ogrn) ParserLoadLogFactory( source=ParserLoadLog.Source.FNS_REPORTS, status="success", diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index dc499bb..17419cb 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -61,8 +61,7 @@ from apps.parsers.tasks import ( ) from django.test import TestCase, override_settings from openpyxl import Workbook -from organizations.models import OrganizationSourceRecord -from registers.models import Organization +from organizations.models import Organization, OrganizationSourceRecord from tests.apps.parsers.factories import ( InspectionRecordFactory, @@ -71,10 +70,6 @@ from tests.apps.parsers.factories import ( ProcurementRecordFactory, ProxyFactory, ) -from tests.apps.registers.factories import ( - OrganizationFactory, - RegistryMembershipPeriodFactory, -) from tests.utils import TestHTTPServer from tests.utils.fixtures import ( build_minpromtorg_certificates_excel, @@ -98,6 +93,76 @@ def _digits(length: int) -> str: return "".join(str(fake.random_int(0, 9)) for _ in range(length)) +def OrganizationFactory(**kwargs) -> Organization: + kwargs.setdefault("name", kwargs.get("pn_name") or fake.company()) + kwargs.setdefault("okpo", kwargs.get("mn_okpo") or _digits(8)) + return Organization.objects.create(**kwargs) + + +def RegistryMembershipPeriodFactory( + *, + organization: Organization, + ended_at=None, + **_kwargs, +): + if ended_at is None: + organization.opk_registry_membership = True + organization.ropk_num = organization.ropk_num or "test-registry" + organization.ropk_razdel_num = organization.ropk_razdel_num or "1" + organization.ropk_razdel_name = ( + organization.ropk_razdel_name or "Тестовый реестр" + ) + else: + organization.opk_registry_membership = False + organization.ropk_num = "" + organization.ropk_razdel_num = "" + organization.ropk_razdel_name = "" + organization.save( + update_fields=[ + "opk_registry_membership", + "ropk_num", + "ropk_razdel_num", + "ropk_razdel_name", + ] + ) + return SimpleNamespace(organization=organization, ended_at=ended_at) + + +def _ensure_directory_organization( + *, + inn: str = "", + ogrn: str = "", + kpp: str = "", + name: str = "", +) -> Organization: + return Organization.objects.create( + name=name or f"Справочник {inn or ogrn}", + inn=str(inn or ""), + ogrn=str(ogrn or ""), + kpp=str(kpp or ""), + okpo=_digits(8), + ) + + +def _ensure_directory_organizations_for_rows(rows) -> None: + for row in rows: + inn = getattr(row, "customer_inn", None) or getattr(row, "inn", "") + ogrn = getattr(row, "customer_ogrn", None) or getattr(row, "ogrn", "") + kpp = getattr(row, "customer_kpp", "") + name = ( + getattr(row, "customer_name", None) + or getattr(row, "organisation_name", None) + or getattr(row, "full_legal_name", None) + or getattr(row, "full_organisation_name", "") + ) + _ensure_directory_organization( + inn=inn, + ogrn=ogrn, + kpp=kpp, + name=name, + ) + + def _build_fns_excel_bytes() -> bytes: wb = Workbook() ws = wb.active @@ -278,7 +343,7 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(records[0].ogrn, "1237700253306") self.assertEqual( records[0].payload["identity_enrichment"]["provider"], - "local_registry", + "local_canonical", ) @override_settings(CHECKO_API_KEY="") @@ -832,7 +897,7 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(record.payload["law"], "44") self.assertEqual( record.payload["target"]["registry_organization_id"], - organization.id, + str(organization.id), ) self.assertEqual( [request.inn for request in _CheckoClient.instances[0].requests], @@ -935,6 +1000,7 @@ class ParseProcurementsTaskTestCase(TestCase): def test_parse_procurements_success(self): xml_content, rows = build_zakupki_xml(count=2) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("data.xml", xml_content)]) with TestHTTPServer() as server: @@ -989,6 +1055,7 @@ class ParseProcurementsTaskTestCase(TestCase): def test_parse_procurements_with_file_url(self): xml_content, rows = build_zakupki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("data.xml", xml_content)]) with TestHTTPServer() as server: @@ -1006,6 +1073,7 @@ class ParseProcurementsTaskTestCase(TestCase): def test_parse_procurements_default_host_with_proxies(self): xml_content, rows = build_zakupki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("data.xml", xml_content)]) region = f"{fake.random_int(min=1, max=99):02d}" year = fake.random_int(min=2020, max=2025) @@ -1030,6 +1098,7 @@ class ParseProcurementsTaskTestCase(TestCase): def test_parse_procurements_without_adapter(self): xml_content, rows = build_zakupki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("data.xml", xml_content)]) region = f"{fake.random_int(min=1, max=99):02d}" year = fake.random_int(min=2020, max=2025) @@ -1117,6 +1186,7 @@ class SyncProcurementsTaskTestCase(TestCase): ) xml_content, rows = build_zakupki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("data.xml", xml_content)]) with TestHTTPServer() as server: file_url = f"/opendata/download/notifications/{region}/{current_year}/{current_month:02d}/fz44.zip" @@ -1160,6 +1230,7 @@ class SyncProcurementsTaskTestCase(TestCase): current_year = 2025 current_month = 1 xml_content, rows = build_zakupki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("data.xml", xml_content)]) with TestHTTPServer() as server: @@ -1307,6 +1378,9 @@ class MinpromtorgTasksTestCase(TestCase): certificates_bytes, cert_rows = build_minpromtorg_certificates_excel(count=2) manufacturers_bytes, manuf_rows = build_minpromtorg_manufacturers_excel(count=2) products_bytes, product_rows = build_minpromtorg_products_excel(count=2) + _ensure_directory_organizations_for_rows(cert_rows) + _ensure_directory_organizations_for_rows(manuf_rows) + _ensure_directory_organizations_for_rows(product_rows) date_str = fake.date_between(start_date="-30d", end_date="today").strftime( "%Y%m%d" @@ -1731,6 +1805,7 @@ class ParseInspectionsTaskTestCase(TestCase): def test_parse_inspections_success(self): xml_content, rows = build_proverki_xml(count=2) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("inspections.xml", xml_content)]) with TestHTTPServer() as server: @@ -1757,6 +1832,7 @@ class ParseInspectionsTaskTestCase(TestCase): def test_parse_inspections_with_default_proxies(self): xml_content, rows = build_proverki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("inspections.xml", xml_content)]) with TestHTTPServer() as server: @@ -1776,6 +1852,7 @@ class ParseInspectionsTaskTestCase(TestCase): def test_parse_inspections_without_adapter(self): xml_content, rows = build_proverki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("inspections.xml", xml_content)]) with TestHTTPServer() as server: @@ -1817,6 +1894,7 @@ class ParseInspectionsTaskTestCase(TestCase): def test_sync_inspections_success(self): xml_content, rows = build_proverki_xml(count=2) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("inspections.xml", xml_content)]) with TestHTTPServer() as server: @@ -1864,6 +1942,7 @@ class ParseInspectionsTaskTestCase(TestCase): def test_sync_inspections_honors_limited_params(self): xml_content, rows = build_proverki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("inspections.xml", xml_content)]) with TestHTTPServer() as server: @@ -1904,6 +1983,7 @@ class ParseInspectionsTaskTestCase(TestCase): is_federal_law_248=True, ) xml_content, rows = build_proverki_xml(count=1) + _ensure_directory_organizations_for_rows(rows) archive = build_zip([("inspections.xml", xml_content)]) with TestHTTPServer() as server: @@ -2029,12 +2109,17 @@ class FNSFileTasksTestCase(TestCase): content = _build_fns_excel_bytes() external_id = _digits(5) ogrn = _digits(13) + Organization.objects.create(name=f"ФНС {ogrn}", ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" file_path = os.path.join(watch_dir, filename) with open(file_path, "wb") as handle: handle.write(content) return file_path + @staticmethod + def _ogrn_from_fns_file_path(file_path: str) -> str: + return os.path.basename(file_path).rsplit("_", 1)[1].removesuffix(".xlsx") + def _scan_with_eager_process(self) -> dict: original_delay = parser_tasks.process_fns_file.delay @@ -2158,7 +2243,7 @@ class FNSFileTasksTestCase(TestCase): FNSReportService.save_report( external_id=fake.pystr(min_chars=6, max_chars=10), - ogrn=_digits(13), + ogrn=self._ogrn_from_fns_file_path(file_path), file_name=os.path.basename(file_path), file_hash=file_hash, source=FinancialReport.SourceType.FILE_WATCH, @@ -2230,7 +2315,7 @@ class FNSFileTasksTestCase(TestCase): FNSReportService.save_report( external_id=fake.pystr(min_chars=6, max_chars=10), - ogrn=_digits(13), + ogrn=self._ogrn_from_fns_file_path(file_path), file_name=os.path.basename(file_path), file_hash=file_hash, source=FinancialReport.SourceType.FILE_WATCH, @@ -2253,6 +2338,33 @@ class FNSFileTasksTestCase(TestCase): os.path.exists(os.path.join(processed_dir, os.path.basename(file_path))) ) + def test_process_fns_file_sync_skips_ambiguous_organization(self): + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + os.makedirs(watch_dir, exist_ok=True) + file_path = self._write_fns_file(watch_dir) + ogrn = self._ogrn_from_fns_file_path(file_path) + Organization.objects.create(name=f"ФНС филиал {ogrn}", ogrn=ogrn) + + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ): + result = _process_fns_file_sync( + file_path, + task_id=str(fake.uuid4()), + ) + + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "ambiguous") + self.assertTrue( + os.path.exists(os.path.join(processed_dir, os.path.basename(file_path))) + ) + self.assertFalse( + os.path.exists(os.path.join(failed_dir, os.path.basename(file_path))) + ) + def test_process_fns_file_sync_handles_integrity_error(self): with tempfile.TemporaryDirectory() as tmpdir: watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) @@ -2671,6 +2783,8 @@ class ParseVacanciesTaskTestCase(TestCase): @override_settings(SUPERJOB_APP_ID="test-superjob-app-id") def test_parse_trudvsem_vacancies_uses_combined_vacancies_client(self): + for inn in ("7701000401", "7701000402", "7701000403"): + _ensure_directory_organization(inn=inn) captured_kwargs = {} captured_fetch_kwargs = {} @@ -2690,18 +2804,21 @@ class ParseVacanciesTaskTestCase(TestCase): GenericParserItem( source=ParserLoadLog.Source.TRUDVSEM, external_id="trudvsem:1", + inn="7701000401", title="Работа России", payload={"vacancy_source": "trudvsem"}, ), GenericParserItem( source=ParserLoadLog.Source.TRUDVSEM, external_id="hh:1", + inn="7701000402", title="HeadHunter", payload={"vacancy_source": "hh"}, ), GenericParserItem( source=ParserLoadLog.Source.TRUDVSEM, external_id="superjob:1", + inn="7701000403", title="SuperJob", payload={"vacancy_source": "superjob"}, ), diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 0723904..df67c31 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -3,6 +3,7 @@ from __future__ import annotations import csv +import hashlib import io import os import tempfile @@ -34,12 +35,6 @@ from tests.apps.parsers.factories import ( ManufacturerRecordFactory, ParserLoadLogFactory, ) -from tests.apps.registers.factories import ( - OrganizationFactory as RegisterOrganizationFactory, -) -from tests.apps.registers.factories import ( - RegistryMembershipPeriodFactory, -) from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake @@ -48,6 +43,35 @@ def _digits(length: int) -> str: return "".join(str(fake.random_int(0, 9)) for _ in range(length)) +def RegisterOrganizationFactory(**kwargs) -> Organization: + kwargs.setdefault("name", kwargs.get("pn_name") or fake.company()) + kwargs.setdefault("okpo", kwargs.get("mn_okpo") or _digits(8)) + return Organization.objects.create(**kwargs) + + +def RegistryMembershipPeriodFactory( + *, + organization: Organization, + **_kwargs, +): + if not organization.ropk_razdel_num: + organization.ropk_razdel_num = str(fake.random_int(min=1, max=9999)) + organization.opk_registry_membership = True + organization.ropk_num = organization.ropk_num or "test-registry" + organization.ropk_razdel_name = ( + organization.ropk_razdel_name or f"Реестр {organization.ropk_razdel_num}" + ) + organization.save( + update_fields=[ + "opk_registry_membership", + "ropk_num", + "ropk_razdel_num", + "ropk_razdel_name", + ] + ) + return Mock(organization=organization, registry_id=organization.ropk_razdel_num) + + def _build_fns_excel_bytes() -> bytes: wb = Workbook() ws = wb.active @@ -81,6 +105,27 @@ def _save_source_record( title: str = "Source record", payload: dict | None = None, ) -> None: + payload = dict(payload or {}) + if inn or ogrn: + query = {} + if inn: + query["inn"] = inn + if ogrn: + query["ogrn"] = ogrn + if not Organization.objects.filter(**query).exists(): + Organization.objects.create( + name=organization_name, + inn=inn, + ogrn=ogrn, + ) + else: + rn_key = organization_name or title or external_id + rn = str(int(hashlib.sha256(rn_key.encode()).hexdigest()[:12], 16) % 10**12) + payload.setdefault("rn", rn) + Organization.objects.get_or_create( + rn=rn, + defaults={"name": organization_name}, + ) OrganizationSourceIngestionService.save_records( source=source, load_batch=1, @@ -91,7 +136,7 @@ def _save_source_record( organization_name=organization_name, inn=inn, ogrn=ogrn, - payload=payload or {}, + payload=payload, ) ], ) @@ -1153,8 +1198,10 @@ class ParsersViewSetTest(APITestCase): processed_dir = os.path.join(tmpdir, "processed") failed_dir = os.path.join(tmpdir, "failed") content = _build_fns_excel_bytes() + ogrn = _digits(13) + Organization.objects.create(name=f"ФНС {ogrn}", ogrn=ogrn) upload = SimpleUploadedFile( - f"fin_{_digits(5)}_{_digits(13)}.xlsx", + f"fin_{_digits(5)}_{ogrn}.xlsx", content, content_type=( "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" @@ -1178,8 +1225,10 @@ class ParsersViewSetTest(APITestCase): processed_dir = os.path.join(tmpdir, "processed") failed_dir = os.path.join(tmpdir, "failed") content = _build_fns_excel_bytes() + ogrn = _digits(13) + Organization.objects.create(name=f"ФНС {ogrn}", ogrn=ogrn) upload = SimpleUploadedFile( - f"fin_{_digits(5)}_{_digits(13)}.xlsx", + f"fin_{_digits(5)}_{ogrn}.xlsx", content, content_type=( "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" @@ -1201,6 +1250,7 @@ class ParsersViewSetTest(APITestCase): self.client.force_authenticate(self.admin) external_id = _digits(5) ogrn = _digits(13) + Organization.objects.create(name=f"ФНС {ogrn}", ogrn=ogrn) filename = f"fin_{external_id}_{ogrn}.xlsx" upload = SimpleUploadedFile( "fns_reports.zip",