Files
mostovik-backend/tests/apps/organizations/test_source_ingestion.py
Aleksandr Meshchryakov f64bd5f664
Some checks failed
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Tests and lint (push) Has been cancelled
fix: scope export freshness and retries to each source group
2026-10-22 22:36:43 +02:00

354 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Tests for direct parser ingestion into organization source storage."""
from decimal import Decimal
from apps.parsers.models import FinancialReport, GenericParserRecord, ParserLoadLog
from django.test import TestCase
from django.utils import timezone
from organizations.models import (
DefenseSupplierExtension,
FinancialIndicatorsExtension,
Organization,
OrganizationSourceFinancialLine,
OrganizationSourceRecord,
SourceGroup,
)
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceFinancialLineInput,
SourceRecordInput,
)
class OrganizationSourceIngestionServiceTest(TestCase):
"""Checks runtime parser writes bypass legacy parser record tables."""
@staticmethod
def _directory_organization(**kwargs) -> Organization:
kwargs.setdefault("directory_imported_at", timezone.now())
return Organization.objects.create(**kwargs)
def test_save_generic_records_writes_source_records_without_legacy_rows(self):
from organizations.source_cache import get_source_data_cache_version
global_before = get_source_data_cache_version()
source_before = get_source_data_cache_version(SourceGroup.DEFENSE_SUPPLIERS)
other_before = get_source_data_cache_version(
SourceGroup.GOVERNMENT_PROCUREMENTS
)
self._directory_organization(
name='ООО "ГОЗ"',
inn="7707083803",
)
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
load_batch=42,
records=[
SourceRecordInput(
external_id="unfair-42",
title="Недобросовестный поставщик",
organization_name='ООО "ГОЗ"',
inn="7707083803",
ogrn="",
record_date="2026-05-18",
amount=Decimal("100.50"),
status="active",
url="https://example.test/unfair-42",
payload={"reason": "contract breach"},
)
],
)
self.assertEqual(result.scanned, 1)
self.assertNotEqual(global_before, get_source_data_cache_version())
self.assertNotEqual(
source_before, get_source_data_cache_version(SourceGroup.DEFENSE_SUPPLIERS)
)
self.assertEqual(
other_before,
get_source_data_cache_version(SourceGroup.GOVERNMENT_PROCUREMENTS),
)
self.assertEqual(result.created_records, 1)
self.assertEqual(result.updated_records, 0)
self.assertEqual(result.created_extensions, 1)
self.assertEqual(result.unresolved, 0)
self.assertEqual(result.created_organizations, 0)
self.assertEqual(GenericParserRecord.objects.count(), 0)
organization = Organization.objects.get(inn="7707083803")
extension = DefenseSupplierExtension.objects.get(organization=organization)
self.assertEqual(extension.source_group, SourceGroup.DEFENSE_SUPPLIERS)
self.assertEqual(extension.records_count, 1)
record = OrganizationSourceRecord.objects.get(extension=extension)
self.assertEqual(record.source, ParserLoadLog.Source.UNFAIR_SUPPLIERS)
self.assertEqual(record.record_type, "unfair_supplier")
self.assertEqual(record.external_id, "unfair-42")
self.assertEqual(record.amount, Decimal("100.50"))
self.assertEqual(record.payload["reason"], "contract breach")
self.assertEqual(record.legacy_model, "")
self.assertEqual(record.legacy_pk, "")
def test_save_records_is_idempotent_by_source_external_id(self):
self._directory_organization(
name='ООО "Идемпотентность"',
inn="7707083810",
)
first = SourceRecordInput(
external_id="unfair-idempotent",
title="Old title",
organization_name='ООО "Идемпотентность"',
inn="7707083810",
status="old",
payload={"version": 1},
)
second = SourceRecordInput(
external_id="unfair-idempotent",
title="New title",
organization_name='ООО "Идемпотентность"',
inn="7707083810",
status="new",
payload={"version": 2},
)
OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
load_batch=43,
records=[first],
)
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
load_batch=44,
records=[second],
)
self.assertEqual(result.created_records, 0)
self.assertEqual(result.updated_records, 1)
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
record = OrganizationSourceRecord.objects.get()
self.assertEqual(record.title, "New title")
self.assertEqual(record.status, "new")
self.assertEqual(record.payload["version"], 2)
self.assertEqual(record.load_batch, 44)
def test_save_records_skips_identified_record_without_directory_match(self):
name_only = Organization.objects.create(name="Acme Security")
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.FSTEC,
load_batch=45,
records=[
SourceRecordInput(
external_id="fstec-5066",
title="Weblock",
organization_name="Acme Security",
inn="7713497980",
ogrn="1237700253306",
payload={"identity_enrichment": {"status": "matched"}},
)
],
)
self.assertEqual(result.created_records, 0)
self.assertEqual(result.unresolved, 1)
self.assertEqual(result.skipped_unmatched, 1)
self.assertFalse(Organization.objects.filter(inn="7713497980").exists())
self.assertFalse(
OrganizationSourceRecord.objects.filter(external_id="fstec-5066").exists()
)
self.assertFalse(name_only.source_extensions.exists())
def test_save_records_does_not_fill_missing_identity_on_resolved_organization(self):
partial = self._directory_organization(
name="Acme Security",
inn="7713497980",
ogrn="1237700253306",
)
OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.FSTEC,
load_batch=46,
records=[
SourceRecordInput(
external_id="fstec-5067",
title="Weblock",
organization_name="Acme Security",
inn="7713497980",
ogrn="1237700253306",
)
],
)
partial.refresh_from_db()
self.assertEqual(partial.inn, "7713497980")
self.assertEqual(partial.ogrn, "1237700253306")
self.assertEqual(partial.name, "Acme Security")
record = OrganizationSourceRecord.objects.get(external_id="fstec-5067")
self.assertEqual(record.extension.organization, partial)
def test_save_records_uses_single_head_organization_for_duplicate_inn_ogrn(self):
head = self._directory_organization(
name="Head",
inn="7713497980",
ogrn="1237700253306",
filial=".F.",
is_branch=False,
)
self._directory_organization(
name="Branch",
inn="7713497980",
ogrn="1237700253306",
filial=".T.",
is_branch=True,
)
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.FSTEC,
load_batch=49,
records=[
SourceRecordInput(
external_id="fstec-head",
title="Weblock",
organization_name="Head",
inn="7713497980",
ogrn="1237700253306",
)
],
)
self.assertEqual(result.created_records, 1)
record = OrganizationSourceRecord.objects.get(external_id="fstec-head")
self.assertEqual(record.extension.organization, head)
def test_save_records_skips_ambiguous_duplicate_inn_ogrn(self):
self._directory_organization(
name="Head 1",
inn="7713497980",
ogrn="1237700253306",
filial=".F.",
is_branch=False,
)
self._directory_organization(
name="Head 2",
inn="7713497980",
ogrn="1237700253306",
filial=".F.",
is_branch=False,
)
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.FSTEC,
load_batch=50,
records=[
SourceRecordInput(
external_id="fstec-ambiguous",
title="Weblock",
organization_name="Head",
inn="7713497980",
ogrn="1237700253306",
)
],
)
self.assertEqual(result.created_records, 0)
self.assertEqual(result.skipped_ambiguous, 1)
self.assertFalse(
OrganizationSourceRecord.objects.filter(
external_id="fstec-ambiguous"
).exists()
)
def test_save_records_ignores_ogrip_like_ogrn_for_legal_entity_inn(self):
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.INDUSTRIAL,
load_batch=47,
records=[
SourceRecordInput(
external_id="cert-legal-entity-bad-ogrn",
title="cert-legal-entity-bad-ogrn",
organization_name='ООО "Металл-Завод"',
inn="7720525156",
ogrn="105774446645395",
)
],
)
self.assertEqual(result.created_records, 0)
self.assertEqual(result.unresolved, 1)
self.assertFalse(Organization.objects.filter(inn="7720525156").exists())
def test_save_records_does_not_add_ogrip_to_existing_legal_entity(self):
organization = self._directory_organization(
name='ООО "Металл-Завод"',
inn="7720525156",
ogrn="1057746645395",
)
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.INDUSTRIAL,
load_batch=48,
records=[
SourceRecordInput(
external_id="cert-legal-entity-existing-bad-ogrn",
title="cert-legal-entity-existing-bad-ogrn",
organization_name='ООО "Металл-Завод"',
inn="7720525156",
ogrn="105774446645395",
)
],
)
self.assertEqual(result.created_records, 1)
self.assertEqual(result.unresolved, 0)
organization.refresh_from_db()
self.assertEqual(organization.ogrn, "1057746645395")
self.assertEqual(organization.ogrip, "")
def test_save_financial_report_writes_financial_lines_without_legacy_report(self):
self._directory_organization(
name='ООО "Финансовый отчет"',
ogrn="1027700132002",
)
result = OrganizationSourceIngestionService.save_records(
source=ParserLoadLog.Source.FNS_REPORTS,
load_batch=88,
records=[
SourceRecordInput(
external_id="fns-report-1",
title="fin_001_1027700132002.xlsx",
organization_name="",
ogrn="1027700132002",
status="success",
payload={
"file_name": "fin_001_1027700132002.xlsx",
"file_hash": "a" * 64,
},
financial_lines=[
SourceFinancialLineInput(
form_code="1",
line_code="1100",
line_name="Нематериальные активы",
year=2025,
period_start=100,
period_end=200,
)
],
)
],
)
self.assertEqual(result.created_records, 1)
self.assertEqual(result.created_financial_lines, 1)
self.assertEqual(FinancialReport.objects.count(), 0)
organization = Organization.objects.get(ogrn="1027700132002")
extension = FinancialIndicatorsExtension.objects.get(organization=organization)
record = OrganizationSourceRecord.objects.get(extension=extension)
self.assertEqual(record.source, ParserLoadLog.Source.FNS_REPORTS)
line = OrganizationSourceFinancialLine.objects.get(source_record=record)
self.assertEqual(line.line_code, "1100")
self.assertEqual(line.period_end, 200)