339 lines
13 KiB
Python
339 lines
13 KiB
Python
"""Tests for direct parser ingestion into organization source storage."""
|
||
|
||
from decimal import Decimal
|
||
|
||
from apps.parsers.models import FinancialReport, GenericParserRecord, ParserLoadLog
|
||
from django.test import TestCase
|
||
from django.utils import timezone
|
||
from organizations.models import (
|
||
DefenseSupplierExtension,
|
||
FinancialIndicatorsExtension,
|
||
Organization,
|
||
OrganizationSourceFinancialLine,
|
||
OrganizationSourceRecord,
|
||
SourceGroup,
|
||
)
|
||
from organizations.source_ingestion import (
|
||
OrganizationSourceIngestionService,
|
||
SourceFinancialLineInput,
|
||
SourceRecordInput,
|
||
)
|
||
|
||
|
||
class OrganizationSourceIngestionServiceTest(TestCase):
|
||
"""Checks runtime parser writes bypass legacy parser record tables."""
|
||
|
||
@staticmethod
|
||
def _directory_organization(**kwargs) -> Organization:
|
||
kwargs.setdefault("directory_imported_at", timezone.now())
|
||
return Organization.objects.create(**kwargs)
|
||
|
||
def test_save_generic_records_writes_source_records_without_legacy_rows(self):
|
||
self._directory_organization(
|
||
name='ООО "ГОЗ"',
|
||
inn="7707083803",
|
||
)
|
||
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||
load_batch=42,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="unfair-42",
|
||
title="Недобросовестный поставщик",
|
||
organization_name='ООО "ГОЗ"',
|
||
inn="7707083803",
|
||
ogrn="",
|
||
record_date="2026-05-18",
|
||
amount=Decimal("100.50"),
|
||
status="active",
|
||
url="https://example.test/unfair-42",
|
||
payload={"reason": "contract breach"},
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.scanned, 1)
|
||
self.assertEqual(result.created_records, 1)
|
||
self.assertEqual(result.updated_records, 0)
|
||
self.assertEqual(result.created_extensions, 1)
|
||
self.assertEqual(result.unresolved, 0)
|
||
self.assertEqual(result.created_organizations, 0)
|
||
self.assertEqual(GenericParserRecord.objects.count(), 0)
|
||
|
||
organization = Organization.objects.get(inn="7707083803")
|
||
extension = DefenseSupplierExtension.objects.get(organization=organization)
|
||
self.assertEqual(extension.source_group, SourceGroup.DEFENSE_SUPPLIERS)
|
||
self.assertEqual(extension.records_count, 1)
|
||
|
||
record = OrganizationSourceRecord.objects.get(extension=extension)
|
||
self.assertEqual(record.source, ParserLoadLog.Source.UNFAIR_SUPPLIERS)
|
||
self.assertEqual(record.record_type, "unfair_supplier")
|
||
self.assertEqual(record.external_id, "unfair-42")
|
||
self.assertEqual(record.amount, Decimal("100.50"))
|
||
self.assertEqual(record.payload["reason"], "contract breach")
|
||
self.assertEqual(record.legacy_model, "")
|
||
self.assertEqual(record.legacy_pk, "")
|
||
|
||
def test_save_records_is_idempotent_by_source_external_id(self):
|
||
self._directory_organization(
|
||
name='ООО "Идемпотентность"',
|
||
inn="7707083810",
|
||
)
|
||
|
||
first = SourceRecordInput(
|
||
external_id="unfair-idempotent",
|
||
title="Old title",
|
||
organization_name='ООО "Идемпотентность"',
|
||
inn="7707083810",
|
||
status="old",
|
||
payload={"version": 1},
|
||
)
|
||
second = SourceRecordInput(
|
||
external_id="unfair-idempotent",
|
||
title="New title",
|
||
organization_name='ООО "Идемпотентность"',
|
||
inn="7707083810",
|
||
status="new",
|
||
payload={"version": 2},
|
||
)
|
||
|
||
OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||
load_batch=43,
|
||
records=[first],
|
||
)
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||
load_batch=44,
|
||
records=[second],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 0)
|
||
self.assertEqual(result.updated_records, 1)
|
||
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
|
||
record = OrganizationSourceRecord.objects.get()
|
||
self.assertEqual(record.title, "New title")
|
||
self.assertEqual(record.status, "new")
|
||
self.assertEqual(record.payload["version"], 2)
|
||
self.assertEqual(record.load_batch, 44)
|
||
|
||
def test_save_records_skips_identified_record_without_directory_match(self):
|
||
name_only = Organization.objects.create(name="Acme Security")
|
||
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.FSTEC,
|
||
load_batch=45,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="fstec-5066",
|
||
title="Weblock",
|
||
organization_name="Acme Security",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
payload={"identity_enrichment": {"status": "matched"}},
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 0)
|
||
self.assertEqual(result.unresolved, 1)
|
||
self.assertEqual(result.skipped_unmatched, 1)
|
||
self.assertFalse(Organization.objects.filter(inn="7713497980").exists())
|
||
self.assertFalse(
|
||
OrganizationSourceRecord.objects.filter(external_id="fstec-5066").exists()
|
||
)
|
||
self.assertFalse(name_only.source_extensions.exists())
|
||
|
||
def test_save_records_does_not_fill_missing_identity_on_resolved_organization(self):
|
||
partial = self._directory_organization(
|
||
name="Acme Security",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
)
|
||
|
||
OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.FSTEC,
|
||
load_batch=46,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="fstec-5067",
|
||
title="Weblock",
|
||
organization_name="Acme Security",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
)
|
||
],
|
||
)
|
||
|
||
partial.refresh_from_db()
|
||
self.assertEqual(partial.inn, "7713497980")
|
||
self.assertEqual(partial.ogrn, "1237700253306")
|
||
self.assertEqual(partial.name, "Acme Security")
|
||
record = OrganizationSourceRecord.objects.get(external_id="fstec-5067")
|
||
self.assertEqual(record.extension.organization, partial)
|
||
|
||
def test_save_records_uses_single_head_organization_for_duplicate_inn_ogrn(self):
|
||
head = self._directory_organization(
|
||
name="Head",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
filial=".F.",
|
||
is_branch=False,
|
||
)
|
||
self._directory_organization(
|
||
name="Branch",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
filial=".T.",
|
||
is_branch=True,
|
||
)
|
||
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.FSTEC,
|
||
load_batch=49,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="fstec-head",
|
||
title="Weblock",
|
||
organization_name="Head",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 1)
|
||
record = OrganizationSourceRecord.objects.get(external_id="fstec-head")
|
||
self.assertEqual(record.extension.organization, head)
|
||
|
||
def test_save_records_skips_ambiguous_duplicate_inn_ogrn(self):
|
||
self._directory_organization(
|
||
name="Head 1",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
filial=".F.",
|
||
is_branch=False,
|
||
)
|
||
self._directory_organization(
|
||
name="Head 2",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
filial=".F.",
|
||
is_branch=False,
|
||
)
|
||
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.FSTEC,
|
||
load_batch=50,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="fstec-ambiguous",
|
||
title="Weblock",
|
||
organization_name="Head",
|
||
inn="7713497980",
|
||
ogrn="1237700253306",
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 0)
|
||
self.assertEqual(result.skipped_ambiguous, 1)
|
||
self.assertFalse(
|
||
OrganizationSourceRecord.objects.filter(
|
||
external_id="fstec-ambiguous"
|
||
).exists()
|
||
)
|
||
|
||
def test_save_records_ignores_ogrip_like_ogrn_for_legal_entity_inn(self):
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.INDUSTRIAL,
|
||
load_batch=47,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="cert-legal-entity-bad-ogrn",
|
||
title="cert-legal-entity-bad-ogrn",
|
||
organization_name='ООО "Металл-Завод"',
|
||
inn="7720525156",
|
||
ogrn="105774446645395",
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 0)
|
||
self.assertEqual(result.unresolved, 1)
|
||
self.assertFalse(Organization.objects.filter(inn="7720525156").exists())
|
||
|
||
def test_save_records_does_not_add_ogrip_to_existing_legal_entity(self):
|
||
organization = self._directory_organization(
|
||
name='ООО "Металл-Завод"',
|
||
inn="7720525156",
|
||
ogrn="1057746645395",
|
||
)
|
||
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.INDUSTRIAL,
|
||
load_batch=48,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="cert-legal-entity-existing-bad-ogrn",
|
||
title="cert-legal-entity-existing-bad-ogrn",
|
||
organization_name='ООО "Металл-Завод"',
|
||
inn="7720525156",
|
||
ogrn="105774446645395",
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 1)
|
||
self.assertEqual(result.unresolved, 0)
|
||
organization.refresh_from_db()
|
||
self.assertEqual(organization.ogrn, "1057746645395")
|
||
self.assertEqual(organization.ogrip, "")
|
||
|
||
def test_save_financial_report_writes_financial_lines_without_legacy_report(self):
|
||
self._directory_organization(
|
||
name='ООО "Финансовый отчет"',
|
||
ogrn="1027700132002",
|
||
)
|
||
|
||
result = OrganizationSourceIngestionService.save_records(
|
||
source=ParserLoadLog.Source.FNS_REPORTS,
|
||
load_batch=88,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id="fns-report-1",
|
||
title="fin_001_1027700132002.xlsx",
|
||
organization_name="",
|
||
ogrn="1027700132002",
|
||
status="success",
|
||
payload={
|
||
"file_name": "fin_001_1027700132002.xlsx",
|
||
"file_hash": "a" * 64,
|
||
},
|
||
financial_lines=[
|
||
SourceFinancialLineInput(
|
||
form_code="1",
|
||
line_code="1100",
|
||
line_name="Нематериальные активы",
|
||
year=2025,
|
||
period_start=100,
|
||
period_end=200,
|
||
)
|
||
],
|
||
)
|
||
],
|
||
)
|
||
|
||
self.assertEqual(result.created_records, 1)
|
||
self.assertEqual(result.created_financial_lines, 1)
|
||
self.assertEqual(FinancialReport.objects.count(), 0)
|
||
|
||
organization = Organization.objects.get(ogrn="1027700132002")
|
||
extension = FinancialIndicatorsExtension.objects.get(organization=organization)
|
||
record = OrganizationSourceRecord.objects.get(extension=extension)
|
||
self.assertEqual(record.source, ParserLoadLog.Source.FNS_REPORTS)
|
||
line = OrganizationSourceFinancialLine.objects.get(source_record=record)
|
||
self.assertEqual(line.line_code, "1100")
|
||
self.assertEqual(line.period_end, 200)
|