avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(organizations): migrate source storage to polymorphic records

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-05-19 10:23:53 +02:00
parent 19a7d5a91c
commit 4ca2fa25d5
44 changed files with 7129 additions and 1551 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
tests/apps/parsers/test_direct_ingestion_services.py Normal file
View File

@@ -0,0 +1,278 @@
"""Tests for parser services writing directly to organization source storage."""
from decimal import Decimal
from apps.parsers.clients.common import GenericParserItem
from apps.parsers.clients.fns.schemas import ReportLine
from apps.parsers.clients.minpromtorg.schemas import (
IndustrialCertificate,
IndustrialProduct,
Manufacturer,
)
from apps.parsers.clients.proverki.schemas import Inspection
from apps.parsers.clients.zakupki.schemas import Procurement
from apps.parsers.models import (
FinancialReport,
GenericParserRecord,
IndustrialCertificateRecord,
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParserLoadLog,
ProcurementRecord,
)
from apps.parsers.services import (
FNSReportService,
GenericParserRecordService,
IndustrialCertificateService,
IndustrialProductService,
InspectionService,
ManufacturerService,
ProcurementService,
)
from django.test import TestCase
from organizations.models import (
OrganizationSourceFinancialLine,
OrganizationSourceRecord,
)
class DirectIngestionParserServicesTest(TestCase):
"""Parser save services should not write legacy parser record rows."""
def test_industrial_certificate_save_records_writes_organization_source_records(self):
saved = IndustrialCertificateService.save_certificates(
[
IndustrialCertificate(
issue_date="01.02.2026",
certificate_number="CERT-DIRECT-1",
expiry_date="2029-02-01",
certificate_file_url="https://example.test/cert.pdf",
organisation_name='ООО "Сертификат"',
inn="7707083801",
ogrn="1027700132001",
)
],
batch_id=47,
)
self.assertEqual(saved, 1)
self.assertEqual(IndustrialCertificateRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.INDUSTRIAL,
external_id="CERT-DIRECT-1",
)
self.assertEqual(record.record_type, "industrial_certificate")
self.assertEqual(record.payload["issue_date_normalized"], "2026-02-01")
self.assertEqual(record.payload["expiry_date_normalized"], "2029-02-01")
self.assertEqual(record.url, "https://example.test/cert.pdf")
def test_manufacturer_save_records_writes_organization_source_records(self):
saved = ManufacturerService.save_manufacturers(
[
Manufacturer(
full_legal_name='ООО "Производитель"',
inn="7707083802",
ogrn="1027700132002",
address="Москва",
)
],
batch_id=48,
)
self.assertEqual(saved, 1)
self.assertEqual(ManufacturerRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.MANUFACTURES,
external_id="7707083802",
)
self.assertEqual(record.record_type, "manufacturer")
self.assertEqual(record.title, 'ООО "Производитель"')
self.assertEqual(record.payload["address"], "Москва")
def test_industrial_product_save_records_writes_organization_source_records(self):
saved = IndustrialProductService.save_products(
[
IndustrialProduct(
full_organisation_name='ООО "Продукция"',
inn="7707083809",
ogrn="1027700132009",
registry_number="PROD-DIRECT-1",
product_name="Станок",
product_model="MODEL-1",
okpd2_code="28.41",
tnved_code="8457109000",
regulatory_document="ГОСТ",
)
],
batch_id=49,
)
self.assertEqual(saved, 1)
self.assertEqual(IndustrialProductRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS,
external_id="PROD-DIRECT-1",
)
self.assertEqual(record.record_type, "industrial_product")
self.assertEqual(record.title, "Станок")
self.assertEqual(record.payload["okpd2_code"], "28.41")
def test_procurement_save_records_writes_organization_source_records(self):
saved = ProcurementService.save_procurements(
[
Procurement(
purchase_number="PROC-DIRECT-1",
purchase_name="Поставка оборудования",
customer_inn="7707083810",
customer_kpp="770701001",
customer_ogrn="1027700132010",
customer_name='ООО "Заказчик"',
max_price="1 234 567,89",
currency_code="RUB",
placement_method="Аукцион",
publish_date="01.03.2026",
end_date="2026-03-15",
status="published",
law_type="44-FZ",
purchase_object_info="Оборудование",
href="https://example.test/procurement",
)
],
batch_id=50,
region_code="77",
data_year=2026,
data_month=3,
)
self.assertEqual(saved, 1)
self.assertEqual(ProcurementRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.PROCUREMENTS,
external_id="PROC-DIRECT-1",
)
self.assertEqual(record.record_type, "procurement")
self.assertEqual(record.amount, Decimal("1234567.89"))
self.assertEqual(record.payload["publish_date_normalized"], "2026-03-01")
self.assertEqual(record.payload["region_code"], "77")
self.assertEqual(record.payload["data_month"], 3)
def test_generic_save_records_writes_organization_source_records(self):
saved = GenericParserRecordService.save_records(
[
GenericParserItem(
source=ParserLoadLog.Source.FAS_GOZ,
external_id="fas-goz-1",
inn="7707083803",
ogrn="",
organisation_name='ООО "ГОЗ"',
title="Уклонение от ГОЗ",
record_date="2026-05-18",
amount=Decimal("12.30"),
status="active",
url="https://example.test/fas-goz-1",
payload={"registry": "fas"},
)
],
batch_id=51,
source=ParserLoadLog.Source.FAS_GOZ,
)
self.assertEqual(saved, 1)
self.assertEqual(GenericParserRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.FAS_GOZ,
external_id="fas-goz-1",
)
self.assertEqual(record.title, "Уклонение от ГОЗ")
self.assertEqual(record.payload["registry"], "fas")
self.assertEqual(record.load_batch, 51)
def test_inspection_save_records_writes_organization_source_records(self):
saved = InspectionService.save_inspections(
[
Inspection(
registration_number="INSP-DIRECT-1",
inn="7707083804",
ogrn="1027700132004",
organisation_name='ООО "Проверка"',
control_authority="Контроль",
inspection_type="Плановая",
inspection_form="Документарная",
start_date="01.03.2026",
end_date="2026-03-15",
status="planned",
legal_basis="ФЗ",
result="",
)
],
batch_id=52,
data_year=2026,
data_month=3,
)
self.assertEqual(saved, 1)
self.assertEqual(InspectionRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.INSPECTIONS,
external_id="INSP-DIRECT-1",
)
self.assertEqual(record.record_type, "inspection")
self.assertEqual(record.payload["control_authority"], "Контроль")
self.assertEqual(record.payload["start_date_normalized"], "2026-03-01")
self.assertEqual(record.payload["data_year"], 2026)
self.assertEqual(record.payload["data_month"], 3)
def test_fns_save_report_writes_source_record_and_financial_lines(self):
report = FNSReportService.save_report(
external_id="fns-direct-1",
ogrn="1027700132005",
file_name="fin_001_1027700132005.xlsx",
file_hash="b" * 64,
source="file_watch",
batch_id=53,
lines_data=[
{
"form_code": "1",
"line_code": "1600",
"line_name": "Баланс",
"year": 2025,
"period_start": 100,
"period_end": 200,
}
],
)
self.assertEqual(FinancialReport.objects.count(), 0)
self.assertEqual(str(report.external_id), "fns-direct-1")
source_record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.FNS_REPORTS,
external_id="fns-direct-1",
)
self.assertEqual(report.uid, source_record.uid)
self.assertEqual(source_record.payload["file_hash"], "b" * 64)
line = OrganizationSourceFinancialLine.objects.get(source_record=source_record)
self.assertEqual(line.line_code, "1600")
self.assertEqual(line.period_end, 200)
def test_fns_exists_by_hash_reads_source_record_payload(self):
FNSReportService.save_report(
external_id="fns-direct-2",
ogrn="1027700132006",
file_name="fin_002_1027700132006.xlsx",
file_hash="c" * 64,
source="file_watch",
batch_id=54,
lines_data=[
ReportLine(
form_code="2",
line_code="2110",
line_name="Выручка",
year=2025,
period_end=500,
).__dict__
],
)
self.assertTrue(FNSReportService.exists_by_hash("c" * 64))
self.assertTrue(FNSReportService.exists_by_external_id("fns-direct-2"))

64
tests/apps/parsers/test_source_cards_service.py
View File

@@ -5,7 +5,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.models import GenericParserRecord, ParserLoadLog
from apps.parsers.models import ParserLoadLog
from apps.parsers.source_cards import (
SOURCE_CARD_DEFINITIONS,
SourceCardDefinition,
@@ -15,9 +15,37 @@ from apps.parsers.source_cards import (
from django.http import Http404
from django.test import SimpleTestCase, TestCase, override_settings
from django.utils import timezone
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
from rest_framework.exceptions import ValidationError
def _save_source_record(
*,
source: str,
external_id: str,
inn: str = "",
organization_name: str = "",
title: str = "",
payload: dict | None = None,
) -> None:
OrganizationSourceIngestionService.save_records(
source=source,
load_batch=1,
records=[
SourceRecordInput(
external_id=external_id,
title=title,
organization_name=organization_name or title or external_id,
inn=inn,
payload=payload or {},
)
],
)
class SourceCardServiceUnitTest(SimpleTestCase):
def test_list_cards_exposes_all_frontend_category_slugs_in_menu_order(self):
self.assertEqual(
@@ -363,19 +391,19 @@ class SourceCardServiceUnitTest(SimpleTestCase):
@override_settings(PARSER_STALE_LOAD_MAX_AGE_MINUTES=90)
class SourceCardServiceDatabaseTest(TestCase):
def test_defense_unreliable_suppliers_counts_unique_generic_organizations(self):
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
load_batch=1,
external_id="unfair-1",
inn="7701234567",
organization_name='ООО "Поставщик"',
title="Недобросовестный поставщик",
payload={"number": "unfair-1"},
)
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.FAS_GOZ,
load_batch=1,
external_id="goz-1",
inn="7701234567",
organization_name='ООО "Поставщик"',
title="Уклонение от ГОЗ",
payload={"number": "goz-1"},
)
@@ -399,19 +427,19 @@ class SourceCardServiceDatabaseTest(TestCase):
self.assertEqual(card["organizations_count"], 1)
def test_public_procurements_counts_generic_eis_sources(self):
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
load_batch=1,
external_id="notice-1",
inn="7701234567",
organization_name="ГБУ Заказчик",
title="Закупка 44-ФЗ",
payload={"number": "notice-1"},
)
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.CONTRACTS,
load_batch=1,
external_id="contract-1",
inn="7701234567",
organization_name="ГБУ Заказчик",
title="Контракт ЕИС",
payload={"number": "contract-1"},
)
@@ -435,30 +463,24 @@ class SourceCardServiceDatabaseTest(TestCase):
self.assertEqual(card["organizations_count"], 1)
def test_public_procurements_counts_generic_buyers_without_inn(self):
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
load_batch=1,
external_id="notice-1",
inn="",
organisation_name="ГБУ Заказчик",
organization_name="ГБУ Заказчик",
title="Закупка 44-ФЗ",
payload={"Заказчик": "ГБУ Заказчик"},
)
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.CONTRACTS,
load_batch=1,
external_id="contract-1",
inn="",
organisation_name="ГБУ Заказчик",
organization_name="ГБУ Заказчик",
title="Контракт ЕИС",
payload={"Заказчик": "ГБУ Заказчик"},
)
GenericParserRecord.objects.create(
_save_source_record(
source=ParserLoadLog.Source.PROCUREMENTS_223FZ,
load_batch=1,
external_id="notice-2",
inn="",
organisation_name="АО Другой заказчик",
organization_name="АО Другой заказчик",
title="Закупка 223-ФЗ",
payload={"Наименование заказчика": "АО Другой заказчик"},
)

67
tests/apps/parsers/test_tasks.py
View File

@@ -170,6 +170,27 @@ class ProxyResolutionTestCase(TestCase):
self.assertIsNone(result)
class OrganizationSourceBackfillQueueTestCase(TestCase):
"""Tests parser tasks queue organization source backfill after DB commit."""
def test_queue_organization_source_backfill_runs_after_commit(self):
with (
patch(
"organizations.tasks.backfill_organization_sources_for_parser_batch.delay",
) as delay_mock,
self.captureOnCommitCallbacks(execute=True),
):
parser_tasks._queue_organization_source_backfill(
ParserLoadLog.Source.INDUSTRIAL,
7,
)
delay_mock.assert_called_once_with(
source=ParserLoadLog.Source.INDUSTRIAL,
batch_id=7,
)
class SyncRuProxiesTaskTestCase(TestCase):
"""Tests for periodic RU proxy sync task."""
@@ -299,6 +320,52 @@ class GenericSourceFetchTestCase(TestCase):
self.assertEqual(captured_inns, [str(organization.mn_inn)])
self.assertNotIn(str(no_membership.mn_inn), captured_inns)
def test_checko_bankruptcy_items_group_messages_by_procedure(self):
company = SimpleNamespace(
ogrn="1052452047450",
inn="2452031093",
short_name='АО "СИБПРОМПРОЕКТ"',
bankruptcy=(
SimpleNamespace(
type="Сообщение ЕФРСБ",
date="2026-04-22",
case_number="",
),
SimpleNamespace(
type="Сообщение ЕФРСБ",
date="2026-04-20",
case_number="",
),
),
)
records = parser_tasks._checko_bankruptcy_items(
company=company,
fallback_inn="2452031093",
fallback_ogrn="1052452047450",
fallback_name='АО "СИБПРОМПРОЕКТ"',
)
self.assertEqual(len(records), 1)
self.assertEqual(records[0].external_id, "checko-fedresurs:2452031093")
self.assertEqual(records[0].record_date, "2026-04-22")
self.assertEqual(records[0].payload["messages_count"], 2)
self.assertEqual(
records[0].payload["messages"],
[
{
"case_number": "",
"date": "2026-04-22",
"type": "Сообщение ЕФРСБ",
},
{
"case_number": "",
"date": "2026-04-20",
"type": "Сообщение ЕФРСБ",
},
],
)
@override_settings(CHECKO_API_KEY="test-key", ARBITRATION_CHECKO_LIMIT=10)
def test_arbitration_fetches_checko_legal_cases_for_active_registry_organizations(
self,

93
tests/apps/parsers/test_views.py
View File

@@ -20,6 +20,10 @@ from django.core.files.uploadedfile import SimpleUploadedFile
from django.urls import reverse
from openpyxl import Workbook
from organizations.models import Organization
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
from rest_framework import status
from rest_framework.test import APITestCase
@@ -67,6 +71,32 @@ def _build_fns_zip_bytes(file_map: dict[str, bytes]) -> bytes:
return buf.getvalue()
def _save_source_record(
*,
source: str,
external_id: str,
inn: str = "",
ogrn: str = "",
organization_name: str = "Test organization",
title: str = "Source record",
payload: dict | None = None,
) -> None:
OrganizationSourceIngestionService.save_records(
source=source,
load_batch=1,
records=[
SourceRecordInput(
external_id=external_id,
title=title,
organization_name=organization_name,
inn=inn,
ogrn=ogrn,
payload=payload or {},
)
],
)
def _create_procurement_record() -> ProcurementRecord:
return ProcurementRecord.objects.create(
load_batch=fake.random_int(min=1, max=1000),
@@ -442,35 +472,33 @@ class ParsersViewSetTest(APITestCase):
self.assertNotIn("kpp", detail)
def test_dashboard_data_exposes_source_groups_for_page(self):
GenericParserRecord.objects.create(
load_batch=1,
_save_source_record(
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
external_id="eis-44fz-1",
organization_name="Customer 1",
title="EIS 44-FZ notice 1",
payload={"registry": "44fz"},
)
GenericParserRecord.objects.create(
load_batch=1,
_save_source_record(
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
external_id="eis-44fz-2",
organization_name="Customer 2",
title="EIS 44-FZ notice 2",
payload={"registry": "44fz"},
)
GenericParserRecord.objects.create(
load_batch=1,
_save_source_record(
source=ParserLoadLog.Source.TRUDVSEM,
external_id="trudvsem-1",
organization_name="Employer",
title="Vacancy",
payload={"registry": "trudvsem"},
)
FinancialReport.objects.create(
_save_source_record(
source=ParserLoadLog.Source.FNS_REPORTS,
external_id=_digits(5),
ogrn=_digits(13),
file_name=f"fin_{_digits(5)}_{_digits(13)}.xlsx",
file_hash=fake.sha256(raw_output=False),
load_batch=1,
status=FinancialReport.Status.SUCCESS,
source=FinancialReport.SourceType.API,
organization_name="FNS organization",
title="FNS report",
)
self.client.force_authenticate(self.user)
@@ -513,17 +541,14 @@ class ParsersViewSetTest(APITestCase):
mn_ogrn=1107746880031,
)
RegistryMembershipPeriodFactory(organization=registry_organization)
FinancialReport.objects.create(
_save_source_record(
source=ParserLoadLog.Source.FNS_REPORTS,
external_id=_digits(5),
ogrn=str(registry_organization.mn_ogrn),
file_name=f"fin_{_digits(5)}_{registry_organization.mn_ogrn}.xlsx",
file_hash=fake.sha256(raw_output=False),
load_batch=1,
status=FinancialReport.Status.SUCCESS,
source=FinancialReport.SourceType.API,
organization_name="Registry FNS organization",
title="FNS report",
)
GenericParserRecord.objects.create(
load_batch=1,
_save_source_record(
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
external_id="unfair-1",
title="Unfair supplier record",
@@ -561,36 +586,40 @@ class ParsersViewSetTest(APITestCase):
roscosmos_membership = RegistryMembershipPeriodFactory(
organization=roscosmos_organization
)
FinancialReport.objects.create(
_save_source_record(
source=ParserLoadLog.Source.FNS_REPORTS,
external_id=_digits(5),
ogrn=str(rosatom_organization.mn_ogrn),
file_name=f"fin_{_digits(5)}_{rosatom_organization.mn_ogrn}.xlsx",
file_hash=fake.sha256(raw_output=False),
load_batch=1,
status=FinancialReport.Status.SUCCESS,
source=FinancialReport.SourceType.API,
organization_name="Rosatom",
title="FNS report",
)
IndustrialCertificateRecordFactory(
_save_source_record(
source=ParserLoadLog.Source.INDUSTRIAL,
external_id="industrial-registry-analytics",
title="Industrial certificate",
organization_name="Rosatom",
inn=str(rosatom_organization.mn_inn),
ogrn=str(rosatom_organization.mn_ogrn),
)
GenericParserRecord.objects.create(
load_batch=1,
_save_source_record(
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
external_id="unfair-registry-analytics",
title="Risk signal",
inn=str(roscosmos_organization.mn_inn),
ogrn=str(roscosmos_organization.mn_ogrn),
)
GenericParserRecord.objects.create(
load_batch=1,
_save_source_record(
source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY,
external_id="bankruptcy-registry-analytics",
title="Bankruptcy risk signal",
inn=str(roscosmos_organization.mn_inn),
ogrn=str(roscosmos_organization.mn_ogrn),
)
InspectionRecordFactory(
_save_source_record(
source=ParserLoadLog.Source.INSPECTIONS,
external_id="inspection-registry-analytics",
title="Inspection risk signal",
organization_name="Rosatom",
inn=str(rosatom_organization.mn_inn),
ogrn=str(rosatom_organization.mn_ogrn),
)