avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(organizations): migrate source storage to polymorphic records

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-05-19 10:23:53 +02:00
parent 19a7d5a91c
commit 4ca2fa25d5
44 changed files with 7129 additions and 1551 deletions
Download Patch File Download Diff File Expand all files Collapse all files

278
tests/apps/parsers/test_direct_ingestion_services.py Normal file
View File

@@ -0,0 +1,278 @@
"""Tests for parser services writing directly to organization source storage."""
from decimal import Decimal
from apps.parsers.clients.common import GenericParserItem
from apps.parsers.clients.fns.schemas import ReportLine
from apps.parsers.clients.minpromtorg.schemas import (
IndustrialCertificate,
IndustrialProduct,
Manufacturer,
)
from apps.parsers.clients.proverki.schemas import Inspection
from apps.parsers.clients.zakupki.schemas import Procurement
from apps.parsers.models import (
FinancialReport,
GenericParserRecord,
IndustrialCertificateRecord,
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParserLoadLog,
ProcurementRecord,
)
from apps.parsers.services import (
FNSReportService,
GenericParserRecordService,
IndustrialCertificateService,
IndustrialProductService,
InspectionService,
ManufacturerService,
ProcurementService,
)
from django.test import TestCase
from organizations.models import (
OrganizationSourceFinancialLine,
OrganizationSourceRecord,
)
class DirectIngestionParserServicesTest(TestCase):
"""Parser save services should not write legacy parser record rows."""
def test_industrial_certificate_save_records_writes_organization_source_records(self):
saved = IndustrialCertificateService.save_certificates(
[
IndustrialCertificate(
issue_date="01.02.2026",
certificate_number="CERT-DIRECT-1",
expiry_date="2029-02-01",
certificate_file_url="https://example.test/cert.pdf",
organisation_name='ООО "Сертификат"',
inn="7707083801",
ogrn="1027700132001",
)
],
batch_id=47,
)
self.assertEqual(saved, 1)
self.assertEqual(IndustrialCertificateRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.INDUSTRIAL,
external_id="CERT-DIRECT-1",
)
self.assertEqual(record.record_type, "industrial_certificate")
self.assertEqual(record.payload["issue_date_normalized"], "2026-02-01")
self.assertEqual(record.payload["expiry_date_normalized"], "2029-02-01")
self.assertEqual(record.url, "https://example.test/cert.pdf")
def test_manufacturer_save_records_writes_organization_source_records(self):
saved = ManufacturerService.save_manufacturers(
[
Manufacturer(
full_legal_name='ООО "Производитель"',
inn="7707083802",
ogrn="1027700132002",
address="Москва",
)
],
batch_id=48,
)
self.assertEqual(saved, 1)
self.assertEqual(ManufacturerRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.MANUFACTURES,
external_id="7707083802",
)
self.assertEqual(record.record_type, "manufacturer")
self.assertEqual(record.title, 'ООО "Производитель"')
self.assertEqual(record.payload["address"], "Москва")
def test_industrial_product_save_records_writes_organization_source_records(self):
saved = IndustrialProductService.save_products(
[
IndustrialProduct(
full_organisation_name='ООО "Продукция"',
inn="7707083809",
ogrn="1027700132009",
registry_number="PROD-DIRECT-1",
product_name="Станок",
product_model="MODEL-1",
okpd2_code="28.41",
tnved_code="8457109000",
regulatory_document="ГОСТ",
)
],
batch_id=49,
)
self.assertEqual(saved, 1)
self.assertEqual(IndustrialProductRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS,
external_id="PROD-DIRECT-1",
)
self.assertEqual(record.record_type, "industrial_product")
self.assertEqual(record.title, "Станок")
self.assertEqual(record.payload["okpd2_code"], "28.41")
def test_procurement_save_records_writes_organization_source_records(self):
saved = ProcurementService.save_procurements(
[
Procurement(
purchase_number="PROC-DIRECT-1",
purchase_name="Поставка оборудования",
customer_inn="7707083810",
customer_kpp="770701001",
customer_ogrn="1027700132010",
customer_name='ООО "Заказчик"',
max_price="1 234 567,89",
currency_code="RUB",
placement_method="Аукцион",
publish_date="01.03.2026",
end_date="2026-03-15",
status="published",
law_type="44-FZ",
purchase_object_info="Оборудование",
href="https://example.test/procurement",
)
],
batch_id=50,
region_code="77",
data_year=2026,
data_month=3,
)
self.assertEqual(saved, 1)
self.assertEqual(ProcurementRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.PROCUREMENTS,
external_id="PROC-DIRECT-1",
)
self.assertEqual(record.record_type, "procurement")
self.assertEqual(record.amount, Decimal("1234567.89"))
self.assertEqual(record.payload["publish_date_normalized"], "2026-03-01")
self.assertEqual(record.payload["region_code"], "77")
self.assertEqual(record.payload["data_month"], 3)
def test_generic_save_records_writes_organization_source_records(self):
saved = GenericParserRecordService.save_records(
[
GenericParserItem(
source=ParserLoadLog.Source.FAS_GOZ,
external_id="fas-goz-1",
inn="7707083803",
ogrn="",
organisation_name='ООО "ГОЗ"',
title="Уклонение от ГОЗ",
record_date="2026-05-18",
amount=Decimal("12.30"),
status="active",
url="https://example.test/fas-goz-1",
payload={"registry": "fas"},
)
],
batch_id=51,
source=ParserLoadLog.Source.FAS_GOZ,
)
self.assertEqual(saved, 1)
self.assertEqual(GenericParserRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.FAS_GOZ,
external_id="fas-goz-1",
)
self.assertEqual(record.title, "Уклонение от ГОЗ")
self.assertEqual(record.payload["registry"], "fas")
self.assertEqual(record.load_batch, 51)
def test_inspection_save_records_writes_organization_source_records(self):
saved = InspectionService.save_inspections(
[
Inspection(
registration_number="INSP-DIRECT-1",
inn="7707083804",
ogrn="1027700132004",
organisation_name='ООО "Проверка"',
control_authority="Контроль",
inspection_type="Плановая",
inspection_form="Документарная",
start_date="01.03.2026",
end_date="2026-03-15",
status="planned",
legal_basis="ФЗ",
result="",
)
],
batch_id=52,
data_year=2026,
data_month=3,
)
self.assertEqual(saved, 1)
self.assertEqual(InspectionRecord.objects.count(), 0)
record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.INSPECTIONS,
external_id="INSP-DIRECT-1",
)
self.assertEqual(record.record_type, "inspection")
self.assertEqual(record.payload["control_authority"], "Контроль")
self.assertEqual(record.payload["start_date_normalized"], "2026-03-01")
self.assertEqual(record.payload["data_year"], 2026)
self.assertEqual(record.payload["data_month"], 3)
def test_fns_save_report_writes_source_record_and_financial_lines(self):
report = FNSReportService.save_report(
external_id="fns-direct-1",
ogrn="1027700132005",
file_name="fin_001_1027700132005.xlsx",
file_hash="b" * 64,
source="file_watch",
batch_id=53,
lines_data=[
{
"form_code": "1",
"line_code": "1600",
"line_name": "Баланс",
"year": 2025,
"period_start": 100,
"period_end": 200,
}
],
)
self.assertEqual(FinancialReport.objects.count(), 0)
self.assertEqual(str(report.external_id), "fns-direct-1")
source_record = OrganizationSourceRecord.objects.get(
source=ParserLoadLog.Source.FNS_REPORTS,
external_id="fns-direct-1",
)
self.assertEqual(report.uid, source_record.uid)
self.assertEqual(source_record.payload["file_hash"], "b" * 64)
line = OrganizationSourceFinancialLine.objects.get(source_record=source_record)
self.assertEqual(line.line_code, "1600")
self.assertEqual(line.period_end, 200)
def test_fns_exists_by_hash_reads_source_record_payload(self):
FNSReportService.save_report(
external_id="fns-direct-2",
ogrn="1027700132006",
file_name="fin_002_1027700132006.xlsx",
file_hash="c" * 64,
source="file_watch",
batch_id=54,
lines_data=[
ReportLine(
form_code="2",
line_code="2110",
line_name="Выручка",
year=2025,
period_end=500,
).__dict__
],
)
self.assertTrue(FNSReportService.exists_by_hash("c" * 64))
self.assertTrue(FNSReportService.exists_by_external_id("fns-direct-2"))