diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index 2d1f508..5cfc1ec 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -9,7 +9,7 @@ import secrets import struct import zlib from dataclasses import dataclass -from datetime import date +from datetime import date, datetime from decimal import Decimal from io import BytesIO from pathlib import Path @@ -20,9 +20,13 @@ from zipfile import ZIP_DEFLATED, ZipFile import requests from apps.parsers.models import ( VACANCY_RECORD_SOURCES, + FinancialReport, + FinancialReportLine, GenericParserRecord, + IndustrialCertificateRecord, IndustrialProductRecord, InspectionRecord, + ManufacturerRecord, ParserLoadLog, ProcurementRecord, ) @@ -56,6 +60,7 @@ class StateCorpExchangeService: AAD = b"state-corp-exchange-v1" PAYLOAD_FORMAT = "state-corp-exchange-payload" BIN_FORMAT = "state-corp-exchange-bin" + SCHEMA_VERSION = 2 ROSATOM_ROSCOSMOS_REGISTRY_NAMES = ( "Реестр госкорпорации Роскосмос", "Реестр госкорпорации Роскосмос ГОЗ", @@ -84,11 +89,21 @@ class StateCorpExchangeService: else cls._get_rosatom_roscosmos_organizations(snapshot_date) ) allowed_inns = {str(item.mn_inn) for item in organizations} + allowed_ogrn_to_inn = { + str(item.mn_ogrn): str(item.mn_inn) + for item in organizations + if item.mn_ogrn + } data = { "organizations": cls._serialize_organizations(organizations), + "industrial_certificates": cls._serialize_industrial_certificates( + allowed_inns + ), + "manufacturers": cls._serialize_manufacturers(allowed_inns), "industrial_products": cls._serialize_industrial_products(allowed_inns), "prosecutor_checks": cls._serialize_prosecutor_checks(allowed_inns), "public_procurements": cls._serialize_public_procurements(allowed_inns), + "financial_reports": cls._serialize_financial_reports(allowed_ogrn_to_inn), "arbitration_cases": cls._serialize_arbitration_cases(allowed_inns), "bankruptcy_procedures": cls._serialize_bankruptcy_procedures(allowed_inns), "defense_unreliable_suppliers": ( @@ -103,11 +118,12 @@ class StateCorpExchangeService: package_id = package_id or cls._build_package_id() payload = { "format": cls.PAYLOAD_FORMAT, - "schema_version": 1, + "schema_version": cls.SCHEMA_VERSION, "manifest": { "package_id": package_id, "source_system": source_system, "produced_at": produced_at.isoformat(), + "schema_version": cls.SCHEMA_VERSION, "sections": [key for key, items in data.items() if items], }, "data": data, @@ -233,7 +249,7 @@ class StateCorpExchangeService: "nonce": cls._b64url(nonce), "aad": cls._b64url(cls.AAD), "package_id": package_id, - "schema_version": 1, + "schema_version": cls.SCHEMA_VERSION, "plaintext_sha256": hashlib.sha256(payload_bytes).hexdigest(), "compressed_sha256": hashlib.sha256(compressed_payload).hexdigest(), "ciphertext_sha256": hashlib.sha256(encrypted_payload).hexdigest(), @@ -307,6 +323,56 @@ class StateCorpExchangeService: for item in organizations ] + @classmethod + def _serialize_industrial_certificates( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str | None]]: + queryset = IndustrialCertificateRecord.objects.filter( + inn__in=allowed_inns + ).order_by("id") + items: list[dict[str, str | None]] = [] + for record in queryset: + if not record.certificate_number: + continue + issue_date = cls._coerce_date( + record.issue_date_normalized, record.issue_date + ) + expiry_date = cls._coerce_date( + record.expiry_date_normalized, record.expiry_date + ) + items.append( + { + "organization_inn": cls._digits(record.inn), + "certificate_number": record.certificate_number, + "issue_date": issue_date.isoformat() if issue_date else None, + "expiry_date": expiry_date.isoformat() if expiry_date else None, + "certificate_file_url": record.certificate_file_url, + "organisation_name": record.organisation_name, + "ogrn": cls._digits(record.ogrn), + } + ) + return items + + @classmethod + def _serialize_manufacturers( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str]]: + queryset = ManufacturerRecord.objects.filter(inn__in=allowed_inns).order_by( + "id" + ) + return [ + { + "organization_inn": cls._digits(record.inn), + "full_legal_name": record.full_legal_name, + "inn": cls._digits(record.inn), + "ogrn": cls._digits(record.ogrn), + "address": record.address, + } + for record in queryset + ] + @classmethod def _serialize_industrial_products( cls, @@ -391,8 +457,131 @@ class StateCorpExchangeService: "purchase_name": record.purchase_name, } ) + + for record in cls._generic_records( + allowed_inns, + sources=[ + ParserLoadLog.Source.PROCUREMENTS_44FZ, + ParserLoadLog.Source.PROCUREMENTS_223FZ, + ParserLoadLog.Source.CONTRACTS, + ], + ): + item = cls._serialize_generic_public_procurement(record) + if item is not None: + items.append(item) return items + @classmethod + def _serialize_generic_public_procurement( + cls, + record: GenericParserRecord, + ) -> dict[str, Any] | None: + payload = cls._record_payload(record) + purchase_number = ( + cls._payload_lookup(payload, ["registry_number", "number"]) + or record.external_id + ) + contract_date = cls._coerce_date( + None, + record.record_date + or cls._payload_lookup( + payload, + ["contract_date", "publish_date", "Размещено", "Обновлено"], + ), + ) + if not purchase_number or contract_date is None: + return None + + execution_end_date = cls._coerce_date( + None, + cls._payload_lookup( + payload, + ["execution_end_date", "end_date", "Окончание подачи заявок"], + ), + ) + return { + "organization_inn": cls._digits(record.inn), + "purchase_number": purchase_number, + "law_type": cls._payload_lookup(payload, ["law"]) + or cls._law_type_for_generic_procurement_source(record.source), + "status": record.status + or cls._payload_lookup(payload, ["status", "Статус"]), + "contract_amount": cls._serialize_decimal(record.amount), + "contract_date": contract_date.isoformat(), + "execution_start_date": contract_date.isoformat(), + "execution_end_date": ( + execution_end_date.isoformat() if execution_end_date else None + ), + "purchase_name": record.title + or cls._payload_lookup( + payload, + ["purchase_name", "Объект закупки", "Объекты закупки"], + ), + } + + @staticmethod + def _law_type_for_generic_procurement_source(source: str) -> str: + if source == ParserLoadLog.Source.PROCUREMENTS_44FZ: + return "44-ФЗ" + if source == ParserLoadLog.Source.PROCUREMENTS_223FZ: + return "223-ФЗ" + if source == ParserLoadLog.Source.CONTRACTS: + return "contract" + return "" + + @classmethod + def _serialize_financial_reports( + cls, + allowed_ogrn_to_inn: dict[str, str], + ) -> list[dict[str, Any]]: + if not allowed_ogrn_to_inn: + return [] + + queryset = ( + FinancialReport.objects.filter(ogrn__in=allowed_ogrn_to_inn) + .prefetch_related("lines") + .order_by("id") + ) + return [ + { + "organization_inn": allowed_ogrn_to_inn[report.ogrn], + "external_id": report.external_id, + "ogrn": cls._digits(report.ogrn), + "file_name": report.file_name, + "file_hash": report.file_hash, + "load_batch": report.load_batch, + "status": report.status, + "source": report.source, + "error_message": report.error_message, + "lines": [ + cls._serialize_financial_report_line(line) + for line in sorted( + report.lines.all(), + key=lambda line: ( + line.year, + line.form_code, + line.line_code, + ), + ) + ], + } + for report in queryset + if report.external_id + ] + + @staticmethod + def _serialize_financial_report_line( + line: FinancialReportLine, + ) -> dict[str, Any]: + return { + "form_code": line.form_code, + "line_code": line.line_code, + "line_name": line.line_name, + "year": line.year, + "period_start": line.period_start, + "period_end": line.period_end, + } + @classmethod def _serialize_arbitration_cases( cls, @@ -679,7 +868,13 @@ class StateCorpExchangeService: try: return date.fromisoformat(raw_text) except ValueError: - return None + pass + for date_format in ("%d.%m.%Y", "%d.%m.%y"): + try: + return datetime.strptime(raw_text, date_format).date() + except ValueError: + continue + return None @staticmethod def _serialize_decimal(value: Decimal | None) -> str | None: diff --git a/src/settings/production.py b/src/settings/production.py index 03e8f82..e5b7535 100644 --- a/src/settings/production.py +++ b/src/settings/production.py @@ -17,12 +17,21 @@ def _require_env(name: str) -> str: return value -def _parse_allowed_hosts(raw_value: str) -> list[str]: +def _is_truthy_env(name: str) -> bool: + return os.getenv(name, "false").strip().lower() in {"1", "true", "yes", "on"} + + +def _parse_allowed_hosts(raw_value: str, *, allow_any_host: bool = False) -> list[str]: hosts = [host.strip() for host in raw_value.split(",") if host.strip()] if not hosts: raise ImproperlyConfigured("ALLOWED_HOSTS must contain at least one host") if "*" in hosts: - raise ImproperlyConfigured("ALLOWED_HOSTS must not contain '*' in production") + if allow_any_host: + return ["*"] + raise ImproperlyConfigured( + "ALLOWED_HOSTS must not contain '*' in production unless " + "ALLOW_ANY_HOSTS=true is set" + ) return hosts @@ -30,7 +39,10 @@ SECRET_KEY = _require_env("SECRET_KEY") DEBUG = os.getenv("DEBUG", "false").strip().lower() == "true" if DEBUG: raise ImproperlyConfigured("DEBUG must be False in production") -ALLOWED_HOSTS = _parse_allowed_hosts(_require_env("ALLOWED_HOSTS")) +ALLOWED_HOSTS = _parse_allowed_hosts( + _require_env("ALLOWED_HOSTS"), + allow_any_host=_is_truthy_env("ALLOW_ANY_HOSTS"), +) # JWT SIMPLE_JWT["SIGNING_KEY"] = SECRET_KEY diff --git a/tests/apps/exchange/test_state_corp_services.py b/tests/apps/exchange/test_state_corp_services.py index d9b8c6c..1c39544 100644 --- a/tests/apps/exchange/test_state_corp_services.py +++ b/tests/apps/exchange/test_state_corp_services.py @@ -12,14 +12,21 @@ from unittest.mock import Mock, patch from zipfile import ZipFile from apps.exchange.state_corp_services import StateCorpExchangeService -from apps.parsers.models import GenericParserRecord, ParserLoadLog +from apps.parsers.models import ( + FinancialReport, + FinancialReportLine, + GenericParserRecord, + ParserLoadLog, +) from apps.registers.models import Register from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.test import TestCase, override_settings from tests.apps.parsers.factories import ( + IndustrialCertificateRecordFactory, IndustrialProductRecordFactory, InspectionRecordFactory, + ManufacturerRecordFactory, ProcurementRecordFactory, ) from tests.apps.registers.factories import ( @@ -78,6 +85,25 @@ class StateCorpExchangeServiceTest(TestCase): started_at="2026-01-01", ended_at=None, ) + IndustrialCertificateRecordFactory.create( + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + organisation_name=organization.pn_name, + certificate_number="cert-001", + issue_date="2026-01-10", + issue_date_normalized="2026-01-10", + expiry_date="2027-01-10", + expiry_date_normalized="2027-01-10", + certificate_file_url="https://minpromtorg.gov.ru/cert/001", + registry_organization=organization, + ) + ManufacturerRecordFactory.create( + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + full_legal_name=organization.pn_name, + address="г. Москва, ул. Тверская, д. 1", + registry_organization=organization, + ) IndustrialProductRecordFactory.create( inn=str(organization.mn_inn), ogrn=str(organization.mn_ogrn), @@ -114,6 +140,42 @@ class StateCorpExchangeServiceTest(TestCase): max_price_amount="4500000.75", registry_organization=organization, ) + financial_report = FinancialReport.objects.create( + external_id="fin-001", + ogrn=str(organization.mn_ogrn), + registry_organization=organization, + file_name="fin_001_1027700132195.xlsx", + file_hash="f" * 64, + load_batch=1, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + FinancialReportLine.objects.create( + report=financial_report, + form_code="1", + line_code="1600", + line_name="Баланс", + year=2025, + period_start=1000, + period_end=1500, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.PROCUREMENTS_44FZ, + load_batch=1, + external_id="purchase-generic-001", + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + title="Поставка generic-оборудования", + record_date="15.02.2026", + amount="2500000.00", + status="published", + payload={ + "law": "44-ФЗ", + "registry_number": "purchase-generic-001", + "Окончание подачи заявок": "20.02.2026", + }, + registry_organization=organization, + ) GenericParserRecord.objects.create( source=ParserLoadLog.Source.ARBITRATION, load_batch=1, @@ -198,9 +260,12 @@ class StateCorpExchangeServiceTest(TestCase): package = StateCorpExchangeService.build_package(actual_date="2026-03-15") self.assertEqual(package.payload_counts["organizations"], 1) + self.assertEqual(package.payload_counts["industrial_certificates"], 1) + self.assertEqual(package.payload_counts["manufacturers"], 1) self.assertEqual(package.payload_counts["industrial_products"], 1) self.assertEqual(package.payload_counts["prosecutor_checks"], 1) - self.assertEqual(package.payload_counts["public_procurements"], 1) + self.assertEqual(package.payload_counts["public_procurements"], 2) + self.assertEqual(package.payload_counts["financial_reports"], 1) self.assertEqual(package.payload_counts["arbitration_cases"], 1) self.assertEqual(package.payload_counts["bankruptcy_procedures"], 1) self.assertEqual(package.payload_counts["defense_unreliable_suppliers"], 1) @@ -212,6 +277,14 @@ class StateCorpExchangeServiceTest(TestCase): self.assertEqual(payload["format"], StateCorpExchangeService.PAYLOAD_FORMAT) self.assertEqual(payload["manifest"]["source_system"], "mostovik") self.assertEqual(payload["data"]["organizations"][0]["inn"], "7707083893") + self.assertEqual( + payload["data"]["industrial_certificates"][0]["certificate_number"], + "cert-001", + ) + self.assertEqual( + payload["data"]["manufacturers"][0]["full_legal_name"], + "АО Альфа", + ) self.assertEqual( payload["data"]["industrial_products"][0]["registry_number"], "prod-001", @@ -224,6 +297,10 @@ class StateCorpExchangeServiceTest(TestCase): payload["data"]["public_procurements"][0]["purchase_number"], "purchase-001", ) + self.assertEqual( + payload["data"]["financial_reports"][0]["lines"][0]["line_code"], + "1600", + ) self.assertEqual( payload["data"]["arbitration_cases"][0]["case_number"], "А40-1/2026",