From d69ba600d4241447371237fb541e661d57760b85 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 17:14:10 +0200 Subject: [PATCH] feat: publish unified exchange activity snapshots --- src/apps/parsers/exchange_snapshot.py | 252 ++++++++++++++++++ .../0046_stock_exchange_activity.py | 220 +++++++++++++++ src/apps/parsers/models.py | 2 + .../0018_stock_exchange_activity.py | 74 +++++ src/organizations/models.py | 13 + src/organizations/source_groups.py | 8 + src/settings/base.py | 8 + tests/apps/parsers/test_exchange_snapshot.py | 198 ++++++++++++++ 8 files changed, 775 insertions(+) create mode 100644 src/apps/parsers/exchange_snapshot.py create mode 100644 src/apps/parsers/migrations/0046_stock_exchange_activity.py create mode 100644 src/organizations/migrations/0018_stock_exchange_activity.py create mode 100644 tests/apps/parsers/test_exchange_snapshot.py diff --git a/src/apps/parsers/exchange_snapshot.py b/src/apps/parsers/exchange_snapshot.py new file mode 100644 index 0000000..1db603a --- /dev/null +++ b/src/apps/parsers/exchange_snapshot.py @@ -0,0 +1,252 @@ +"""Two private exchange inputs, one validated atomic canonical snapshot.""" +from __future__ import annotations + +import json +import re +from collections import Counter +from collections.abc import Callable +from io import BytesIO + +from apps.parsers.exchange_moex import iter_moex_records +from apps.parsers.exchange_spb import ( + EXCHANGE_SOURCE, + download_spb_artifact, + iter_spb_records, +) +from apps.parsers.fssp_csv import valid_legal_inn +from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord +from apps.parsers.registry_snapshots import ( + SNAPSHOT_CHUNK_SIZE, + OwnOrganizationIndex, + SnapshotResult, + SnapshotValidationError, + publish_snapshot, + reject_snapshot, + save_artifact_file, + stage_records, +) + + +def _identity(row, index): + if not valid_legal_inn(row.inn): + return None, "invalid_issuer_inn" + organization, reason = index.resolve(inn=row.inn, ogrn="") + if not reason and ( + not re.fullmatch(r"[0-9]{13}", organization.ogrn or "") + or not re.fullmatch(r"[0-9]{8}", organization.okpo or "") + ): + return organization, "invalid_canonical_requisites" + return organization, reason + + +def _stage( + rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress +): + row_number = aggregate.staged_records.count() + for row in rows: + counts["parsed"] += 1 + if ( + row.payload.get("exchange") != exchange + or not row.external_id.startswith(exchange + ":") + or row.record_type != "exchange_instrument" + or row.amount is not None + ): + raise SnapshotValidationError("invalid_exchange_adapter_record") + organization, reason = _identity(row, index) + if reason: + counts[reason] += 1 + counts["skipped" if reason == "outside_directory" else "quarantined"] += 1 + if reason == "outside_directory": + continue + row.payload["provenance"] = { + **row.payload.get("provenance", {}), + "artifact_id": str(raw.uid), + "source_published_at": raw.source_published_at.isoformat() + if raw.source_published_at + else None, + } + row_number += 1 + pending.append( + ParserStagedRecord( + artifact=aggregate, + row_number=row_number, + organization=organization, + external_id=row.external_id, + record_type=row.record_type, + raw_data=row.payload.get("upstream", {}), + normalized_data={ + "title": row.title, + "record_date": row.record_date, + "amount": None, + "status": row.status, + "url": row.url, + "payload": row.payload, + }, + disposition="quarantined" if reason else "staged", + reason=reason, + ) + ) + if len(pending) >= SNAPSHOT_CHUNK_SIZE: + stage_records(pending) + if on_progress: + on_progress( + sum(c["parsed"] for c in counters.values()), + 0, + aggregate.staged_records.count(), + sum(c["quarantined"] for c in counters.values()), + ) + stage_records(pending) + + +def _save_manifest(aggregate, raws, counters): + aggregate.parsed_count = sum(c["parsed"] for c in counters.values()) + aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values()) + aggregate.metadata = { + **aggregate.metadata, + "inputs": [ + { + "exchange": raw.metadata["dataset"], + "artifact_id": str(raw.uid), + "checksum_sha256": raw.sha256, + "size_bytes": raw.size_bytes, + "published_at": raw.source_published_at.isoformat() + if raw.source_published_at + else None, + "parsed_records_count": raw.parsed_count, + } + for raw in raws + ], + "adapter_counters": {key: dict(value) for key, value in counters.items()}, + "skipped_records_count": sum(c["skipped"] for c in counters.values()), + "reasons": dict( + sum( + ( + Counter( + { + k: v + for k, v in c.items() + if k not in {"parsed", "skipped", "quarantined"} + } + ) + for c in counters.values() + ), + Counter(), + ) + ), + } + aggregate.rejection_reasons = aggregate.metadata["reasons"] + dates = [raw.source_published_at for raw in raws if raw.source_published_at] + aggregate.source_published_at = max(dates) if dates else None + save_artifact_file( + aggregate, + BytesIO( + json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode() + ), + ) + + +def refresh_stock_exchange_activity( + *, + load_batch: int, + uploaded_by_id: int | None = None, + on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None, + on_progress: Callable[[int, int, int, int], None] | None = None, +) -> tuple[ParserSourceArtifact, SnapshotResult]: + aggregate = ParserSourceArtifact.objects.create( + source=EXCHANGE_SOURCE, + original_name="exchange-manifest.json", + content_type="application/json", + version="spb-csv-v1+moex-iss-v1", + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + ) + raws = [] + counters = {"spb": Counter(), "moex": Counter()} + pending = [] + try: + index = OwnOrganizationIndex() + spb = download_spb_artifact( + load_batch=load_batch, uploaded_by_id=uploaded_by_id + ) + raws.append(spb) + spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)} + spb.save(update_fields=("metadata",)) + with spb.file.open("rb") as handle: + _stage( + iter_spb_records(handle), + exchange="spb", + raw=spb, + aggregate=aggregate, + index=index, + counts=counters["spb"], + counters=counters, + pending=pending, + on_progress=on_progress, + ) + spb.parsed_count = counters["spb"]["parsed"] + spb.status = "parsed" + spb.save() + moex = ParserSourceArtifact.objects.create( + source=EXCHANGE_SOURCE, + original_name="moex-responses.ndjson", + content_type="application/x-ndjson", + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + metadata={ + "dataset": "moex", + "input_only": True, + "aggregate_artifact_id": str(aggregate.uid), + }, + ) + raws.append(moex) + _stage( + iter_moex_records(organization_index=index, artifact=moex), + exchange="moex", + raw=moex, + aggregate=aggregate, + index=index, + counts=counters["moex"], + counters=counters, + pending=pending, + on_progress=on_progress, + ) + extra = moex.metadata.get("diagnostics", {}) + counters["moex"]["parsed"] = moex.parsed_count + counters["moex"]["skipped"] += extra.get("outside_directory", 0) + counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0) + counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get( + "missing_or_invalid_issuer_inn", 0 + ) + # Raw checksum/date are final only once the generator reaches EOF. + for staged in aggregate.staged_records.all().iterator( + chunk_size=SNAPSHOT_CHUNK_SIZE + ): + raw = ( + spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex + ) + staged.normalized_data["payload"]["provenance"].update( + checksum_sha256=raw.sha256, + source_published_at=raw.source_published_at.isoformat() + if raw.source_published_at + else None, + ) + staged.save(update_fields=("normalized_data",)) + _save_manifest(aggregate, raws, counters) + aggregate.status = "parsed" + aggregate.save() + return aggregate, publish_snapshot(aggregate, on_publish=on_publish) + except Exception as exc: + # If discovery failed before returning the SPB input, retain its bounded diagnostics too. + if not raws: + raws = list( + ParserSourceArtifact.objects.filter( + source=EXCHANGE_SOURCE, + load_batch=load_batch, + metadata__input_only=True, + ) + ) + _save_manifest(aggregate, raws, counters) + for raw in raws: + reject_snapshot(raw, exc) + reject_snapshot(aggregate, exc) + raise diff --git a/src/apps/parsers/migrations/0046_stock_exchange_activity.py b/src/apps/parsers/migrations/0046_stock_exchange_activity.py new file mode 100644 index 0000000..a23c2d8 --- /dev/null +++ b/src/apps/parsers/migrations/0046_stock_exchange_activity.py @@ -0,0 +1,220 @@ +# Generated by Django 3.2.25 on 2026-10-05 15:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0045_checko_disabled_schedules"), + ] + + operations = [ + migrations.AlterField( + model_name="genericparserrecord", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ("stock_exchange_activity", "Биржевая активность"), + ("checko_leasing", "Лизинг"), + ("checko_messages", "Сообщения организаций"), + ("checko_licenses", "Лицензии организаций"), + ("hh", "Вакансии HeadHunter"), + ("superjob", "Вакансии SuperJob"), + ], + db_index=True, + help_text="Источник данных", + max_length=50, + verbose_name="источник", + ), + ), + migrations.AlterField( + model_name="parserbatchsequence", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ("stock_exchange_activity", "Биржевая активность"), + ("checko_leasing", "Лизинг"), + ("checko_messages", "Сообщения организаций"), + ("checko_licenses", "Лицензии организаций"), + ], + help_text="Источник данных", + max_length=50, + unique=True, + verbose_name="источник", + ), + ), + migrations.AlterField( + model_name="parserloadlog", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ("stock_exchange_activity", "Биржевая активность"), + ("checko_leasing", "Лизинг"), + ("checko_messages", "Сообщения организаций"), + ("checko_licenses", "Лицензии организаций"), + ], + db_index=True, + help_text="Источник данных", + max_length=50, + verbose_name="источник", + ), + ), + migrations.AlterField( + model_name="parsersourceartifact", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Сертификаты промышленного производства"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Единая информационная система закупок"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ("procurements_44fz", "Закупки 44-ФЗ"), + ("procurements_223fz", "Закупки 223-ФЗ"), + ("contracts", "Контракты ЕИС"), + ("unfair_suppliers", "Недобросовестные поставщики"), + ("fas_goz", "Уклонение от ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("fedresurs_bankruptcy", "Банкротства Федресурс"), + ("fstec", "Реестры ФСТЭК"), + ("trudvsem", "Вакансии Работа России"), + ( + "gosedo_address_directory", + "Глобальный адресный справочник ГосЭДО", + ), + ("media_news", "Новости СМИ"), + ("ropk_sanctions", "РОПК — Санкции"), + ( + "fns_sme_support_recipients", + "Реестр субъектов МСП — получателей поддержки", + ), + ("budget_ubpandnubp", "Реестр участников бюджетного процесса"), + ("sro_membership_check", "Проверка членства в СРО"), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ( + "fssp_enforcement_proceedings", + "Исполнительные производства ФССП", + ), + ("stock_exchange_activity", "Биржевая активность"), + ("checko_leasing", "Лизинг"), + ("checko_messages", "Сообщения организаций"), + ("checko_licenses", "Лицензии организаций"), + ], + db_index=True, + max_length=50, + ), + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 3b14eaa..26ed1f9 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -66,6 +66,8 @@ class ParserLoadLog(TimestampMixin, models.Model): _("Исполнительные производства ФССП"), ) + STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность") + CHECKO_LEASING = "checko_leasing", _("Лизинг") CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций") CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций") diff --git a/src/organizations/migrations/0018_stock_exchange_activity.py b/src/organizations/migrations/0018_stock_exchange_activity.py new file mode 100644 index 0000000..dda3340 --- /dev/null +++ b/src/organizations/migrations/0018_stock_exchange_activity.py @@ -0,0 +1,74 @@ +# Generated by Django 3.2.25 on 2026-10-05 15:12 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("organizations", "0017_checko_sections"), + ] + + operations = [ + migrations.CreateModel( + name="StockExchangeActivityExtension", + fields=[ + ( + "organizationsourceextension_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="organizations.organizationsourceextension", + ), + ), + ], + options={ + "verbose_name": "биржевая активность", + "verbose_name_plural": "биржевая активность", + "db_table": "organizations_stock_exchange_activity_extension", + }, + bases=("organizations.organizationsourceextension",), + ), + migrations.AlterField( + model_name="organizationsourceextension", + name="source_group", + field=models.CharField( + choices=[ + ("financial_indicators", "Финансово-экономические показатели"), + ("government_procurements", "Государственные закупки"), + ("industrial_production", "Производители и продукция России"), + ("planned_inspections", "Плановые проверки"), + ("bankruptcy", "Сведения о процедурах банкротства"), + ("defense_suppliers", "Недобросовестные поставщики ГОЗ"), + ("arbitration", "Арбитражные дела"), + ("security_registries", "Реестры по информационной безопасности"), + ("vacancies", "Вакансии"), + ("electronic_document_exchange", "Электронный документооборот"), + ("media_mentions", "Упоминания в СМИ"), + ("sanctions", "Санкции"), + ("government_support", "Государственная поддержка"), + ("sro_membership", "Членство в СРО"), + ( + "budget_process_registry", + "Реестр участников бюджетного процесса", + ), + ( + "integrated_structure_documents", + "Документы интегрированных структур", + ), + ("fedresurs_company_profile", "Карточки компаний Федресурса"), + ("enforcement_proceedings", "Исполнительные производства"), + ("stock_exchange_activity", "Биржевая активность"), + ("leasing", "Лизинг"), + ("company_messages", "Сообщения организаций"), + ("company_licenses", "Лицензии организаций"), + ], + db_index=True, + max_length=64, + verbose_name="группа источников", + ), + ), + ] diff --git a/src/organizations/models.py b/src/organizations/models.py index 204c9b7..7885de3 100644 --- a/src/organizations/models.py +++ b/src/organizations/models.py @@ -57,6 +57,8 @@ class SourceGroup(models.TextChoices): _("Исполнительные производства"), ) + STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность") + LEASING = "leasing", _("Лизинг") COMPANY_MESSAGES = "company_messages", _("Сообщения организаций") COMPANY_LICENSES = "company_licenses", _("Лицензии организаций") @@ -1070,3 +1072,14 @@ class CompanyLicenseExtension(OrganizationSourceExtension): db_table = "organizations_company_license_extension" verbose_name = _("лицензии организации") verbose_name_plural = _("лицензии организации") + + +class StockExchangeActivityExtension(OrganizationSourceExtension): + """Combined SPB and MOEX observations for one canonical organization.""" + + source_group_value = SourceGroup.STOCK_EXCHANGE_ACTIVITY + + class Meta: + db_table = "organizations_stock_exchange_activity_extension" + verbose_name = _("биржевая активность") + verbose_name_plural = _("биржевая активность") diff --git a/src/organizations/source_groups.py b/src/organizations/source_groups.py index 8d73fac..cacaf3b 100644 --- a/src/organizations/source_groups.py +++ b/src/organizations/source_groups.py @@ -29,6 +29,7 @@ from organizations.models import ( SecurityRegistryExtension, SourceGroup, SroMembershipExtension, + StockExchangeActivityExtension, VacancyExtension, ) @@ -45,6 +46,13 @@ class SourceGroupDescriptor: SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = { + ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY: SourceGroupDescriptor( + source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY, + source_group=SourceGroup.STOCK_EXCHANGE_ACTIVITY, + record_type="exchange_instrument", + title="Биржевая активность", + extension_model=StockExchangeActivityExtension, + ), ParserLoadLog.Source.CHECKO_LICENSES: SourceGroupDescriptor( source=ParserLoadLog.Source.CHECKO_LICENSES, source_group=SourceGroup.COMPANY_LICENSES, diff --git a/src/settings/base.py b/src/settings/base.py index 1bf1cd3..c61aac3 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -528,3 +528,11 @@ SRO_REQUEST_INTERVAL_SECONDS = float(os.getenv("SRO_REQUEST_INTERVAL_SECONDS", " SRO_HTTP_MAX_RESPONSE_BYTES = int(os.getenv("SRO_HTTP_MAX_RESPONSE_BYTES", "2097152")) SRO_HTTP_TIMEOUT_SECONDS = int(os.getenv("SRO_HTTP_TIMEOUT_SECONDS", "30")) SRO_MAX_REQUESTS_PER_RUN = int(os.getenv("SRO_MAX_REQUESTS_PER_RUN", "10000")) + +# Public exchange collection bounds; TLS verification stays enabled. +EXCHANGE_SPB_MAX_BYTES = 50 * 1024**2 +EXCHANGE_SPB_MAX_ROWS = 100000 +EXCHANGE_MOEX_MAX_RESPONSE_BYTES = 10 * 1024**2 +EXCHANGE_MOEX_MAX_PAGES = 1000 +EXCHANGE_MOEX_MAX_REQUESTS = 10000 +EXCHANGE_MOEX_MAX_DURATION_ROWS = 100000 diff --git a/tests/apps/parsers/test_exchange_snapshot.py b/tests/apps/parsers/test_exchange_snapshot.py new file mode 100644 index 0000000..30602df --- /dev/null +++ b/tests/apps/parsers/test_exchange_snapshot.py @@ -0,0 +1,198 @@ +"""One source, two validated inputs, atomic publication with stable identities.""" +import csv +import importlib +from io import BytesIO, StringIO +from pathlib import Path +from unittest.mock import patch + +import pytest +from apps.parsers.models import ParserSourceArtifact +from apps.parsers.registry_snapshots import SnapshotValidationError, save_artifact_file +from django.test import TestCase, override_settings +from django.utils import timezone +from organizations.models import Organization, OrganizationSourceRecord +from organizations.source_ingestion import SourceRecordInput + +FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges" + + +def adapter(): + return importlib.import_module("apps.parsers.exchange_snapshot") + + +class ExchangeSnapshotTests(TestCase): + def setUp(self): + from tempfile import TemporaryDirectory + + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.settings = override_settings( + PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private", + MEDIA_ROOT=self.tmp.name + "/public", + ) + self.settings.enable() + self.addCleanup(self.settings.disable) + self.org = Organization.objects.create( + name="Canonical", + inn="7707083893", + ogrn="1027700132195", + okpo="00032537", + directory_imported_at=timezone.now(), + ) + + def run_import( + self, batch=1, fail=False, inn=None, duplicates=False, count=1, on_publish=None + ): + module = adapter() + + def download(**kwargs): + raw = ParserSourceArtifact.objects.create( + source="stock_exchange_activity", + original_name="spb.csv", + load_batch=batch, + source_published_at="2026-10-05", + metadata={"dataset": "spb", "input_only": True}, + ) + rows = list( + csv.reader( + StringIO( + (FIXTURES / "listing.csv").read_text(encoding="utf-8-sig") + ), + delimiter=";", + ) + ) + rows[1][rows[0].index("e_INN_code")] = ( + inn if inn is not None else self.org.inn + ) + out = StringIO() + csv.writer(out, delimiter=";").writerows(rows) + save_artifact_file(raw, BytesIO(out.getvalue().encode())) + raw.refresh_from_db() + return raw + + def moex(*, organization_index, artifact): + save_artifact_file(artifact, BytesIO(b'{"synthetic":true}\n')) + for n in range(count): + if fail: + raise SnapshotValidationError("synthetic_moex_failed") + yield SourceRecordInput( + external_id=f"moex:{0 if duplicates else n}", + record_type="exchange_instrument", + title="Synthetic bond", + organization_name=self.org.name, + inn=self.org.inn, + payload={ + "exchange": "moex", + "instrument_id": str(n), + "issuer_inn": self.org.inn, + "isin": "RU0000000001", + "provenance": {}, + "upstream": {"id": n}, + }, + ) + artifact.parsed_count = count + artifact.source_published_at = timezone.now().date() + artifact.status = "parsed" + artifact.save() + + with patch.object(module, "download_spb_artifact", download), patch.object( + module, "iter_moex_records", moex + ): + return module.refresh_stock_exchange_activity( + load_batch=batch, on_publish=on_publish + ) + + def rows(self): + return list( + OrganizationSourceRecord.objects.filter(source="stock_exchange_activity") + .order_by("external_id") + .values("uid", "payload", "load_batch") + ) + + def test_same_isin_on_two_exchanges_two_stable_records_no_new_org(self): + artifact, result = self.run_import() + before = self.rows() + assert result.published == 2 + assert {r["payload"]["exchange"] for r in before} == {"moex", "spb"} + assert len({r["uid"] for r in before}) == 2 + self.run_import(batch=2) + assert [r["uid"] for r in self.rows()] == [r["uid"] for r in before] + assert Organization.objects.count() == 1 + assert not OrganizationSourceRecord.objects.exclude(legacy_model="").exists() + assert len(artifact.metadata["inputs"]) == 2 + assert {i["exchange"] for i in artifact.metadata["inputs"]} == {"moex", "spb"} + assert all(i["checksum_sha256"] for i in artifact.metadata["inputs"]) + assert artifact.metadata["adapter_counters"]["spb"]["parsed"] == 1 + assert artifact.metadata["adapter_counters"]["moex"]["parsed"] == 1 + + def test_second_adapter_failure_keeps_combined_snapshot_and_two_inputs(self): + self.run_import() + before = self.rows() + with pytest.raises(SnapshotValidationError): + self.run_import(batch=2, fail=True) + assert self.rows() == before + rejected = ParserSourceArtifact.objects.get( + load_batch=2, original_name="exchange-manifest.json" + ) + assert rejected.status == "rejected" + assert len(rejected.metadata["inputs"]) == 2 + + def test_conflicting_vendor_identity_rejects_whole_snapshot(self): + self.run_import() + before = self.rows() + with pytest.raises(SnapshotValidationError): + self.run_import(batch=2, duplicates=True, count=2) + assert self.rows() == before + + def test_foreign_org_skipped_and_missing_inn_quarantined(self): + artifact, result = self.run_import(inn="7810150688") + assert result.skipped == 1 + assert result.published == 1 + artifact, result = self.run_import(batch=2, inn="") + assert result.quarantined == 1 + assert artifact.metadata["adapter_counters"]["spb"]["invalid_issuer_inn"] == 1 + + def test_ambiguous_organization_quarantined_without_new_org(self): + Organization.objects.create( + name="Ambiguous", + inn=self.org.inn, + ogrn="1027700132194", + okpo="01234567", + directory_imported_at=timezone.now(), + ) + artifact, result = self.run_import() + assert result.quarantined == 2 + assert result.published == 0 + assert artifact.metadata["reasons"]["ambiguous_organization"] == 2 + + def test_invalid_canonical_requisites_never_published(self): + self.org.ogrn = "bad" + self.org.save() + _, result = self.run_import() + assert result.quarantined == 2 + assert result.published == 0 + + def test_staging_flushes_at_500(self): + from apps.parsers.registry_snapshots import stage_records + + sizes = [] + + def stage(rows): + sizes.append(len(rows)) + stage_records(rows) + + with patch.object(adapter(), "stage_records", stage): + _, result = self.run_import(count=501) + assert result.published == 502 + assert max(sizes) == 500 + + def test_publication_callback_failure_rolls_back(self): + self.run_import() + before = self.rows() + + def fail(artifact, result): + raise RuntimeError("synthetic callback failure") + + with pytest.raises(RuntimeError): + self.run_import(batch=2, on_publish=fail) + assert self.rows() == before