Files
mostovik-backend/tests/apps/parsers/test_exchange_snapshot.py
2026-10-05 17:14:10 +02:00

199 lines
7.5 KiB
Python

"""One source, two validated inputs, atomic publication with stable identities."""
import csv
import importlib
from io import BytesIO, StringIO
from pathlib import Path
from unittest.mock import patch
import pytest
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError, save_artifact_file
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import SourceRecordInput
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
def adapter():
return importlib.import_module("apps.parsers.exchange_snapshot")
class ExchangeSnapshotTests(TestCase):
def setUp(self):
from tempfile import TemporaryDirectory
self.tmp = TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.settings = override_settings(
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
MEDIA_ROOT=self.tmp.name + "/public",
)
self.settings.enable()
self.addCleanup(self.settings.disable)
self.org = Organization.objects.create(
name="Canonical",
inn="7707083893",
ogrn="1027700132195",
okpo="00032537",
directory_imported_at=timezone.now(),
)
def run_import(
self, batch=1, fail=False, inn=None, duplicates=False, count=1, on_publish=None
):
module = adapter()
def download(**kwargs):
raw = ParserSourceArtifact.objects.create(
source="stock_exchange_activity",
original_name="spb.csv",
load_batch=batch,
source_published_at="2026-10-05",
metadata={"dataset": "spb", "input_only": True},
)
rows = list(
csv.reader(
StringIO(
(FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")
),
delimiter=";",
)
)
rows[1][rows[0].index("e_INN_code")] = (
inn if inn is not None else self.org.inn
)
out = StringIO()
csv.writer(out, delimiter=";").writerows(rows)
save_artifact_file(raw, BytesIO(out.getvalue().encode()))
raw.refresh_from_db()
return raw
def moex(*, organization_index, artifact):
save_artifact_file(artifact, BytesIO(b'{"synthetic":true}\n'))
for n in range(count):
if fail:
raise SnapshotValidationError("synthetic_moex_failed")
yield SourceRecordInput(
external_id=f"moex:{0 if duplicates else n}",
record_type="exchange_instrument",
title="Synthetic bond",
organization_name=self.org.name,
inn=self.org.inn,
payload={
"exchange": "moex",
"instrument_id": str(n),
"issuer_inn": self.org.inn,
"isin": "RU0000000001",
"provenance": {},
"upstream": {"id": n},
},
)
artifact.parsed_count = count
artifact.source_published_at = timezone.now().date()
artifact.status = "parsed"
artifact.save()
with patch.object(module, "download_spb_artifact", download), patch.object(
module, "iter_moex_records", moex
):
return module.refresh_stock_exchange_activity(
load_batch=batch, on_publish=on_publish
)
def rows(self):
return list(
OrganizationSourceRecord.objects.filter(source="stock_exchange_activity")
.order_by("external_id")
.values("uid", "payload", "load_batch")
)
def test_same_isin_on_two_exchanges_two_stable_records_no_new_org(self):
artifact, result = self.run_import()
before = self.rows()
assert result.published == 2
assert {r["payload"]["exchange"] for r in before} == {"moex", "spb"}
assert len({r["uid"] for r in before}) == 2
self.run_import(batch=2)
assert [r["uid"] for r in self.rows()] == [r["uid"] for r in before]
assert Organization.objects.count() == 1
assert not OrganizationSourceRecord.objects.exclude(legacy_model="").exists()
assert len(artifact.metadata["inputs"]) == 2
assert {i["exchange"] for i in artifact.metadata["inputs"]} == {"moex", "spb"}
assert all(i["checksum_sha256"] for i in artifact.metadata["inputs"])
assert artifact.metadata["adapter_counters"]["spb"]["parsed"] == 1
assert artifact.metadata["adapter_counters"]["moex"]["parsed"] == 1
def test_second_adapter_failure_keeps_combined_snapshot_and_two_inputs(self):
self.run_import()
before = self.rows()
with pytest.raises(SnapshotValidationError):
self.run_import(batch=2, fail=True)
assert self.rows() == before
rejected = ParserSourceArtifact.objects.get(
load_batch=2, original_name="exchange-manifest.json"
)
assert rejected.status == "rejected"
assert len(rejected.metadata["inputs"]) == 2
def test_conflicting_vendor_identity_rejects_whole_snapshot(self):
self.run_import()
before = self.rows()
with pytest.raises(SnapshotValidationError):
self.run_import(batch=2, duplicates=True, count=2)
assert self.rows() == before
def test_foreign_org_skipped_and_missing_inn_quarantined(self):
artifact, result = self.run_import(inn="7810150688")
assert result.skipped == 1
assert result.published == 1
artifact, result = self.run_import(batch=2, inn="")
assert result.quarantined == 1
assert artifact.metadata["adapter_counters"]["spb"]["invalid_issuer_inn"] == 1
def test_ambiguous_organization_quarantined_without_new_org(self):
Organization.objects.create(
name="Ambiguous",
inn=self.org.inn,
ogrn="1027700132194",
okpo="01234567",
directory_imported_at=timezone.now(),
)
artifact, result = self.run_import()
assert result.quarantined == 2
assert result.published == 0
assert artifact.metadata["reasons"]["ambiguous_organization"] == 2
def test_invalid_canonical_requisites_never_published(self):
self.org.ogrn = "bad"
self.org.save()
_, result = self.run_import()
assert result.quarantined == 2
assert result.published == 0
def test_staging_flushes_at_500(self):
from apps.parsers.registry_snapshots import stage_records
sizes = []
def stage(rows):
sizes.append(len(rows))
stage_records(rows)
with patch.object(adapter(), "stage_records", stage):
_, result = self.run_import(count=501)
assert result.published == 502
assert max(sizes) == 500
def test_publication_callback_failure_rolls_back(self):
self.run_import()
before = self.rows()
def fail(artifact, result):
raise RuntimeError("synthetic callback failure")
with pytest.raises(RuntimeError):
self.run_import(batch=2, on_publish=fail)
assert self.rows() == before