199 lines
7.5 KiB
Python
199 lines
7.5 KiB
Python
"""One source, two validated inputs, atomic publication with stable identities."""
|
|
import csv
|
|
import importlib
|
|
from io import BytesIO, StringIO
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from apps.parsers.models import ParserSourceArtifact
|
|
from apps.parsers.registry_snapshots import SnapshotValidationError, save_artifact_file
|
|
from django.test import TestCase, override_settings
|
|
from django.utils import timezone
|
|
from organizations.models import Organization, OrganizationSourceRecord
|
|
from organizations.source_ingestion import SourceRecordInput
|
|
|
|
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
|
|
|
|
|
def adapter():
|
|
return importlib.import_module("apps.parsers.exchange_snapshot")
|
|
|
|
|
|
class ExchangeSnapshotTests(TestCase):
|
|
def setUp(self):
|
|
from tempfile import TemporaryDirectory
|
|
|
|
self.tmp = TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.settings = override_settings(
|
|
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
|
|
MEDIA_ROOT=self.tmp.name + "/public",
|
|
)
|
|
self.settings.enable()
|
|
self.addCleanup(self.settings.disable)
|
|
self.org = Organization.objects.create(
|
|
name="Canonical",
|
|
inn="7707083893",
|
|
ogrn="1027700132195",
|
|
okpo="00032537",
|
|
directory_imported_at=timezone.now(),
|
|
)
|
|
|
|
def run_import(
|
|
self, batch=1, fail=False, inn=None, duplicates=False, count=1, on_publish=None
|
|
):
|
|
module = adapter()
|
|
|
|
def download(**kwargs):
|
|
raw = ParserSourceArtifact.objects.create(
|
|
source="stock_exchange_activity",
|
|
original_name="spb.csv",
|
|
load_batch=batch,
|
|
source_published_at="2026-10-05",
|
|
metadata={"dataset": "spb", "input_only": True},
|
|
)
|
|
rows = list(
|
|
csv.reader(
|
|
StringIO(
|
|
(FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")
|
|
),
|
|
delimiter=";",
|
|
)
|
|
)
|
|
rows[1][rows[0].index("e_INN_code")] = (
|
|
inn if inn is not None else self.org.inn
|
|
)
|
|
out = StringIO()
|
|
csv.writer(out, delimiter=";").writerows(rows)
|
|
save_artifact_file(raw, BytesIO(out.getvalue().encode()))
|
|
raw.refresh_from_db()
|
|
return raw
|
|
|
|
def moex(*, organization_index, artifact):
|
|
save_artifact_file(artifact, BytesIO(b'{"synthetic":true}\n'))
|
|
for n in range(count):
|
|
if fail:
|
|
raise SnapshotValidationError("synthetic_moex_failed")
|
|
yield SourceRecordInput(
|
|
external_id=f"moex:{0 if duplicates else n}",
|
|
record_type="exchange_instrument",
|
|
title="Synthetic bond",
|
|
organization_name=self.org.name,
|
|
inn=self.org.inn,
|
|
payload={
|
|
"exchange": "moex",
|
|
"instrument_id": str(n),
|
|
"issuer_inn": self.org.inn,
|
|
"isin": "RU0000000001",
|
|
"provenance": {},
|
|
"upstream": {"id": n},
|
|
},
|
|
)
|
|
artifact.parsed_count = count
|
|
artifact.source_published_at = timezone.now().date()
|
|
artifact.status = "parsed"
|
|
artifact.save()
|
|
|
|
with patch.object(module, "download_spb_artifact", download), patch.object(
|
|
module, "iter_moex_records", moex
|
|
):
|
|
return module.refresh_stock_exchange_activity(
|
|
load_batch=batch, on_publish=on_publish
|
|
)
|
|
|
|
def rows(self):
|
|
return list(
|
|
OrganizationSourceRecord.objects.filter(source="stock_exchange_activity")
|
|
.order_by("external_id")
|
|
.values("uid", "payload", "load_batch")
|
|
)
|
|
|
|
def test_same_isin_on_two_exchanges_two_stable_records_no_new_org(self):
|
|
artifact, result = self.run_import()
|
|
before = self.rows()
|
|
assert result.published == 2
|
|
assert {r["payload"]["exchange"] for r in before} == {"moex", "spb"}
|
|
assert len({r["uid"] for r in before}) == 2
|
|
self.run_import(batch=2)
|
|
assert [r["uid"] for r in self.rows()] == [r["uid"] for r in before]
|
|
assert Organization.objects.count() == 1
|
|
assert not OrganizationSourceRecord.objects.exclude(legacy_model="").exists()
|
|
assert len(artifact.metadata["inputs"]) == 2
|
|
assert {i["exchange"] for i in artifact.metadata["inputs"]} == {"moex", "spb"}
|
|
assert all(i["checksum_sha256"] for i in artifact.metadata["inputs"])
|
|
assert artifact.metadata["adapter_counters"]["spb"]["parsed"] == 1
|
|
assert artifact.metadata["adapter_counters"]["moex"]["parsed"] == 1
|
|
|
|
def test_second_adapter_failure_keeps_combined_snapshot_and_two_inputs(self):
|
|
self.run_import()
|
|
before = self.rows()
|
|
with pytest.raises(SnapshotValidationError):
|
|
self.run_import(batch=2, fail=True)
|
|
assert self.rows() == before
|
|
rejected = ParserSourceArtifact.objects.get(
|
|
load_batch=2, original_name="exchange-manifest.json"
|
|
)
|
|
assert rejected.status == "rejected"
|
|
assert len(rejected.metadata["inputs"]) == 2
|
|
|
|
def test_conflicting_vendor_identity_rejects_whole_snapshot(self):
|
|
self.run_import()
|
|
before = self.rows()
|
|
with pytest.raises(SnapshotValidationError):
|
|
self.run_import(batch=2, duplicates=True, count=2)
|
|
assert self.rows() == before
|
|
|
|
def test_foreign_org_skipped_and_missing_inn_quarantined(self):
|
|
artifact, result = self.run_import(inn="7810150688")
|
|
assert result.skipped == 1
|
|
assert result.published == 1
|
|
artifact, result = self.run_import(batch=2, inn="")
|
|
assert result.quarantined == 1
|
|
assert artifact.metadata["adapter_counters"]["spb"]["invalid_issuer_inn"] == 1
|
|
|
|
def test_ambiguous_organization_quarantined_without_new_org(self):
|
|
Organization.objects.create(
|
|
name="Ambiguous",
|
|
inn=self.org.inn,
|
|
ogrn="1027700132194",
|
|
okpo="01234567",
|
|
directory_imported_at=timezone.now(),
|
|
)
|
|
artifact, result = self.run_import()
|
|
assert result.quarantined == 2
|
|
assert result.published == 0
|
|
assert artifact.metadata["reasons"]["ambiguous_organization"] == 2
|
|
|
|
def test_invalid_canonical_requisites_never_published(self):
|
|
self.org.ogrn = "bad"
|
|
self.org.save()
|
|
_, result = self.run_import()
|
|
assert result.quarantined == 2
|
|
assert result.published == 0
|
|
|
|
def test_staging_flushes_at_500(self):
|
|
from apps.parsers.registry_snapshots import stage_records
|
|
|
|
sizes = []
|
|
|
|
def stage(rows):
|
|
sizes.append(len(rows))
|
|
stage_records(rows)
|
|
|
|
with patch.object(adapter(), "stage_records", stage):
|
|
_, result = self.run_import(count=501)
|
|
assert result.published == 502
|
|
assert max(sizes) == 500
|
|
|
|
def test_publication_callback_failure_rolls_back(self):
|
|
self.run_import()
|
|
before = self.rows()
|
|
|
|
def fail(artifact, result):
|
|
raise RuntimeError("synthetic callback failure")
|
|
|
|
with pytest.raises(RuntimeError):
|
|
self.run_import(batch=2, on_publish=fail)
|
|
assert self.rows() == before
|