"""Issuer linkage, complete pagination and independent observations.""" import importlib import json from contextlib import contextmanager from pathlib import Path from unittest.mock import patch import pytest from apps.parsers.models import ParserSourceArtifact from apps.parsers.registry_snapshots import ( OwnOrganizationIndex, SnapshotValidationError, ) from django.test import override_settings from django.utils import timezone from organizations.models import Organization FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges" def adapter(): return importlib.import_module("apps.parsers.exchange_moex") def block(columns, rows): return {"columns": columns, "data": rows} @pytest.fixture def scenario(db, tmp_path): org = Organization.objects.create( name="Synthetic issuer", inn="7707083893", ogrn="1027700132195", okpo="00032537", directory_imported_at=timezone.now(), ) wrong = Organization.objects.create( name="Wrong issuer", inn="0123456789", ogrn="1027700132194", okpo="01234567", directory_imported_at=timezone.now(), ) artifact = ParserSourceArtifact.objects.create( source="stock_exchange_activity", original_name="moex.ndjson", load_batch=1, metadata={"dataset": "moex", "input_only": True}, ) reference = json.loads((FIXTURES / "reference.json").read_text()) raw = dict( zip( reference["securities"]["columns"], reference["securities"]["data"][0], strict=False, ) ) raw.update(emitent_inn=org.inn, emitent_id=wrong.inn) requests = [] changes = {} def response_value(url, params): if url.endswith("/durations"): return b"
" if url.endswith("/durations.json"): return { "durations": block( [ "tradedate", "secid", "effectiveyield", "duration", "title", "matdate", ], [ [ "2026-10-02", "FIXTURE1", changes.get("yield", None), 260, "Fixture", "2027-01-01", ] ], ) } if url.endswith("/iss/securities.json"): rows = ( [] if params["start"] or params["market"] == "shares" else [list(raw.values())] ) if changes.get("repeat") and params["market"] == "bonds": rows = [list(raw.values())] if changes.get("two") and rows: other = dict(raw, secid="FIXTURE2") rows.append(list(other.values())) value = {"securities": block(list(raw), rows)} if changes.get("cursor"): value["securities.cursor"] = block( ["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]] ) return value if "/iss/securities/" in url: secid = url.rsplit("/", 1)[1][:-5] return { "description": block( ["name", "value"], [ ["SECID", changes.get("secid", secid)], ["EMITTER_ID", changes.get("issuer", raw["emitent_id"])], ["FACEVALUE", "1000"], ["FACEUNIT", "RUB"], ], ), "boards": block(["boardid"], [["TQCB"]]), } secid = url.rsplit("/", 1)[1][:-5] return { "securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]), "marketdata": block( ["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]] ), "dataversion": block(["trade_date"], [["2026-10-05"]]), } @contextmanager def response(session, url, params=None, **kwargs): requests.append((url, params)) value = response_value(url, params) body = value if isinstance(value, bytes) else json.dumps(value).encode() class Response: def iter_content(self, chunk_size): yield body yield Response() with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch( "apps.parsers.exchange_moex.registry_response", response ): yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests def collect(scenario): return list( adapter().iter_moex_records( organization_index=scenario[3], artifact=scenario[2] ) ) def test_internal_issuer_id_cannot_match_inn(scenario): rows = collect(scenario) assert len(rows) == 1 assert rows[0].organization_uid == scenario[0].uid assert rows[0].organization_uid != scenario[1].uid assert rows[0].payload["issuer_provider_id"] == scenario[1].inn assert rows[0].amount is None def test_all_reference_pages_and_raw_checksums(scenario): import hashlib rows = collect(scenario) artifact = scenario[2] assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3 assert artifact.metadata["http_requests"] == 7 with artifact.file.open("rb") as f: entries = [json.loads(line) for line in f] assert len(entries) == 7 assert all( hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"] for e in entries ) assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid) def test_multiple_bonds_one_issuer(scenario): scenario[4]["two"] = True assert {r.external_id for r in collect(scenario)} == { "moex:FIXTURE1", "moex:FIXTURE2", } def test_nullable_yield_and_duration_date_separate_from_trading(scenario): row = collect(scenario)[0] assert row.payload["duration"] == { "value": "260", "unit": "days", "calculated_on": "2026-10-02", } assert row.payload["yield"] == { "value": None, "unit": None, "calculated_on": "2026-10-02", } assert row.payload["trading"]["traded_on"] == "2026-10-05" assert row.payload["trading"]["upstream"]["LAST"] == "101.2300" def test_zero_yield_not_null(scenario): scenario[4]["yield"] = 0 assert collect(scenario)[0].payload["yield"]["value"] == "0" @pytest.mark.parametrize( "key,value", [("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)], ) def test_identity_or_incomplete_pagination_rejected(scenario, key, value): scenario[4][key] = value with pytest.raises(SnapshotValidationError): collect(scenario) def test_request_limit_prevents_unbounded_collection(scenario): with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises( SnapshotValidationError ): collect(scenario) assert len(scenario[5]) == 1 def test_missing_dated_export_link_rejected(): with pytest.raises(SnapshotValidationError): adapter().discover_duration_date("
no date
") def test_unknown_columns_rejected(): with pytest.raises(SnapshotValidationError): adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"}) @pytest.mark.parametrize("total", [1, 100, 101]) def test_cursor_stops_after_complete_final_page(total): from io import BytesIO from unittest.mock import Mock module = adapter() collector = module._Collector(None, BytesIO()) columns = sorted(module.REFERENCE_COLUMNS) def read(url, params): start = params["start"] rows = [dict.fromkeys(columns) for _ in range(start, min(start + 100, total))] for n, row in enumerate(rows, start): row["secid"] = f"S{n}" return { "securities": block(columns, [[r[c] for c in columns] for r in rows]), "securities.cursor": block( ["INDEX", "TOTAL", "PAGESIZE"], [[start, total, 100]] ), } collector.read = Mock(side_effect=read) rows = list(collector.reference("bonds")) assert len(rows) == total assert collector.read.call_count == (total + 99) // 100 def test_outside_directory_is_counted_without_parameter_requests(scenario): scenario[0].delete() scenario[3].by_inn.pop("7707083893") assert collect(scenario) == [] assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1 assert len(scenario[5]) == 5 def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario): scenario[3].by_inn.clear() assert collect(scenario) == [] def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario): scenario[4]["yield"] = "NaN" with pytest.raises(SnapshotValidationError): collect(scenario) assert scenario[2].file.name scenario[2].refresh_from_db() assert scenario[2].status == "rejected" def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario): scenario[4]["yield"] = "15.212400" row = collect(scenario)[0] assert row.payload["yield"]["value"] == "15.212400" assert row.payload["yield"]["unit"] is None assert row.payload["provenance"]["yield_kind"] == "effectiveyield" assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]