Files
mostovik-backend/tests/apps/parsers/test_exchange_moex.py
2026-10-05 17:49:22 +02:00

294 lines
9.4 KiB
Python

"""Issuer linkage, complete pagination and independent observations."""
import importlib
import json
from contextlib import contextmanager
from pathlib import Path
from unittest.mock import patch
import pytest
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import (
OwnOrganizationIndex,
SnapshotValidationError,
)
from django.test import override_settings
from django.utils import timezone
from organizations.models import Organization
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
def adapter():
return importlib.import_module("apps.parsers.exchange_moex")
def block(columns, rows):
return {"columns": columns, "data": rows}
@pytest.fixture
def scenario(db, tmp_path):
org = Organization.objects.create(
name="Synthetic issuer",
inn="7707083893",
ogrn="1027700132195",
okpo="00032537",
directory_imported_at=timezone.now(),
)
wrong = Organization.objects.create(
name="Wrong issuer",
inn="0123456789",
ogrn="1027700132194",
okpo="01234567",
directory_imported_at=timezone.now(),
)
artifact = ParserSourceArtifact.objects.create(
source="stock_exchange_activity",
original_name="moex.ndjson",
load_batch=1,
metadata={"dataset": "moex", "input_only": True},
)
reference = json.loads((FIXTURES / "reference.json").read_text())
raw = dict(
zip(
reference["securities"]["columns"],
reference["securities"]["data"][0],
strict=False,
)
)
raw.update(emitent_inn=org.inn, emitent_id=wrong.inn)
requests = []
changes = {}
def response_value(url, params):
if url.endswith("/durations"):
return b"<div :init-dt=\"new Date('2026-10-02')\"></div>"
if url.endswith("/durations.json"):
return {
"durations": block(
[
"tradedate",
"secid",
"effectiveyield",
"duration",
"title",
"matdate",
],
[
[
"2026-10-02",
"FIXTURE1",
changes.get("yield", None),
260,
"Fixture",
"2027-01-01",
]
],
)
}
if url.endswith("/iss/securities.json"):
rows = (
[]
if params["start"] or params["market"] == "shares"
else [list(raw.values())]
)
if changes.get("repeat") and params["market"] == "bonds":
rows = [list(raw.values())]
if changes.get("two") and rows:
other = dict(raw, secid="FIXTURE2")
rows.append(list(other.values()))
value = {"securities": block(list(raw), rows)}
if changes.get("cursor"):
value["securities.cursor"] = block(
["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]]
)
return value
if "/iss/securities/" in url:
secid = url.rsplit("/", 1)[1][:-5]
return {
"description": block(
["name", "value"],
[
["SECID", changes.get("secid", secid)],
["EMITTER_ID", changes.get("issuer", raw["emitent_id"])],
["FACEVALUE", "1000"],
["FACEUNIT", "RUB"],
],
),
"boards": block(["boardid"], [["TQCB"]]),
}
secid = url.rsplit("/", 1)[1][:-5]
return {
"securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]),
"marketdata": block(
["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]]
),
"dataversion": block(["trade_date"], [["2026-10-05"]]),
}
@contextmanager
def response(session, url, params=None, **kwargs):
requests.append((url, params))
value = response_value(url, params)
body = value if isinstance(value, bytes) else json.dumps(value).encode()
class Response:
def iter_content(self, chunk_size):
yield body
yield Response()
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch(
"apps.parsers.exchange_moex.registry_response", response
):
yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests
def collect(scenario):
return list(
adapter().iter_moex_records(
organization_index=scenario[3], artifact=scenario[2]
)
)
def test_internal_issuer_id_cannot_match_inn(scenario):
rows = collect(scenario)
assert len(rows) == 1
assert rows[0].organization_uid == scenario[0].uid
assert rows[0].organization_uid != scenario[1].uid
assert rows[0].payload["issuer_provider_id"] == scenario[1].inn
assert rows[0].amount is None
def test_all_reference_pages_and_raw_checksums(scenario):
import hashlib
rows = collect(scenario)
artifact = scenario[2]
assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3
assert artifact.metadata["http_requests"] == 7
with artifact.file.open("rb") as f:
entries = [json.loads(line) for line in f]
assert len(entries) == 7
assert all(
hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"]
for e in entries
)
assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid)
def test_multiple_bonds_one_issuer(scenario):
scenario[4]["two"] = True
assert {r.external_id for r in collect(scenario)} == {
"moex:FIXTURE1",
"moex:FIXTURE2",
}
def test_nullable_yield_and_duration_date_separate_from_trading(scenario):
row = collect(scenario)[0]
assert row.payload["duration"] == {
"value": "260",
"unit": "days",
"calculated_on": "2026-10-02",
}
assert row.payload["yield"] == {
"value": None,
"unit": None,
"calculated_on": "2026-10-02",
}
assert row.payload["trading"]["traded_on"] == "2026-10-05"
assert row.payload["trading"]["upstream"]["LAST"] == "101.2300"
def test_zero_yield_not_null(scenario):
scenario[4]["yield"] = 0
assert collect(scenario)[0].payload["yield"]["value"] == "0"
@pytest.mark.parametrize(
"key,value",
[("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)],
)
def test_identity_or_incomplete_pagination_rejected(scenario, key, value):
scenario[4][key] = value
with pytest.raises(SnapshotValidationError):
collect(scenario)
def test_request_limit_prevents_unbounded_collection(scenario):
with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises(
SnapshotValidationError
):
collect(scenario)
assert len(scenario[5]) == 1
def test_missing_dated_export_link_rejected():
with pytest.raises(SnapshotValidationError):
adapter().discover_duration_date("<div>no date</div>")
def test_unknown_columns_rejected():
with pytest.raises(SnapshotValidationError):
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
@pytest.mark.parametrize("total", [1, 100, 101])
def test_cursor_stops_after_complete_final_page(total):
from io import BytesIO
from unittest.mock import Mock
module = adapter()
collector = module._Collector(None, BytesIO())
columns = sorted(module.REFERENCE_COLUMNS)
def read(url, params):
start = params["start"]
rows = [dict.fromkeys(columns) for _ in range(start, min(start + 100, total))]
for n, row in enumerate(rows, start):
row["secid"] = f"S{n}"
return {
"securities": block(columns, [[r[c] for c in columns] for r in rows]),
"securities.cursor": block(
["INDEX", "TOTAL", "PAGESIZE"], [[start, total, 100]]
),
}
collector.read = Mock(side_effect=read)
rows = list(collector.reference("bonds"))
assert len(rows) == total
assert collector.read.call_count == (total + 99) // 100
def test_outside_directory_is_counted_without_parameter_requests(scenario):
scenario[0].delete()
scenario[3].by_inn.pop("7707083893")
assert collect(scenario) == []
assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1
assert len(scenario[5]) == 5
def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario):
scenario[3].by_inn.clear()
assert collect(scenario) == []
def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario):
scenario[4]["yield"] = "NaN"
with pytest.raises(SnapshotValidationError):
collect(scenario)
assert scenario[2].file.name
scenario[2].refresh_from_db()
assert scenario[2].status == "rejected"
def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario):
scenario[4]["yield"] = "15.212400"
row = collect(scenario)[0]
assert row.payload["yield"]["value"] == "15.212400"
assert row.payload["yield"]["unit"] is None
assert row.payload["provenance"]["yield_kind"] == "effectiveyield"
assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]