feat: collect MOEX issuer-linked instruments and metrics
This commit is contained in:
266
tests/apps/parsers/test_exchange_moex.py
Normal file
266
tests/apps/parsers/test_exchange_moex.py
Normal file
@@ -0,0 +1,266 @@
|
||||
"""Issuer linkage, complete pagination and independent observations."""
|
||||
import importlib
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_snapshots import (
|
||||
OwnOrganizationIndex,
|
||||
SnapshotValidationError,
|
||||
)
|
||||
from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization
|
||||
|
||||
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
||||
|
||||
|
||||
def adapter():
|
||||
return importlib.import_module("apps.parsers.exchange_moex")
|
||||
|
||||
|
||||
def block(columns, rows):
|
||||
return {"columns": columns, "data": rows}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario(db, tmp_path):
|
||||
org = Organization.objects.create(
|
||||
name="Synthetic issuer",
|
||||
inn="7707083893",
|
||||
ogrn="1027700132195",
|
||||
okpo="00032537",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
wrong = Organization.objects.create(
|
||||
name="Wrong issuer",
|
||||
inn="0123456789",
|
||||
ogrn="1027700132194",
|
||||
okpo="01234567",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source="stock_exchange_activity",
|
||||
original_name="moex.ndjson",
|
||||
load_batch=1,
|
||||
metadata={"dataset": "moex", "input_only": True},
|
||||
)
|
||||
reference = json.loads((FIXTURES / "reference.json").read_text())
|
||||
raw = dict(
|
||||
zip(
|
||||
reference["securities"]["columns"],
|
||||
reference["securities"]["data"][0],
|
||||
strict=False,
|
||||
)
|
||||
)
|
||||
raw.update(emitent_inn=org.inn, emitent_id=wrong.inn)
|
||||
requests = []
|
||||
changes = {}
|
||||
|
||||
def response_value(url, params):
|
||||
if url.endswith("/durations"):
|
||||
return b"<div :init-dt=\"new Date('2026-10-02')\"></div>"
|
||||
if url.endswith("/durations.json"):
|
||||
return {
|
||||
"durations": block(
|
||||
[
|
||||
"tradedate",
|
||||
"secid",
|
||||
"effectiveyield",
|
||||
"duration",
|
||||
"title",
|
||||
"matdate",
|
||||
],
|
||||
[
|
||||
[
|
||||
"2026-10-02",
|
||||
"FIXTURE1",
|
||||
changes.get("yield", None),
|
||||
260,
|
||||
"Fixture",
|
||||
"2027-01-01",
|
||||
]
|
||||
],
|
||||
)
|
||||
}
|
||||
if url.endswith("/iss/securities.json"):
|
||||
rows = (
|
||||
[]
|
||||
if params["start"] or params["market"] == "shares"
|
||||
else [list(raw.values())]
|
||||
)
|
||||
if changes.get("repeat") and params["market"] == "bonds":
|
||||
rows = [list(raw.values())]
|
||||
if changes.get("two") and rows:
|
||||
other = dict(raw, secid="FIXTURE2")
|
||||
rows.append(list(other.values()))
|
||||
value = {"securities": block(list(raw), rows)}
|
||||
if changes.get("cursor"):
|
||||
value["securities.cursor"] = block(
|
||||
["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]]
|
||||
)
|
||||
return value
|
||||
if "/iss/securities/" in url:
|
||||
secid = url.rsplit("/", 1)[1][:-5]
|
||||
return {
|
||||
"description": block(
|
||||
["name", "value"],
|
||||
[
|
||||
["SECID", changes.get("secid", secid)],
|
||||
["EMITTER_ID", changes.get("issuer", raw["emitent_id"])],
|
||||
["FACEVALUE", "1000"],
|
||||
["FACEUNIT", "RUB"],
|
||||
],
|
||||
),
|
||||
"boards": block(["boardid"], [["TQCB"]]),
|
||||
}
|
||||
secid = url.rsplit("/", 1)[1][:-5]
|
||||
return {
|
||||
"securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]),
|
||||
"marketdata": block(
|
||||
["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]]
|
||||
),
|
||||
"dataversion": block(["trade_date"], [["2026-10-05"]]),
|
||||
}
|
||||
|
||||
@contextmanager
|
||||
def response(session, url, params=None, **kwargs):
|
||||
requests.append((url, params))
|
||||
value = response_value(url, params)
|
||||
body = value if isinstance(value, bytes) else json.dumps(value).encode()
|
||||
|
||||
class Response:
|
||||
def iter_content(self, chunk_size):
|
||||
yield body
|
||||
|
||||
yield Response()
|
||||
|
||||
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch(
|
||||
"apps.parsers.exchange_moex.registry_response", response
|
||||
):
|
||||
yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests
|
||||
|
||||
|
||||
def collect(scenario):
|
||||
return list(
|
||||
adapter().iter_moex_records(
|
||||
organization_index=scenario[3], artifact=scenario[2]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_internal_issuer_id_cannot_match_inn(scenario):
|
||||
rows = collect(scenario)
|
||||
assert len(rows) == 1
|
||||
assert rows[0].organization_uid == scenario[0].uid
|
||||
assert rows[0].organization_uid != scenario[1].uid
|
||||
assert rows[0].payload["issuer_provider_id"] == scenario[1].inn
|
||||
assert rows[0].amount is None
|
||||
|
||||
|
||||
def test_all_reference_pages_and_raw_checksums(scenario):
|
||||
import hashlib
|
||||
|
||||
rows = collect(scenario)
|
||||
artifact = scenario[2]
|
||||
assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3
|
||||
assert artifact.metadata["http_requests"] == 7
|
||||
with artifact.file.open("rb") as f:
|
||||
entries = [json.loads(line) for line in f]
|
||||
assert len(entries) == 7
|
||||
assert all(
|
||||
hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"]
|
||||
for e in entries
|
||||
)
|
||||
assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid)
|
||||
|
||||
|
||||
def test_multiple_bonds_one_issuer(scenario):
|
||||
scenario[4]["two"] = True
|
||||
assert {r.external_id for r in collect(scenario)} == {
|
||||
"moex:FIXTURE1",
|
||||
"moex:FIXTURE2",
|
||||
}
|
||||
|
||||
|
||||
def test_nullable_yield_and_duration_date_separate_from_trading(scenario):
|
||||
row = collect(scenario)[0]
|
||||
assert row.payload["duration"] == {
|
||||
"value": "260",
|
||||
"unit": "days",
|
||||
"calculated_on": "2026-10-02",
|
||||
}
|
||||
assert row.payload["yield"] == {
|
||||
"value": None,
|
||||
"unit": None,
|
||||
"calculated_on": "2026-10-02",
|
||||
}
|
||||
assert row.payload["trading"]["traded_on"] == "2026-10-05"
|
||||
assert row.payload["trading"]["upstream"]["LAST"] == "101.2300"
|
||||
|
||||
|
||||
def test_zero_yield_not_null(scenario):
|
||||
scenario[4]["yield"] = 0
|
||||
assert collect(scenario)[0].payload["yield"]["value"] == "0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,value",
|
||||
[("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)],
|
||||
)
|
||||
def test_identity_or_incomplete_pagination_rejected(scenario, key, value):
|
||||
scenario[4][key] = value
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
collect(scenario)
|
||||
|
||||
|
||||
def test_request_limit_prevents_unbounded_collection(scenario):
|
||||
with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises(
|
||||
SnapshotValidationError
|
||||
):
|
||||
collect(scenario)
|
||||
assert len(scenario[5]) == 1
|
||||
|
||||
|
||||
def test_missing_dated_export_link_rejected():
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
adapter().discover_duration_date("<div>no date</div>")
|
||||
|
||||
|
||||
def test_unknown_columns_rejected():
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
|
||||
|
||||
|
||||
def test_outside_directory_is_counted_without_parameter_requests(scenario):
|
||||
scenario[0].delete()
|
||||
scenario[3].by_inn.pop("7707083893")
|
||||
assert collect(scenario) == []
|
||||
assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1
|
||||
assert len(scenario[5]) == 5
|
||||
|
||||
|
||||
def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario):
|
||||
scenario[3].by_inn.clear()
|
||||
assert collect(scenario) == []
|
||||
|
||||
|
||||
def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario):
|
||||
scenario[4]["yield"] = "NaN"
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
collect(scenario)
|
||||
assert scenario[2].file.name
|
||||
scenario[2].refresh_from_db()
|
||||
assert scenario[2].status == "rejected"
|
||||
|
||||
|
||||
def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario):
|
||||
scenario[4]["yield"] = "15.212400"
|
||||
row = collect(scenario)[0]
|
||||
assert row.payload["yield"]["value"] == "15.212400"
|
||||
assert row.payload["yield"]["unit"] is None
|
||||
assert row.payload["provenance"]["yield_kind"] == "effectiveyield"
|
||||
assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]
|
||||
Reference in New Issue
Block a user