294 lines
9.4 KiB
Python
294 lines
9.4 KiB
Python
"""Issuer linkage, complete pagination and independent observations."""
|
|
import importlib
|
|
import json
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from apps.parsers.models import ParserSourceArtifact
|
|
from apps.parsers.registry_snapshots import (
|
|
OwnOrganizationIndex,
|
|
SnapshotValidationError,
|
|
)
|
|
from django.test import override_settings
|
|
from django.utils import timezone
|
|
from organizations.models import Organization
|
|
|
|
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
|
|
|
|
|
def adapter():
|
|
return importlib.import_module("apps.parsers.exchange_moex")
|
|
|
|
|
|
def block(columns, rows):
|
|
return {"columns": columns, "data": rows}
|
|
|
|
|
|
@pytest.fixture
|
|
def scenario(db, tmp_path):
|
|
org = Organization.objects.create(
|
|
name="Synthetic issuer",
|
|
inn="7707083893",
|
|
ogrn="1027700132195",
|
|
okpo="00032537",
|
|
directory_imported_at=timezone.now(),
|
|
)
|
|
wrong = Organization.objects.create(
|
|
name="Wrong issuer",
|
|
inn="0123456789",
|
|
ogrn="1027700132194",
|
|
okpo="01234567",
|
|
directory_imported_at=timezone.now(),
|
|
)
|
|
artifact = ParserSourceArtifact.objects.create(
|
|
source="stock_exchange_activity",
|
|
original_name="moex.ndjson",
|
|
load_batch=1,
|
|
metadata={"dataset": "moex", "input_only": True},
|
|
)
|
|
reference = json.loads((FIXTURES / "reference.json").read_text())
|
|
raw = dict(
|
|
zip(
|
|
reference["securities"]["columns"],
|
|
reference["securities"]["data"][0],
|
|
strict=False,
|
|
)
|
|
)
|
|
raw.update(emitent_inn=org.inn, emitent_id=wrong.inn)
|
|
requests = []
|
|
changes = {}
|
|
|
|
def response_value(url, params):
|
|
if url.endswith("/durations"):
|
|
return b"<div :init-dt=\"new Date('2026-10-02')\"></div>"
|
|
if url.endswith("/durations.json"):
|
|
return {
|
|
"durations": block(
|
|
[
|
|
"tradedate",
|
|
"secid",
|
|
"effectiveyield",
|
|
"duration",
|
|
"title",
|
|
"matdate",
|
|
],
|
|
[
|
|
[
|
|
"2026-10-02",
|
|
"FIXTURE1",
|
|
changes.get("yield", None),
|
|
260,
|
|
"Fixture",
|
|
"2027-01-01",
|
|
]
|
|
],
|
|
)
|
|
}
|
|
if url.endswith("/iss/securities.json"):
|
|
rows = (
|
|
[]
|
|
if params["start"] or params["market"] == "shares"
|
|
else [list(raw.values())]
|
|
)
|
|
if changes.get("repeat") and params["market"] == "bonds":
|
|
rows = [list(raw.values())]
|
|
if changes.get("two") and rows:
|
|
other = dict(raw, secid="FIXTURE2")
|
|
rows.append(list(other.values()))
|
|
value = {"securities": block(list(raw), rows)}
|
|
if changes.get("cursor"):
|
|
value["securities.cursor"] = block(
|
|
["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]]
|
|
)
|
|
return value
|
|
if "/iss/securities/" in url:
|
|
secid = url.rsplit("/", 1)[1][:-5]
|
|
return {
|
|
"description": block(
|
|
["name", "value"],
|
|
[
|
|
["SECID", changes.get("secid", secid)],
|
|
["EMITTER_ID", changes.get("issuer", raw["emitent_id"])],
|
|
["FACEVALUE", "1000"],
|
|
["FACEUNIT", "RUB"],
|
|
],
|
|
),
|
|
"boards": block(["boardid"], [["TQCB"]]),
|
|
}
|
|
secid = url.rsplit("/", 1)[1][:-5]
|
|
return {
|
|
"securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]),
|
|
"marketdata": block(
|
|
["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]]
|
|
),
|
|
"dataversion": block(["trade_date"], [["2026-10-05"]]),
|
|
}
|
|
|
|
@contextmanager
|
|
def response(session, url, params=None, **kwargs):
|
|
requests.append((url, params))
|
|
value = response_value(url, params)
|
|
body = value if isinstance(value, bytes) else json.dumps(value).encode()
|
|
|
|
class Response:
|
|
def iter_content(self, chunk_size):
|
|
yield body
|
|
|
|
yield Response()
|
|
|
|
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch(
|
|
"apps.parsers.exchange_moex.registry_response", response
|
|
):
|
|
yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests
|
|
|
|
|
|
def collect(scenario):
|
|
return list(
|
|
adapter().iter_moex_records(
|
|
organization_index=scenario[3], artifact=scenario[2]
|
|
)
|
|
)
|
|
|
|
|
|
def test_internal_issuer_id_cannot_match_inn(scenario):
|
|
rows = collect(scenario)
|
|
assert len(rows) == 1
|
|
assert rows[0].organization_uid == scenario[0].uid
|
|
assert rows[0].organization_uid != scenario[1].uid
|
|
assert rows[0].payload["issuer_provider_id"] == scenario[1].inn
|
|
assert rows[0].amount is None
|
|
|
|
|
|
def test_all_reference_pages_and_raw_checksums(scenario):
|
|
import hashlib
|
|
|
|
rows = collect(scenario)
|
|
artifact = scenario[2]
|
|
assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3
|
|
assert artifact.metadata["http_requests"] == 7
|
|
with artifact.file.open("rb") as f:
|
|
entries = [json.loads(line) for line in f]
|
|
assert len(entries) == 7
|
|
assert all(
|
|
hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"]
|
|
for e in entries
|
|
)
|
|
assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid)
|
|
|
|
|
|
def test_multiple_bonds_one_issuer(scenario):
|
|
scenario[4]["two"] = True
|
|
assert {r.external_id for r in collect(scenario)} == {
|
|
"moex:FIXTURE1",
|
|
"moex:FIXTURE2",
|
|
}
|
|
|
|
|
|
def test_nullable_yield_and_duration_date_separate_from_trading(scenario):
|
|
row = collect(scenario)[0]
|
|
assert row.payload["duration"] == {
|
|
"value": "260",
|
|
"unit": "days",
|
|
"calculated_on": "2026-10-02",
|
|
}
|
|
assert row.payload["yield"] == {
|
|
"value": None,
|
|
"unit": None,
|
|
"calculated_on": "2026-10-02",
|
|
}
|
|
assert row.payload["trading"]["traded_on"] == "2026-10-05"
|
|
assert row.payload["trading"]["upstream"]["LAST"] == "101.2300"
|
|
|
|
|
|
def test_zero_yield_not_null(scenario):
|
|
scenario[4]["yield"] = 0
|
|
assert collect(scenario)[0].payload["yield"]["value"] == "0"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"key,value",
|
|
[("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)],
|
|
)
|
|
def test_identity_or_incomplete_pagination_rejected(scenario, key, value):
|
|
scenario[4][key] = value
|
|
with pytest.raises(SnapshotValidationError):
|
|
collect(scenario)
|
|
|
|
|
|
def test_request_limit_prevents_unbounded_collection(scenario):
|
|
with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises(
|
|
SnapshotValidationError
|
|
):
|
|
collect(scenario)
|
|
assert len(scenario[5]) == 1
|
|
|
|
|
|
def test_missing_dated_export_link_rejected():
|
|
with pytest.raises(SnapshotValidationError):
|
|
adapter().discover_duration_date("<div>no date</div>")
|
|
|
|
|
|
def test_unknown_columns_rejected():
|
|
with pytest.raises(SnapshotValidationError):
|
|
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
|
|
|
|
|
|
@pytest.mark.parametrize("total", [1, 100, 101])
|
|
def test_cursor_stops_after_complete_final_page(total):
|
|
from io import BytesIO
|
|
from unittest.mock import Mock
|
|
|
|
module = adapter()
|
|
collector = module._Collector(None, BytesIO())
|
|
columns = sorted(module.REFERENCE_COLUMNS)
|
|
|
|
def read(url, params):
|
|
start = params["start"]
|
|
rows = [dict.fromkeys(columns) for _ in range(start, min(start + 100, total))]
|
|
for n, row in enumerate(rows, start):
|
|
row["secid"] = f"S{n}"
|
|
return {
|
|
"securities": block(columns, [[r[c] for c in columns] for r in rows]),
|
|
"securities.cursor": block(
|
|
["INDEX", "TOTAL", "PAGESIZE"], [[start, total, 100]]
|
|
),
|
|
}
|
|
|
|
collector.read = Mock(side_effect=read)
|
|
rows = list(collector.reference("bonds"))
|
|
assert len(rows) == total
|
|
assert collector.read.call_count == (total + 99) // 100
|
|
|
|
|
|
def test_outside_directory_is_counted_without_parameter_requests(scenario):
|
|
scenario[0].delete()
|
|
scenario[3].by_inn.pop("7707083893")
|
|
assert collect(scenario) == []
|
|
assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1
|
|
assert len(scenario[5]) == 5
|
|
|
|
|
|
def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario):
|
|
scenario[3].by_inn.clear()
|
|
assert collect(scenario) == []
|
|
|
|
|
|
def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario):
|
|
scenario[4]["yield"] = "NaN"
|
|
with pytest.raises(SnapshotValidationError):
|
|
collect(scenario)
|
|
assert scenario[2].file.name
|
|
scenario[2].refresh_from_db()
|
|
assert scenario[2].status == "rejected"
|
|
|
|
|
|
def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario):
|
|
scenario[4]["yield"] = "15.212400"
|
|
row = collect(scenario)[0]
|
|
assert row.payload["yield"]["value"] == "15.212400"
|
|
assert row.payload["yield"]["unit"] is None
|
|
assert row.payload["provenance"]["yield_kind"] == "effectiveyield"
|
|
assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]
|