feat: collect MOEX issuer-linked instruments and metrics

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:09:07 +02:00
parent 084965b16d
commit e2cf384853
2 changed files with 644 additions and 0 deletions

View File

@@ -0,0 +1,378 @@
"""Bounded ISS collection, exact issuer linkage and dated independent observations."""
from __future__ import annotations
import hashlib
import json
import re
import tempfile
from collections import Counter
from collections.abc import Iterator
from datetime import date
from decimal import Decimal, InvalidOperation
from urllib.parse import quote
from apps.parsers.fssp_csv import valid_legal_inn
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_http import (
limited_bytes,
registry_response,
registry_session,
)
from apps.parsers.registry_snapshots import (
OwnOrganizationIndex,
SnapshotValidationError,
reject_snapshot,
save_artifact_file,
)
from django.conf import settings
from organizations.source_ingestion import SourceRecordInput
ISS_BASE = "https://iss.moex.com/iss/"
DURATION_PAGE = "https://www.moex.com/ru/bondization/durations"
DURATION_ROUTE = "statistics/engines/stock/markets/bonds/durations.json"
MOEX_HOSTS = frozenset({"iss.moex.com", "www.moex.com"})
REFERENCE_COLUMNS = {
"secid",
"emitent_id",
"emitent_inn",
"name",
"isin",
"type",
"group",
"primary_boardid",
}
def _strings(value):
if isinstance(value, Decimal):
return format(value, "f")
if isinstance(value, list):
return [_strings(v) for v in value]
if isinstance(value, dict):
return {k: _strings(v) for k, v in value.items()}
return value
def _date(value) -> str:
if not isinstance(value, str):
raise SnapshotValidationError("missing_moex_observation_date")
try:
return date.fromisoformat(value).isoformat()
except ValueError as exc:
raise SnapshotValidationError("invalid_moex_observation_date") from exc
def discover_duration_date(html: str) -> str:
matches = re.findall(
r':init-dt=["\']new Date\([\'\"](\d{4}-\d{2}-\d{2})[\'\"]\)', html
)
if len(set(matches)) != 1:
raise SnapshotValidationError("moex_duration_publication_date_missing")
return _date(matches[0])
def iss_rows(document: dict, block: str, required: set[str]) -> list[dict]:
value = document.get(block)
if not isinstance(value, dict):
raise SnapshotValidationError("missing_moex_block")
columns = value.get("columns")
rows = value.get("data")
if (
not isinstance(columns, list)
or any(not isinstance(c, str) for c in columns)
or len(set(columns)) != len(columns)
or not required.issubset(columns)
or not isinstance(rows, list)
):
raise SnapshotValidationError("unknown_moex_columns")
result = []
for row in rows:
if not isinstance(row, list) or len(row) != len(columns):
raise SnapshotValidationError("invalid_moex_row_width")
result.append(dict(zip(columns, row, strict=False)))
return result
def _metric(value, unit, calculated_on):
number = None
if value is not None:
try:
decimal = Decimal(str(value))
if not decimal.is_finite():
raise InvalidOperation
number = format(decimal, "f")
except InvalidOperation as exc:
raise SnapshotValidationError("invalid_moex_metric") from exc
return {"value": number, "unit": unit, "calculated_on": calculated_on}
class _Collector:
def __init__(self, session, handle):
self.session = session
self.handle = handle
self.requests = 0
def read(self, url, params=None, *, html=False):
if self.requests >= getattr(settings, "EXCHANGE_MOEX_MAX_REQUESTS", 10000):
raise SnapshotValidationError("moex_request_limit")
self.requests += 1
with registry_response(
self.session, url, hosts=MOEX_HOSTS, params=params
) as response:
body = limited_bytes(
response,
4 * 1024**2
if html
else getattr(
settings, "EXCHANGE_MOEX_MAX_RESPONSE_BYTES", 10 * 1024**2
),
)
try:
text = body.decode("utf-8")
value = text if html else _strings(json.loads(text, parse_float=Decimal))
except (UnicodeError, ValueError) as exc:
raise SnapshotValidationError("invalid_moex_response") from exc
envelope = {
"url": url,
"params": params,
"sha256": hashlib.sha256(body).hexdigest(),
"body_utf8": text,
}
self.handle.write((json.dumps(envelope, ensure_ascii=False) + "\n").encode())
if not html and not isinstance(value, dict):
raise SnapshotValidationError("invalid_moex_document")
return value
def reference(self, market):
seen = set()
total = None
for page in range(getattr(settings, "EXCHANGE_MOEX_MAX_PAGES", 1000)):
start = page * 100
document = self.read(
ISS_BASE + "securities.json",
{
"engine": "stock",
"market": market,
"is_trading": 1,
"limit": 100,
"start": start,
},
)
rows = iss_rows(document, "securities", REFERENCE_COLUMNS)
if len(rows) > 100:
raise SnapshotValidationError("moex_page_too_large")
if "securities.cursor" in document:
cursors = iss_rows(
document, "securities.cursor", {"INDEX", "TOTAL", "PAGESIZE"}
)
if len(cursors) != 1:
raise SnapshotValidationError("invalid_moex_cursor")
cursor = cursors[0]
if (
any(
not isinstance(cursor[key], int)
or isinstance(cursor[key], bool)
for key in ("INDEX", "TOTAL", "PAGESIZE")
)
or cursor["INDEX"] != start
or cursor["PAGESIZE"] != 100
or cursor["TOTAL"] < start
or (total is not None and total != cursor["TOTAL"])
or len(rows) != min(100, cursor["TOTAL"] - start)
):
raise SnapshotValidationError("incomplete_moex_cursor")
total = cursor["TOTAL"]
if not rows:
return
for row in rows:
secid = row["secid"]
if (
not isinstance(secid, str)
or not secid
or len(secid) > 200
or secid in seen
):
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
seen.add(secid)
yield row
raise SnapshotValidationError("moex_page_limit")
def _duration_map(collector):
html = collector.read(DURATION_PAGE, html=True)
published = discover_duration_date(html)
document = collector.read(ISS_BASE + DURATION_ROUTE, {"date": published})
rows = iss_rows(
document, "durations", {"tradedate", "secid", "effectiveyield", "duration"}
)
if len(rows) > getattr(settings, "EXCHANGE_MOEX_MAX_DURATION_ROWS", 100000):
raise SnapshotValidationError("moex_duration_row_limit")
result = {}
for row in rows:
if (
_date(row["tradedate"]) != published
or not isinstance(row["secid"], str)
or not row["secid"]
or row["secid"] in result
):
raise SnapshotValidationError("conflicting_moex_duration")
result[row["secid"]] = row
return published, result
def _parameters(collector, reference):
code = quote(reference["secid"], safe="")
document = collector.read(ISS_BASE + f"securities/{code}.json")
rows = iss_rows(document, "description", {"name", "value"})
parameters = {}
for row in rows:
if not isinstance(row["name"], str) or row["name"] in parameters:
raise SnapshotValidationError("conflicting_moex_parameters")
parameters[row["name"]] = row["value"]
if parameters.get("SECID") != reference["secid"] or str(
parameters.get("EMITTER_ID")
) != str(reference["emitent_id"]):
raise SnapshotValidationError("moex_parameter_identity_mismatch")
boards = iss_rows(document, "boards", {"boardid"})
if reference["primary_boardid"] not in {b["boardid"] for b in boards}:
raise SnapshotValidationError("moex_primary_board_missing")
return parameters, document
def _trading(collector, reference, market):
code = quote(reference["secid"], safe="")
document = collector.read(
ISS_BASE + f"engines/stock/markets/{market}/securities/{code}.json"
)
securities = iss_rows(document, "securities", {"SECID", "BOARDID"})
observations = iss_rows(document, "marketdata", {"SECID", "BOARDID"})
versions = iss_rows(document, "dataversion", {"trade_date"})
if any(r["SECID"] != reference["secid"] for r in securities + observations):
raise SnapshotValidationError("moex_trading_identity_mismatch")
rows = [r for r in observations if r["BOARDID"] == reference["primary_boardid"]]
if len(rows) > 1:
raise SnapshotValidationError("conflicting_moex_trading")
if not rows:
return None, document
if len(versions) != 1:
raise SnapshotValidationError("moex_trading_date_missing")
return {
"traded_on": _date(versions[0]["trade_date"]),
"board_id": reference["primary_boardid"],
"upstream": rows[0],
}, document
def _collect(collector, index, artifact, counts):
published, durations = _duration_map(collector)
artifact.source_published_at = date.fromisoformat(published)
seen = set()
for market in ("shares", "bonds"):
for reference in collector.reference(market):
counts["parsed"] += 1
secid = reference["secid"]
if secid in seen:
raise SnapshotValidationError("conflicting_moex_market_identity")
seen.add(secid)
inn = reference["emitent_inn"]
if not isinstance(inn, str) or not valid_legal_inn(inn):
counts["missing_or_invalid_issuer_inn"] += 1
continue
organization, reason = index.resolve(inn=inn, ogrn="")
if reason == "outside_directory":
counts["outside_directory"] += 1
continue
payload = {
"exchange": "moex",
"instrument_id": secid,
"issuer_inn": inn,
"issuer_provider_id": str(reference["emitent_id"])
if reference["emitent_id"] is not None
else None,
"isin": reference["isin"],
"instrument_kind": reference["type"],
"listing_date": None,
"nominal_value": None,
"nominal_currency": None,
"parameters": {},
"trading": None,
"duration": None,
"yield": None,
"upstream": {"reference": reference},
"provenance": {
"artifact_id": str(artifact.uid),
"duration_published_at": published,
},
}
if not reason:
parameters, raw_parameters = _parameters(collector, reference)
trading, raw_trading = _trading(collector, reference, market)
payload.update(
parameters=parameters,
trading=trading,
nominal_currency=parameters.get("FACEUNIT"),
)
nominal = parameters.get("FACEVALUE")
if nominal is not None:
payload["nominal_value"] = _metric(nominal, None, None)["value"]
payload["upstream"].update(
parameters=raw_parameters, trading=raw_trading
)
metrics = durations.get(secid) if market == "bonds" else None
if metrics is not None:
calculated_on = _date(metrics["tradedate"])
payload.update(
{
"duration": _metric(
metrics["duration"], "days", calculated_on
),
"yield": _metric(
metrics["effectiveyield"], None, calculated_on
),
}
)
payload["upstream"]["duration"] = metrics
payload["provenance"]["yield_kind"] = "effectiveyield"
payload["provenance"]["warnings"] = ["yield_unit_unverified"]
yield SourceRecordInput(
external_id=f"moex:{secid}",
title=reference["name"] or secid,
organization_name=organization.name if organization else "",
organization_uid=organization.uid if organization else None,
inn=inn,
record_type="exchange_instrument",
record_date=payload["trading"]["traded_on"]
if payload["trading"]
else "",
url=ISS_BASE + f'securities/{quote(secid,safe="")}.json',
payload=payload,
)
def iter_moex_records(
*, organization_index: OwnOrganizationIndex, artifact: ParserSourceArtifact
) -> Iterator[SourceRecordInput]:
counts = Counter()
collector = None
complete = False
try:
with tempfile.TemporaryFile() as handle, registry_session() as session:
collector = _Collector(session, handle)
try:
yield from _collect(collector, organization_index, artifact, counts)
complete = True
finally:
save_artifact_file(artifact, handle)
artifact.parsed_count = counts["parsed"]
artifact.metadata = {
**artifact.metadata,
"http_requests": collector.requests,
"diagnostics": dict(counts),
}
artifact.save()
if complete:
artifact.status = ParserSourceArtifact.Status.PARSED
artifact.save(update_fields=("status",))
except Exception as exc:
reject_snapshot(artifact, exc)
raise

View File

@@ -0,0 +1,266 @@
"""Issuer linkage, complete pagination and independent observations."""
import importlib
import json
from contextlib import contextmanager
from pathlib import Path
from unittest.mock import patch
import pytest
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import (
OwnOrganizationIndex,
SnapshotValidationError,
)
from django.test import override_settings
from django.utils import timezone
from organizations.models import Organization
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
def adapter():
return importlib.import_module("apps.parsers.exchange_moex")
def block(columns, rows):
return {"columns": columns, "data": rows}
@pytest.fixture
def scenario(db, tmp_path):
org = Organization.objects.create(
name="Synthetic issuer",
inn="7707083893",
ogrn="1027700132195",
okpo="00032537",
directory_imported_at=timezone.now(),
)
wrong = Organization.objects.create(
name="Wrong issuer",
inn="0123456789",
ogrn="1027700132194",
okpo="01234567",
directory_imported_at=timezone.now(),
)
artifact = ParserSourceArtifact.objects.create(
source="stock_exchange_activity",
original_name="moex.ndjson",
load_batch=1,
metadata={"dataset": "moex", "input_only": True},
)
reference = json.loads((FIXTURES / "reference.json").read_text())
raw = dict(
zip(
reference["securities"]["columns"],
reference["securities"]["data"][0],
strict=False,
)
)
raw.update(emitent_inn=org.inn, emitent_id=wrong.inn)
requests = []
changes = {}
def response_value(url, params):
if url.endswith("/durations"):
return b"<div :init-dt=\"new Date('2026-10-02')\"></div>"
if url.endswith("/durations.json"):
return {
"durations": block(
[
"tradedate",
"secid",
"effectiveyield",
"duration",
"title",
"matdate",
],
[
[
"2026-10-02",
"FIXTURE1",
changes.get("yield", None),
260,
"Fixture",
"2027-01-01",
]
],
)
}
if url.endswith("/iss/securities.json"):
rows = (
[]
if params["start"] or params["market"] == "shares"
else [list(raw.values())]
)
if changes.get("repeat") and params["market"] == "bonds":
rows = [list(raw.values())]
if changes.get("two") and rows:
other = dict(raw, secid="FIXTURE2")
rows.append(list(other.values()))
value = {"securities": block(list(raw), rows)}
if changes.get("cursor"):
value["securities.cursor"] = block(
["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]]
)
return value
if "/iss/securities/" in url:
secid = url.rsplit("/", 1)[1][:-5]
return {
"description": block(
["name", "value"],
[
["SECID", changes.get("secid", secid)],
["EMITTER_ID", changes.get("issuer", raw["emitent_id"])],
["FACEVALUE", "1000"],
["FACEUNIT", "RUB"],
],
),
"boards": block(["boardid"], [["TQCB"]]),
}
secid = url.rsplit("/", 1)[1][:-5]
return {
"securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]),
"marketdata": block(
["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]]
),
"dataversion": block(["trade_date"], [["2026-10-05"]]),
}
@contextmanager
def response(session, url, params=None, **kwargs):
requests.append((url, params))
value = response_value(url, params)
body = value if isinstance(value, bytes) else json.dumps(value).encode()
class Response:
def iter_content(self, chunk_size):
yield body
yield Response()
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch(
"apps.parsers.exchange_moex.registry_response", response
):
yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests
def collect(scenario):
return list(
adapter().iter_moex_records(
organization_index=scenario[3], artifact=scenario[2]
)
)
def test_internal_issuer_id_cannot_match_inn(scenario):
rows = collect(scenario)
assert len(rows) == 1
assert rows[0].organization_uid == scenario[0].uid
assert rows[0].organization_uid != scenario[1].uid
assert rows[0].payload["issuer_provider_id"] == scenario[1].inn
assert rows[0].amount is None
def test_all_reference_pages_and_raw_checksums(scenario):
import hashlib
rows = collect(scenario)
artifact = scenario[2]
assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3
assert artifact.metadata["http_requests"] == 7
with artifact.file.open("rb") as f:
entries = [json.loads(line) for line in f]
assert len(entries) == 7
assert all(
hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"]
for e in entries
)
assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid)
def test_multiple_bonds_one_issuer(scenario):
scenario[4]["two"] = True
assert {r.external_id for r in collect(scenario)} == {
"moex:FIXTURE1",
"moex:FIXTURE2",
}
def test_nullable_yield_and_duration_date_separate_from_trading(scenario):
row = collect(scenario)[0]
assert row.payload["duration"] == {
"value": "260",
"unit": "days",
"calculated_on": "2026-10-02",
}
assert row.payload["yield"] == {
"value": None,
"unit": None,
"calculated_on": "2026-10-02",
}
assert row.payload["trading"]["traded_on"] == "2026-10-05"
assert row.payload["trading"]["upstream"]["LAST"] == "101.2300"
def test_zero_yield_not_null(scenario):
scenario[4]["yield"] = 0
assert collect(scenario)[0].payload["yield"]["value"] == "0"
@pytest.mark.parametrize(
"key,value",
[("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)],
)
def test_identity_or_incomplete_pagination_rejected(scenario, key, value):
scenario[4][key] = value
with pytest.raises(SnapshotValidationError):
collect(scenario)
def test_request_limit_prevents_unbounded_collection(scenario):
with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises(
SnapshotValidationError
):
collect(scenario)
assert len(scenario[5]) == 1
def test_missing_dated_export_link_rejected():
with pytest.raises(SnapshotValidationError):
adapter().discover_duration_date("<div>no date</div>")
def test_unknown_columns_rejected():
with pytest.raises(SnapshotValidationError):
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
def test_outside_directory_is_counted_without_parameter_requests(scenario):
scenario[0].delete()
scenario[3].by_inn.pop("7707083893")
assert collect(scenario) == []
assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1
assert len(scenario[5]) == 5
def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario):
scenario[3].by_inn.clear()
assert collect(scenario) == []
def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario):
scenario[4]["yield"] = "NaN"
with pytest.raises(SnapshotValidationError):
collect(scenario)
assert scenario[2].file.name
scenario[2].refresh_from_db()
assert scenario[2].status == "rejected"
def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario):
scenario[4]["yield"] = "15.212400"
row = collect(scenario)[0]
assert row.payload["yield"]["value"] == "15.212400"
assert row.payload["yield"]["unit"] is None
assert row.payload["provenance"]["yield_kind"] == "effectiveyield"
assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]