From e2cf384853d26b8ea0adead1b2019f6206414194 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 17:09:07 +0200 Subject: [PATCH] feat: collect MOEX issuer-linked instruments and metrics --- src/apps/parsers/exchange_moex.py | 378 +++++++++++++++++++++++ tests/apps/parsers/test_exchange_moex.py | 266 ++++++++++++++++ 2 files changed, 644 insertions(+) create mode 100644 src/apps/parsers/exchange_moex.py create mode 100644 tests/apps/parsers/test_exchange_moex.py diff --git a/src/apps/parsers/exchange_moex.py b/src/apps/parsers/exchange_moex.py new file mode 100644 index 0000000..f200430 --- /dev/null +++ b/src/apps/parsers/exchange_moex.py @@ -0,0 +1,378 @@ +"""Bounded ISS collection, exact issuer linkage and dated independent observations.""" +from __future__ import annotations + +import hashlib +import json +import re +import tempfile +from collections import Counter +from collections.abc import Iterator +from datetime import date +from decimal import Decimal, InvalidOperation +from urllib.parse import quote + +from apps.parsers.fssp_csv import valid_legal_inn +from apps.parsers.models import ParserSourceArtifact +from apps.parsers.registry_http import ( + limited_bytes, + registry_response, + registry_session, +) +from apps.parsers.registry_snapshots import ( + OwnOrganizationIndex, + SnapshotValidationError, + reject_snapshot, + save_artifact_file, +) +from django.conf import settings +from organizations.source_ingestion import SourceRecordInput + +ISS_BASE = "https://iss.moex.com/iss/" +DURATION_PAGE = "https://www.moex.com/ru/bondization/durations" +DURATION_ROUTE = "statistics/engines/stock/markets/bonds/durations.json" +MOEX_HOSTS = frozenset({"iss.moex.com", "www.moex.com"}) +REFERENCE_COLUMNS = { + "secid", + "emitent_id", + "emitent_inn", + "name", + "isin", + "type", + "group", + "primary_boardid", +} + + +def _strings(value): + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, list): + return [_strings(v) for v in value] + if isinstance(value, dict): + return {k: _strings(v) for k, v in value.items()} + return value + + +def _date(value) -> str: + if not isinstance(value, str): + raise SnapshotValidationError("missing_moex_observation_date") + try: + return date.fromisoformat(value).isoformat() + except ValueError as exc: + raise SnapshotValidationError("invalid_moex_observation_date") from exc + + +def discover_duration_date(html: str) -> str: + matches = re.findall( + r':init-dt=["\']new Date\([\'\"](\d{4}-\d{2}-\d{2})[\'\"]\)', html + ) + if len(set(matches)) != 1: + raise SnapshotValidationError("moex_duration_publication_date_missing") + return _date(matches[0]) + + +def iss_rows(document: dict, block: str, required: set[str]) -> list[dict]: + value = document.get(block) + if not isinstance(value, dict): + raise SnapshotValidationError("missing_moex_block") + columns = value.get("columns") + rows = value.get("data") + if ( + not isinstance(columns, list) + or any(not isinstance(c, str) for c in columns) + or len(set(columns)) != len(columns) + or not required.issubset(columns) + or not isinstance(rows, list) + ): + raise SnapshotValidationError("unknown_moex_columns") + result = [] + for row in rows: + if not isinstance(row, list) or len(row) != len(columns): + raise SnapshotValidationError("invalid_moex_row_width") + result.append(dict(zip(columns, row, strict=False))) + return result + + +def _metric(value, unit, calculated_on): + number = None + if value is not None: + try: + decimal = Decimal(str(value)) + if not decimal.is_finite(): + raise InvalidOperation + number = format(decimal, "f") + except InvalidOperation as exc: + raise SnapshotValidationError("invalid_moex_metric") from exc + return {"value": number, "unit": unit, "calculated_on": calculated_on} + + +class _Collector: + def __init__(self, session, handle): + self.session = session + self.handle = handle + self.requests = 0 + + def read(self, url, params=None, *, html=False): + if self.requests >= getattr(settings, "EXCHANGE_MOEX_MAX_REQUESTS", 10000): + raise SnapshotValidationError("moex_request_limit") + self.requests += 1 + with registry_response( + self.session, url, hosts=MOEX_HOSTS, params=params + ) as response: + body = limited_bytes( + response, + 4 * 1024**2 + if html + else getattr( + settings, "EXCHANGE_MOEX_MAX_RESPONSE_BYTES", 10 * 1024**2 + ), + ) + try: + text = body.decode("utf-8") + value = text if html else _strings(json.loads(text, parse_float=Decimal)) + except (UnicodeError, ValueError) as exc: + raise SnapshotValidationError("invalid_moex_response") from exc + envelope = { + "url": url, + "params": params, + "sha256": hashlib.sha256(body).hexdigest(), + "body_utf8": text, + } + self.handle.write((json.dumps(envelope, ensure_ascii=False) + "\n").encode()) + if not html and not isinstance(value, dict): + raise SnapshotValidationError("invalid_moex_document") + return value + + def reference(self, market): + seen = set() + total = None + for page in range(getattr(settings, "EXCHANGE_MOEX_MAX_PAGES", 1000)): + start = page * 100 + document = self.read( + ISS_BASE + "securities.json", + { + "engine": "stock", + "market": market, + "is_trading": 1, + "limit": 100, + "start": start, + }, + ) + rows = iss_rows(document, "securities", REFERENCE_COLUMNS) + if len(rows) > 100: + raise SnapshotValidationError("moex_page_too_large") + if "securities.cursor" in document: + cursors = iss_rows( + document, "securities.cursor", {"INDEX", "TOTAL", "PAGESIZE"} + ) + if len(cursors) != 1: + raise SnapshotValidationError("invalid_moex_cursor") + cursor = cursors[0] + if ( + any( + not isinstance(cursor[key], int) + or isinstance(cursor[key], bool) + for key in ("INDEX", "TOTAL", "PAGESIZE") + ) + or cursor["INDEX"] != start + or cursor["PAGESIZE"] != 100 + or cursor["TOTAL"] < start + or (total is not None and total != cursor["TOTAL"]) + or len(rows) != min(100, cursor["TOTAL"] - start) + ): + raise SnapshotValidationError("incomplete_moex_cursor") + total = cursor["TOTAL"] + if not rows: + return + for row in rows: + secid = row["secid"] + if ( + not isinstance(secid, str) + or not secid + or len(secid) > 200 + or secid in seen + ): + raise SnapshotValidationError("duplicate_or_invalid_moex_security") + seen.add(secid) + yield row + raise SnapshotValidationError("moex_page_limit") + + +def _duration_map(collector): + html = collector.read(DURATION_PAGE, html=True) + published = discover_duration_date(html) + document = collector.read(ISS_BASE + DURATION_ROUTE, {"date": published}) + rows = iss_rows( + document, "durations", {"tradedate", "secid", "effectiveyield", "duration"} + ) + if len(rows) > getattr(settings, "EXCHANGE_MOEX_MAX_DURATION_ROWS", 100000): + raise SnapshotValidationError("moex_duration_row_limit") + result = {} + for row in rows: + if ( + _date(row["tradedate"]) != published + or not isinstance(row["secid"], str) + or not row["secid"] + or row["secid"] in result + ): + raise SnapshotValidationError("conflicting_moex_duration") + result[row["secid"]] = row + return published, result + + +def _parameters(collector, reference): + code = quote(reference["secid"], safe="") + document = collector.read(ISS_BASE + f"securities/{code}.json") + rows = iss_rows(document, "description", {"name", "value"}) + parameters = {} + for row in rows: + if not isinstance(row["name"], str) or row["name"] in parameters: + raise SnapshotValidationError("conflicting_moex_parameters") + parameters[row["name"]] = row["value"] + if parameters.get("SECID") != reference["secid"] or str( + parameters.get("EMITTER_ID") + ) != str(reference["emitent_id"]): + raise SnapshotValidationError("moex_parameter_identity_mismatch") + boards = iss_rows(document, "boards", {"boardid"}) + if reference["primary_boardid"] not in {b["boardid"] for b in boards}: + raise SnapshotValidationError("moex_primary_board_missing") + return parameters, document + + +def _trading(collector, reference, market): + code = quote(reference["secid"], safe="") + document = collector.read( + ISS_BASE + f"engines/stock/markets/{market}/securities/{code}.json" + ) + securities = iss_rows(document, "securities", {"SECID", "BOARDID"}) + observations = iss_rows(document, "marketdata", {"SECID", "BOARDID"}) + versions = iss_rows(document, "dataversion", {"trade_date"}) + if any(r["SECID"] != reference["secid"] for r in securities + observations): + raise SnapshotValidationError("moex_trading_identity_mismatch") + rows = [r for r in observations if r["BOARDID"] == reference["primary_boardid"]] + if len(rows) > 1: + raise SnapshotValidationError("conflicting_moex_trading") + if not rows: + return None, document + if len(versions) != 1: + raise SnapshotValidationError("moex_trading_date_missing") + return { + "traded_on": _date(versions[0]["trade_date"]), + "board_id": reference["primary_boardid"], + "upstream": rows[0], + }, document + + +def _collect(collector, index, artifact, counts): + published, durations = _duration_map(collector) + artifact.source_published_at = date.fromisoformat(published) + seen = set() + for market in ("shares", "bonds"): + for reference in collector.reference(market): + counts["parsed"] += 1 + secid = reference["secid"] + if secid in seen: + raise SnapshotValidationError("conflicting_moex_market_identity") + seen.add(secid) + inn = reference["emitent_inn"] + if not isinstance(inn, str) or not valid_legal_inn(inn): + counts["missing_or_invalid_issuer_inn"] += 1 + continue + organization, reason = index.resolve(inn=inn, ogrn="") + if reason == "outside_directory": + counts["outside_directory"] += 1 + continue + payload = { + "exchange": "moex", + "instrument_id": secid, + "issuer_inn": inn, + "issuer_provider_id": str(reference["emitent_id"]) + if reference["emitent_id"] is not None + else None, + "isin": reference["isin"], + "instrument_kind": reference["type"], + "listing_date": None, + "nominal_value": None, + "nominal_currency": None, + "parameters": {}, + "trading": None, + "duration": None, + "yield": None, + "upstream": {"reference": reference}, + "provenance": { + "artifact_id": str(artifact.uid), + "duration_published_at": published, + }, + } + if not reason: + parameters, raw_parameters = _parameters(collector, reference) + trading, raw_trading = _trading(collector, reference, market) + payload.update( + parameters=parameters, + trading=trading, + nominal_currency=parameters.get("FACEUNIT"), + ) + nominal = parameters.get("FACEVALUE") + if nominal is not None: + payload["nominal_value"] = _metric(nominal, None, None)["value"] + payload["upstream"].update( + parameters=raw_parameters, trading=raw_trading + ) + metrics = durations.get(secid) if market == "bonds" else None + if metrics is not None: + calculated_on = _date(metrics["tradedate"]) + payload.update( + { + "duration": _metric( + metrics["duration"], "days", calculated_on + ), + "yield": _metric( + metrics["effectiveyield"], None, calculated_on + ), + } + ) + payload["upstream"]["duration"] = metrics + payload["provenance"]["yield_kind"] = "effectiveyield" + payload["provenance"]["warnings"] = ["yield_unit_unverified"] + yield SourceRecordInput( + external_id=f"moex:{secid}", + title=reference["name"] or secid, + organization_name=organization.name if organization else "", + organization_uid=organization.uid if organization else None, + inn=inn, + record_type="exchange_instrument", + record_date=payload["trading"]["traded_on"] + if payload["trading"] + else "", + url=ISS_BASE + f'securities/{quote(secid,safe="")}.json', + payload=payload, + ) + + +def iter_moex_records( + *, organization_index: OwnOrganizationIndex, artifact: ParserSourceArtifact +) -> Iterator[SourceRecordInput]: + counts = Counter() + collector = None + complete = False + try: + with tempfile.TemporaryFile() as handle, registry_session() as session: + collector = _Collector(session, handle) + try: + yield from _collect(collector, organization_index, artifact, counts) + complete = True + finally: + save_artifact_file(artifact, handle) + artifact.parsed_count = counts["parsed"] + artifact.metadata = { + **artifact.metadata, + "http_requests": collector.requests, + "diagnostics": dict(counts), + } + artifact.save() + if complete: + artifact.status = ParserSourceArtifact.Status.PARSED + artifact.save(update_fields=("status",)) + except Exception as exc: + reject_snapshot(artifact, exc) + raise diff --git a/tests/apps/parsers/test_exchange_moex.py b/tests/apps/parsers/test_exchange_moex.py new file mode 100644 index 0000000..c71791e --- /dev/null +++ b/tests/apps/parsers/test_exchange_moex.py @@ -0,0 +1,266 @@ +"""Issuer linkage, complete pagination and independent observations.""" +import importlib +import json +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import patch + +import pytest +from apps.parsers.models import ParserSourceArtifact +from apps.parsers.registry_snapshots import ( + OwnOrganizationIndex, + SnapshotValidationError, +) +from django.test import override_settings +from django.utils import timezone +from organizations.models import Organization + +FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges" + + +def adapter(): + return importlib.import_module("apps.parsers.exchange_moex") + + +def block(columns, rows): + return {"columns": columns, "data": rows} + + +@pytest.fixture +def scenario(db, tmp_path): + org = Organization.objects.create( + name="Synthetic issuer", + inn="7707083893", + ogrn="1027700132195", + okpo="00032537", + directory_imported_at=timezone.now(), + ) + wrong = Organization.objects.create( + name="Wrong issuer", + inn="0123456789", + ogrn="1027700132194", + okpo="01234567", + directory_imported_at=timezone.now(), + ) + artifact = ParserSourceArtifact.objects.create( + source="stock_exchange_activity", + original_name="moex.ndjson", + load_batch=1, + metadata={"dataset": "moex", "input_only": True}, + ) + reference = json.loads((FIXTURES / "reference.json").read_text()) + raw = dict( + zip( + reference["securities"]["columns"], + reference["securities"]["data"][0], + strict=False, + ) + ) + raw.update(emitent_inn=org.inn, emitent_id=wrong.inn) + requests = [] + changes = {} + + def response_value(url, params): + if url.endswith("/durations"): + return b"
" + if url.endswith("/durations.json"): + return { + "durations": block( + [ + "tradedate", + "secid", + "effectiveyield", + "duration", + "title", + "matdate", + ], + [ + [ + "2026-10-02", + "FIXTURE1", + changes.get("yield", None), + 260, + "Fixture", + "2027-01-01", + ] + ], + ) + } + if url.endswith("/iss/securities.json"): + rows = ( + [] + if params["start"] or params["market"] == "shares" + else [list(raw.values())] + ) + if changes.get("repeat") and params["market"] == "bonds": + rows = [list(raw.values())] + if changes.get("two") and rows: + other = dict(raw, secid="FIXTURE2") + rows.append(list(other.values())) + value = {"securities": block(list(raw), rows)} + if changes.get("cursor"): + value["securities.cursor"] = block( + ["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]] + ) + return value + if "/iss/securities/" in url: + secid = url.rsplit("/", 1)[1][:-5] + return { + "description": block( + ["name", "value"], + [ + ["SECID", changes.get("secid", secid)], + ["EMITTER_ID", changes.get("issuer", raw["emitent_id"])], + ["FACEVALUE", "1000"], + ["FACEUNIT", "RUB"], + ], + ), + "boards": block(["boardid"], [["TQCB"]]), + } + secid = url.rsplit("/", 1)[1][:-5] + return { + "securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]), + "marketdata": block( + ["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]] + ), + "dataversion": block(["trade_date"], [["2026-10-05"]]), + } + + @contextmanager + def response(session, url, params=None, **kwargs): + requests.append((url, params)) + value = response_value(url, params) + body = value if isinstance(value, bytes) else json.dumps(value).encode() + + class Response: + def iter_content(self, chunk_size): + yield body + + yield Response() + + with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch( + "apps.parsers.exchange_moex.registry_response", response + ): + yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests + + +def collect(scenario): + return list( + adapter().iter_moex_records( + organization_index=scenario[3], artifact=scenario[2] + ) + ) + + +def test_internal_issuer_id_cannot_match_inn(scenario): + rows = collect(scenario) + assert len(rows) == 1 + assert rows[0].organization_uid == scenario[0].uid + assert rows[0].organization_uid != scenario[1].uid + assert rows[0].payload["issuer_provider_id"] == scenario[1].inn + assert rows[0].amount is None + + +def test_all_reference_pages_and_raw_checksums(scenario): + import hashlib + + rows = collect(scenario) + artifact = scenario[2] + assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3 + assert artifact.metadata["http_requests"] == 7 + with artifact.file.open("rb") as f: + entries = [json.loads(line) for line in f] + assert len(entries) == 7 + assert all( + hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"] + for e in entries + ) + assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid) + + +def test_multiple_bonds_one_issuer(scenario): + scenario[4]["two"] = True + assert {r.external_id for r in collect(scenario)} == { + "moex:FIXTURE1", + "moex:FIXTURE2", + } + + +def test_nullable_yield_and_duration_date_separate_from_trading(scenario): + row = collect(scenario)[0] + assert row.payload["duration"] == { + "value": "260", + "unit": "days", + "calculated_on": "2026-10-02", + } + assert row.payload["yield"] == { + "value": None, + "unit": None, + "calculated_on": "2026-10-02", + } + assert row.payload["trading"]["traded_on"] == "2026-10-05" + assert row.payload["trading"]["upstream"]["LAST"] == "101.2300" + + +def test_zero_yield_not_null(scenario): + scenario[4]["yield"] = 0 + assert collect(scenario)[0].payload["yield"]["value"] == "0" + + +@pytest.mark.parametrize( + "key,value", + [("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)], +) +def test_identity_or_incomplete_pagination_rejected(scenario, key, value): + scenario[4][key] = value + with pytest.raises(SnapshotValidationError): + collect(scenario) + + +def test_request_limit_prevents_unbounded_collection(scenario): + with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises( + SnapshotValidationError + ): + collect(scenario) + assert len(scenario[5]) == 1 + + +def test_missing_dated_export_link_rejected(): + with pytest.raises(SnapshotValidationError): + adapter().discover_duration_date("
no date
") + + +def test_unknown_columns_rejected(): + with pytest.raises(SnapshotValidationError): + adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"}) + + +def test_outside_directory_is_counted_without_parameter_requests(scenario): + scenario[0].delete() + scenario[3].by_inn.pop("7707083893") + assert collect(scenario) == [] + assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1 + assert len(scenario[5]) == 5 + + +def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario): + scenario[3].by_inn.clear() + assert collect(scenario) == [] + + +def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario): + scenario[4]["yield"] = "NaN" + with pytest.raises(SnapshotValidationError): + collect(scenario) + assert scenario[2].file.name + scenario[2].refresh_from_db() + assert scenario[2].status == "rejected" + + +def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario): + scenario[4]["yield"] = "15.212400" + row = collect(scenario)[0] + assert row.payload["yield"]["value"] == "15.212400" + assert row.payload["yield"]["unit"] is None + assert row.payload["provenance"]["yield_kind"] == "effectiveyield" + assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]