feat: collect MOEX issuer-linked instruments and metrics

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:09:07 +02:00
parent 084965b16d
commit e2cf384853
2 changed files with 644 additions and 0 deletions

View File

@@ -0,0 +1,378 @@
"""Bounded ISS collection, exact issuer linkage and dated independent observations."""
from __future__ import annotations
import hashlib
import json
import re
import tempfile
from collections import Counter
from collections.abc import Iterator
from datetime import date
from decimal import Decimal, InvalidOperation
from urllib.parse import quote
from apps.parsers.fssp_csv import valid_legal_inn
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_http import (
limited_bytes,
registry_response,
registry_session,
)
from apps.parsers.registry_snapshots import (
OwnOrganizationIndex,
SnapshotValidationError,
reject_snapshot,
save_artifact_file,
)
from django.conf import settings
from organizations.source_ingestion import SourceRecordInput
ISS_BASE = "https://iss.moex.com/iss/"
DURATION_PAGE = "https://www.moex.com/ru/bondization/durations"
DURATION_ROUTE = "statistics/engines/stock/markets/bonds/durations.json"
MOEX_HOSTS = frozenset({"iss.moex.com", "www.moex.com"})
REFERENCE_COLUMNS = {
"secid",
"emitent_id",
"emitent_inn",
"name",
"isin",
"type",
"group",
"primary_boardid",
}
def _strings(value):
if isinstance(value, Decimal):
return format(value, "f")
if isinstance(value, list):
return [_strings(v) for v in value]
if isinstance(value, dict):
return {k: _strings(v) for k, v in value.items()}
return value
def _date(value) -> str:
if not isinstance(value, str):
raise SnapshotValidationError("missing_moex_observation_date")
try:
return date.fromisoformat(value).isoformat()
except ValueError as exc:
raise SnapshotValidationError("invalid_moex_observation_date") from exc
def discover_duration_date(html: str) -> str:
matches = re.findall(
r':init-dt=["\']new Date\([\'\"](\d{4}-\d{2}-\d{2})[\'\"]\)', html
)
if len(set(matches)) != 1:
raise SnapshotValidationError("moex_duration_publication_date_missing")
return _date(matches[0])
def iss_rows(document: dict, block: str, required: set[str]) -> list[dict]:
value = document.get(block)
if not isinstance(value, dict):
raise SnapshotValidationError("missing_moex_block")
columns = value.get("columns")
rows = value.get("data")
if (
not isinstance(columns, list)
or any(not isinstance(c, str) for c in columns)
or len(set(columns)) != len(columns)
or not required.issubset(columns)
or not isinstance(rows, list)
):
raise SnapshotValidationError("unknown_moex_columns")
result = []
for row in rows:
if not isinstance(row, list) or len(row) != len(columns):
raise SnapshotValidationError("invalid_moex_row_width")
result.append(dict(zip(columns, row, strict=False)))
return result
def _metric(value, unit, calculated_on):
number = None
if value is not None:
try:
decimal = Decimal(str(value))
if not decimal.is_finite():
raise InvalidOperation
number = format(decimal, "f")
except InvalidOperation as exc:
raise SnapshotValidationError("invalid_moex_metric") from exc
return {"value": number, "unit": unit, "calculated_on": calculated_on}
class _Collector:
def __init__(self, session, handle):
self.session = session
self.handle = handle
self.requests = 0
def read(self, url, params=None, *, html=False):
if self.requests >= getattr(settings, "EXCHANGE_MOEX_MAX_REQUESTS", 10000):
raise SnapshotValidationError("moex_request_limit")
self.requests += 1
with registry_response(
self.session, url, hosts=MOEX_HOSTS, params=params
) as response:
body = limited_bytes(
response,
4 * 1024**2
if html
else getattr(
settings, "EXCHANGE_MOEX_MAX_RESPONSE_BYTES", 10 * 1024**2
),
)
try:
text = body.decode("utf-8")
value = text if html else _strings(json.loads(text, parse_float=Decimal))
except (UnicodeError, ValueError) as exc:
raise SnapshotValidationError("invalid_moex_response") from exc
envelope = {
"url": url,
"params": params,
"sha256": hashlib.sha256(body).hexdigest(),
"body_utf8": text,
}
self.handle.write((json.dumps(envelope, ensure_ascii=False) + "\n").encode())
if not html and not isinstance(value, dict):
raise SnapshotValidationError("invalid_moex_document")
return value
def reference(self, market):
seen = set()
total = None
for page in range(getattr(settings, "EXCHANGE_MOEX_MAX_PAGES", 1000)):
start = page * 100
document = self.read(
ISS_BASE + "securities.json",
{
"engine": "stock",
"market": market,
"is_trading": 1,
"limit": 100,
"start": start,
},
)
rows = iss_rows(document, "securities", REFERENCE_COLUMNS)
if len(rows) > 100:
raise SnapshotValidationError("moex_page_too_large")
if "securities.cursor" in document:
cursors = iss_rows(
document, "securities.cursor", {"INDEX", "TOTAL", "PAGESIZE"}
)
if len(cursors) != 1:
raise SnapshotValidationError("invalid_moex_cursor")
cursor = cursors[0]
if (
any(
not isinstance(cursor[key], int)
or isinstance(cursor[key], bool)
for key in ("INDEX", "TOTAL", "PAGESIZE")
)
or cursor["INDEX"] != start
or cursor["PAGESIZE"] != 100
or cursor["TOTAL"] < start
or (total is not None and total != cursor["TOTAL"])
or len(rows) != min(100, cursor["TOTAL"] - start)
):
raise SnapshotValidationError("incomplete_moex_cursor")
total = cursor["TOTAL"]
if not rows:
return
for row in rows:
secid = row["secid"]
if (
not isinstance(secid, str)
or not secid
or len(secid) > 200
or secid in seen
):
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
seen.add(secid)
yield row
raise SnapshotValidationError("moex_page_limit")
def _duration_map(collector):
html = collector.read(DURATION_PAGE, html=True)
published = discover_duration_date(html)
document = collector.read(ISS_BASE + DURATION_ROUTE, {"date": published})
rows = iss_rows(
document, "durations", {"tradedate", "secid", "effectiveyield", "duration"}
)
if len(rows) > getattr(settings, "EXCHANGE_MOEX_MAX_DURATION_ROWS", 100000):
raise SnapshotValidationError("moex_duration_row_limit")
result = {}
for row in rows:
if (
_date(row["tradedate"]) != published
or not isinstance(row["secid"], str)
or not row["secid"]
or row["secid"] in result
):
raise SnapshotValidationError("conflicting_moex_duration")
result[row["secid"]] = row
return published, result
def _parameters(collector, reference):
code = quote(reference["secid"], safe="")
document = collector.read(ISS_BASE + f"securities/{code}.json")
rows = iss_rows(document, "description", {"name", "value"})
parameters = {}
for row in rows:
if not isinstance(row["name"], str) or row["name"] in parameters:
raise SnapshotValidationError("conflicting_moex_parameters")
parameters[row["name"]] = row["value"]
if parameters.get("SECID") != reference["secid"] or str(
parameters.get("EMITTER_ID")
) != str(reference["emitent_id"]):
raise SnapshotValidationError("moex_parameter_identity_mismatch")
boards = iss_rows(document, "boards", {"boardid"})
if reference["primary_boardid"] not in {b["boardid"] for b in boards}:
raise SnapshotValidationError("moex_primary_board_missing")
return parameters, document
def _trading(collector, reference, market):
code = quote(reference["secid"], safe="")
document = collector.read(
ISS_BASE + f"engines/stock/markets/{market}/securities/{code}.json"
)
securities = iss_rows(document, "securities", {"SECID", "BOARDID"})
observations = iss_rows(document, "marketdata", {"SECID", "BOARDID"})
versions = iss_rows(document, "dataversion", {"trade_date"})
if any(r["SECID"] != reference["secid"] for r in securities + observations):
raise SnapshotValidationError("moex_trading_identity_mismatch")
rows = [r for r in observations if r["BOARDID"] == reference["primary_boardid"]]
if len(rows) > 1:
raise SnapshotValidationError("conflicting_moex_trading")
if not rows:
return None, document
if len(versions) != 1:
raise SnapshotValidationError("moex_trading_date_missing")
return {
"traded_on": _date(versions[0]["trade_date"]),
"board_id": reference["primary_boardid"],
"upstream": rows[0],
}, document
def _collect(collector, index, artifact, counts):
published, durations = _duration_map(collector)
artifact.source_published_at = date.fromisoformat(published)
seen = set()
for market in ("shares", "bonds"):
for reference in collector.reference(market):
counts["parsed"] += 1
secid = reference["secid"]
if secid in seen:
raise SnapshotValidationError("conflicting_moex_market_identity")
seen.add(secid)
inn = reference["emitent_inn"]
if not isinstance(inn, str) or not valid_legal_inn(inn):
counts["missing_or_invalid_issuer_inn"] += 1
continue
organization, reason = index.resolve(inn=inn, ogrn="")
if reason == "outside_directory":
counts["outside_directory"] += 1
continue
payload = {
"exchange": "moex",
"instrument_id": secid,
"issuer_inn": inn,
"issuer_provider_id": str(reference["emitent_id"])
if reference["emitent_id"] is not None
else None,
"isin": reference["isin"],
"instrument_kind": reference["type"],
"listing_date": None,
"nominal_value": None,
"nominal_currency": None,
"parameters": {},
"trading": None,
"duration": None,
"yield": None,
"upstream": {"reference": reference},
"provenance": {
"artifact_id": str(artifact.uid),
"duration_published_at": published,
},
}
if not reason:
parameters, raw_parameters = _parameters(collector, reference)
trading, raw_trading = _trading(collector, reference, market)
payload.update(
parameters=parameters,
trading=trading,
nominal_currency=parameters.get("FACEUNIT"),
)
nominal = parameters.get("FACEVALUE")
if nominal is not None:
payload["nominal_value"] = _metric(nominal, None, None)["value"]
payload["upstream"].update(
parameters=raw_parameters, trading=raw_trading
)
metrics = durations.get(secid) if market == "bonds" else None
if metrics is not None:
calculated_on = _date(metrics["tradedate"])
payload.update(
{
"duration": _metric(
metrics["duration"], "days", calculated_on
),
"yield": _metric(
metrics["effectiveyield"], None, calculated_on
),
}
)
payload["upstream"]["duration"] = metrics
payload["provenance"]["yield_kind"] = "effectiveyield"
payload["provenance"]["warnings"] = ["yield_unit_unverified"]
yield SourceRecordInput(
external_id=f"moex:{secid}",
title=reference["name"] or secid,
organization_name=organization.name if organization else "",
organization_uid=organization.uid if organization else None,
inn=inn,
record_type="exchange_instrument",
record_date=payload["trading"]["traded_on"]
if payload["trading"]
else "",
url=ISS_BASE + f'securities/{quote(secid,safe="")}.json',
payload=payload,
)
def iter_moex_records(
*, organization_index: OwnOrganizationIndex, artifact: ParserSourceArtifact
) -> Iterator[SourceRecordInput]:
counts = Counter()
collector = None
complete = False
try:
with tempfile.TemporaryFile() as handle, registry_session() as session:
collector = _Collector(session, handle)
try:
yield from _collect(collector, organization_index, artifact, counts)
complete = True
finally:
save_artifact_file(artifact, handle)
artifact.parsed_count = counts["parsed"]
artifact.metadata = {
**artifact.metadata,
"http_requests": collector.requests,
"diagnostics": dict(counts),
}
artifact.save()
if complete:
artifact.status = ParserSourceArtifact.Status.PARSED
artifact.save(update_fields=("status",))
except Exception as exc:
reject_snapshot(artifact, exc)
raise