feat: collect MOEX issuer-linked instruments and metrics
This commit is contained in:
378
src/apps/parsers/exchange_moex.py
Normal file
378
src/apps/parsers/exchange_moex.py
Normal file
@@ -0,0 +1,378 @@
|
||||
"""Bounded ISS collection, exact issuer linkage and dated independent observations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from urllib.parse import quote
|
||||
|
||||
from apps.parsers.fssp_csv import valid_legal_inn
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_http import (
|
||||
limited_bytes,
|
||||
registry_response,
|
||||
registry_session,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import (
|
||||
OwnOrganizationIndex,
|
||||
SnapshotValidationError,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
)
|
||||
from django.conf import settings
|
||||
from organizations.source_ingestion import SourceRecordInput
|
||||
|
||||
ISS_BASE = "https://iss.moex.com/iss/"
|
||||
DURATION_PAGE = "https://www.moex.com/ru/bondization/durations"
|
||||
DURATION_ROUTE = "statistics/engines/stock/markets/bonds/durations.json"
|
||||
MOEX_HOSTS = frozenset({"iss.moex.com", "www.moex.com"})
|
||||
REFERENCE_COLUMNS = {
|
||||
"secid",
|
||||
"emitent_id",
|
||||
"emitent_inn",
|
||||
"name",
|
||||
"isin",
|
||||
"type",
|
||||
"group",
|
||||
"primary_boardid",
|
||||
}
|
||||
|
||||
|
||||
def _strings(value):
|
||||
if isinstance(value, Decimal):
|
||||
return format(value, "f")
|
||||
if isinstance(value, list):
|
||||
return [_strings(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: _strings(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _date(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise SnapshotValidationError("missing_moex_observation_date")
|
||||
try:
|
||||
return date.fromisoformat(value).isoformat()
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_moex_observation_date") from exc
|
||||
|
||||
|
||||
def discover_duration_date(html: str) -> str:
|
||||
matches = re.findall(
|
||||
r':init-dt=["\']new Date\([\'\"](\d{4}-\d{2}-\d{2})[\'\"]\)', html
|
||||
)
|
||||
if len(set(matches)) != 1:
|
||||
raise SnapshotValidationError("moex_duration_publication_date_missing")
|
||||
return _date(matches[0])
|
||||
|
||||
|
||||
def iss_rows(document: dict, block: str, required: set[str]) -> list[dict]:
|
||||
value = document.get(block)
|
||||
if not isinstance(value, dict):
|
||||
raise SnapshotValidationError("missing_moex_block")
|
||||
columns = value.get("columns")
|
||||
rows = value.get("data")
|
||||
if (
|
||||
not isinstance(columns, list)
|
||||
or any(not isinstance(c, str) for c in columns)
|
||||
or len(set(columns)) != len(columns)
|
||||
or not required.issubset(columns)
|
||||
or not isinstance(rows, list)
|
||||
):
|
||||
raise SnapshotValidationError("unknown_moex_columns")
|
||||
result = []
|
||||
for row in rows:
|
||||
if not isinstance(row, list) or len(row) != len(columns):
|
||||
raise SnapshotValidationError("invalid_moex_row_width")
|
||||
result.append(dict(zip(columns, row, strict=False)))
|
||||
return result
|
||||
|
||||
|
||||
def _metric(value, unit, calculated_on):
|
||||
number = None
|
||||
if value is not None:
|
||||
try:
|
||||
decimal = Decimal(str(value))
|
||||
if not decimal.is_finite():
|
||||
raise InvalidOperation
|
||||
number = format(decimal, "f")
|
||||
except InvalidOperation as exc:
|
||||
raise SnapshotValidationError("invalid_moex_metric") from exc
|
||||
return {"value": number, "unit": unit, "calculated_on": calculated_on}
|
||||
|
||||
|
||||
class _Collector:
|
||||
def __init__(self, session, handle):
|
||||
self.session = session
|
||||
self.handle = handle
|
||||
self.requests = 0
|
||||
|
||||
def read(self, url, params=None, *, html=False):
|
||||
if self.requests >= getattr(settings, "EXCHANGE_MOEX_MAX_REQUESTS", 10000):
|
||||
raise SnapshotValidationError("moex_request_limit")
|
||||
self.requests += 1
|
||||
with registry_response(
|
||||
self.session, url, hosts=MOEX_HOSTS, params=params
|
||||
) as response:
|
||||
body = limited_bytes(
|
||||
response,
|
||||
4 * 1024**2
|
||||
if html
|
||||
else getattr(
|
||||
settings, "EXCHANGE_MOEX_MAX_RESPONSE_BYTES", 10 * 1024**2
|
||||
),
|
||||
)
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
value = text if html else _strings(json.loads(text, parse_float=Decimal))
|
||||
except (UnicodeError, ValueError) as exc:
|
||||
raise SnapshotValidationError("invalid_moex_response") from exc
|
||||
envelope = {
|
||||
"url": url,
|
||||
"params": params,
|
||||
"sha256": hashlib.sha256(body).hexdigest(),
|
||||
"body_utf8": text,
|
||||
}
|
||||
self.handle.write((json.dumps(envelope, ensure_ascii=False) + "\n").encode())
|
||||
if not html and not isinstance(value, dict):
|
||||
raise SnapshotValidationError("invalid_moex_document")
|
||||
return value
|
||||
|
||||
def reference(self, market):
|
||||
seen = set()
|
||||
total = None
|
||||
for page in range(getattr(settings, "EXCHANGE_MOEX_MAX_PAGES", 1000)):
|
||||
start = page * 100
|
||||
document = self.read(
|
||||
ISS_BASE + "securities.json",
|
||||
{
|
||||
"engine": "stock",
|
||||
"market": market,
|
||||
"is_trading": 1,
|
||||
"limit": 100,
|
||||
"start": start,
|
||||
},
|
||||
)
|
||||
rows = iss_rows(document, "securities", REFERENCE_COLUMNS)
|
||||
if len(rows) > 100:
|
||||
raise SnapshotValidationError("moex_page_too_large")
|
||||
if "securities.cursor" in document:
|
||||
cursors = iss_rows(
|
||||
document, "securities.cursor", {"INDEX", "TOTAL", "PAGESIZE"}
|
||||
)
|
||||
if len(cursors) != 1:
|
||||
raise SnapshotValidationError("invalid_moex_cursor")
|
||||
cursor = cursors[0]
|
||||
if (
|
||||
any(
|
||||
not isinstance(cursor[key], int)
|
||||
or isinstance(cursor[key], bool)
|
||||
for key in ("INDEX", "TOTAL", "PAGESIZE")
|
||||
)
|
||||
or cursor["INDEX"] != start
|
||||
or cursor["PAGESIZE"] != 100
|
||||
or cursor["TOTAL"] < start
|
||||
or (total is not None and total != cursor["TOTAL"])
|
||||
or len(rows) != min(100, cursor["TOTAL"] - start)
|
||||
):
|
||||
raise SnapshotValidationError("incomplete_moex_cursor")
|
||||
total = cursor["TOTAL"]
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
secid = row["secid"]
|
||||
if (
|
||||
not isinstance(secid, str)
|
||||
or not secid
|
||||
or len(secid) > 200
|
||||
or secid in seen
|
||||
):
|
||||
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
|
||||
seen.add(secid)
|
||||
yield row
|
||||
raise SnapshotValidationError("moex_page_limit")
|
||||
|
||||
|
||||
def _duration_map(collector):
|
||||
html = collector.read(DURATION_PAGE, html=True)
|
||||
published = discover_duration_date(html)
|
||||
document = collector.read(ISS_BASE + DURATION_ROUTE, {"date": published})
|
||||
rows = iss_rows(
|
||||
document, "durations", {"tradedate", "secid", "effectiveyield", "duration"}
|
||||
)
|
||||
if len(rows) > getattr(settings, "EXCHANGE_MOEX_MAX_DURATION_ROWS", 100000):
|
||||
raise SnapshotValidationError("moex_duration_row_limit")
|
||||
result = {}
|
||||
for row in rows:
|
||||
if (
|
||||
_date(row["tradedate"]) != published
|
||||
or not isinstance(row["secid"], str)
|
||||
or not row["secid"]
|
||||
or row["secid"] in result
|
||||
):
|
||||
raise SnapshotValidationError("conflicting_moex_duration")
|
||||
result[row["secid"]] = row
|
||||
return published, result
|
||||
|
||||
|
||||
def _parameters(collector, reference):
|
||||
code = quote(reference["secid"], safe="")
|
||||
document = collector.read(ISS_BASE + f"securities/{code}.json")
|
||||
rows = iss_rows(document, "description", {"name", "value"})
|
||||
parameters = {}
|
||||
for row in rows:
|
||||
if not isinstance(row["name"], str) or row["name"] in parameters:
|
||||
raise SnapshotValidationError("conflicting_moex_parameters")
|
||||
parameters[row["name"]] = row["value"]
|
||||
if parameters.get("SECID") != reference["secid"] or str(
|
||||
parameters.get("EMITTER_ID")
|
||||
) != str(reference["emitent_id"]):
|
||||
raise SnapshotValidationError("moex_parameter_identity_mismatch")
|
||||
boards = iss_rows(document, "boards", {"boardid"})
|
||||
if reference["primary_boardid"] not in {b["boardid"] for b in boards}:
|
||||
raise SnapshotValidationError("moex_primary_board_missing")
|
||||
return parameters, document
|
||||
|
||||
|
||||
def _trading(collector, reference, market):
|
||||
code = quote(reference["secid"], safe="")
|
||||
document = collector.read(
|
||||
ISS_BASE + f"engines/stock/markets/{market}/securities/{code}.json"
|
||||
)
|
||||
securities = iss_rows(document, "securities", {"SECID", "BOARDID"})
|
||||
observations = iss_rows(document, "marketdata", {"SECID", "BOARDID"})
|
||||
versions = iss_rows(document, "dataversion", {"trade_date"})
|
||||
if any(r["SECID"] != reference["secid"] for r in securities + observations):
|
||||
raise SnapshotValidationError("moex_trading_identity_mismatch")
|
||||
rows = [r for r in observations if r["BOARDID"] == reference["primary_boardid"]]
|
||||
if len(rows) > 1:
|
||||
raise SnapshotValidationError("conflicting_moex_trading")
|
||||
if not rows:
|
||||
return None, document
|
||||
if len(versions) != 1:
|
||||
raise SnapshotValidationError("moex_trading_date_missing")
|
||||
return {
|
||||
"traded_on": _date(versions[0]["trade_date"]),
|
||||
"board_id": reference["primary_boardid"],
|
||||
"upstream": rows[0],
|
||||
}, document
|
||||
|
||||
|
||||
def _collect(collector, index, artifact, counts):
|
||||
published, durations = _duration_map(collector)
|
||||
artifact.source_published_at = date.fromisoformat(published)
|
||||
seen = set()
|
||||
for market in ("shares", "bonds"):
|
||||
for reference in collector.reference(market):
|
||||
counts["parsed"] += 1
|
||||
secid = reference["secid"]
|
||||
if secid in seen:
|
||||
raise SnapshotValidationError("conflicting_moex_market_identity")
|
||||
seen.add(secid)
|
||||
inn = reference["emitent_inn"]
|
||||
if not isinstance(inn, str) or not valid_legal_inn(inn):
|
||||
counts["missing_or_invalid_issuer_inn"] += 1
|
||||
continue
|
||||
organization, reason = index.resolve(inn=inn, ogrn="")
|
||||
if reason == "outside_directory":
|
||||
counts["outside_directory"] += 1
|
||||
continue
|
||||
payload = {
|
||||
"exchange": "moex",
|
||||
"instrument_id": secid,
|
||||
"issuer_inn": inn,
|
||||
"issuer_provider_id": str(reference["emitent_id"])
|
||||
if reference["emitent_id"] is not None
|
||||
else None,
|
||||
"isin": reference["isin"],
|
||||
"instrument_kind": reference["type"],
|
||||
"listing_date": None,
|
||||
"nominal_value": None,
|
||||
"nominal_currency": None,
|
||||
"parameters": {},
|
||||
"trading": None,
|
||||
"duration": None,
|
||||
"yield": None,
|
||||
"upstream": {"reference": reference},
|
||||
"provenance": {
|
||||
"artifact_id": str(artifact.uid),
|
||||
"duration_published_at": published,
|
||||
},
|
||||
}
|
||||
if not reason:
|
||||
parameters, raw_parameters = _parameters(collector, reference)
|
||||
trading, raw_trading = _trading(collector, reference, market)
|
||||
payload.update(
|
||||
parameters=parameters,
|
||||
trading=trading,
|
||||
nominal_currency=parameters.get("FACEUNIT"),
|
||||
)
|
||||
nominal = parameters.get("FACEVALUE")
|
||||
if nominal is not None:
|
||||
payload["nominal_value"] = _metric(nominal, None, None)["value"]
|
||||
payload["upstream"].update(
|
||||
parameters=raw_parameters, trading=raw_trading
|
||||
)
|
||||
metrics = durations.get(secid) if market == "bonds" else None
|
||||
if metrics is not None:
|
||||
calculated_on = _date(metrics["tradedate"])
|
||||
payload.update(
|
||||
{
|
||||
"duration": _metric(
|
||||
metrics["duration"], "days", calculated_on
|
||||
),
|
||||
"yield": _metric(
|
||||
metrics["effectiveyield"], None, calculated_on
|
||||
),
|
||||
}
|
||||
)
|
||||
payload["upstream"]["duration"] = metrics
|
||||
payload["provenance"]["yield_kind"] = "effectiveyield"
|
||||
payload["provenance"]["warnings"] = ["yield_unit_unverified"]
|
||||
yield SourceRecordInput(
|
||||
external_id=f"moex:{secid}",
|
||||
title=reference["name"] or secid,
|
||||
organization_name=organization.name if organization else "",
|
||||
organization_uid=organization.uid if organization else None,
|
||||
inn=inn,
|
||||
record_type="exchange_instrument",
|
||||
record_date=payload["trading"]["traded_on"]
|
||||
if payload["trading"]
|
||||
else "",
|
||||
url=ISS_BASE + f'securities/{quote(secid,safe="")}.json',
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
|
||||
def iter_moex_records(
|
||||
*, organization_index: OwnOrganizationIndex, artifact: ParserSourceArtifact
|
||||
) -> Iterator[SourceRecordInput]:
|
||||
counts = Counter()
|
||||
collector = None
|
||||
complete = False
|
||||
try:
|
||||
with tempfile.TemporaryFile() as handle, registry_session() as session:
|
||||
collector = _Collector(session, handle)
|
||||
try:
|
||||
yield from _collect(collector, organization_index, artifact, counts)
|
||||
complete = True
|
||||
finally:
|
||||
save_artifact_file(artifact, handle)
|
||||
artifact.parsed_count = counts["parsed"]
|
||||
artifact.metadata = {
|
||||
**artifact.metadata,
|
||||
"http_requests": collector.requests,
|
||||
"diagnostics": dict(counts),
|
||||
}
|
||||
artifact.save()
|
||||
if complete:
|
||||
artifact.status = ParserSourceArtifact.Status.PARSED
|
||||
artifact.save(update_fields=("status",))
|
||||
except Exception as exc:
|
||||
reject_snapshot(artifact, exc)
|
||||
raise
|
||||
Reference in New Issue
Block a user