feat: collect MOEX issuer-linked instruments and metrics
This commit is contained in:
378
src/apps/parsers/exchange_moex.py
Normal file
378
src/apps/parsers/exchange_moex.py
Normal file
@@ -0,0 +1,378 @@
|
|||||||
|
"""Bounded ISS collection, exact issuer linkage and dated independent observations."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
from collections import Counter
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from datetime import date
|
||||||
|
from decimal import Decimal, InvalidOperation
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from apps.parsers.fssp_csv import valid_legal_inn
|
||||||
|
from apps.parsers.models import ParserSourceArtifact
|
||||||
|
from apps.parsers.registry_http import (
|
||||||
|
limited_bytes,
|
||||||
|
registry_response,
|
||||||
|
registry_session,
|
||||||
|
)
|
||||||
|
from apps.parsers.registry_snapshots import (
|
||||||
|
OwnOrganizationIndex,
|
||||||
|
SnapshotValidationError,
|
||||||
|
reject_snapshot,
|
||||||
|
save_artifact_file,
|
||||||
|
)
|
||||||
|
from django.conf import settings
|
||||||
|
from organizations.source_ingestion import SourceRecordInput
|
||||||
|
|
||||||
|
ISS_BASE = "https://iss.moex.com/iss/"
|
||||||
|
DURATION_PAGE = "https://www.moex.com/ru/bondization/durations"
|
||||||
|
DURATION_ROUTE = "statistics/engines/stock/markets/bonds/durations.json"
|
||||||
|
MOEX_HOSTS = frozenset({"iss.moex.com", "www.moex.com"})
|
||||||
|
REFERENCE_COLUMNS = {
|
||||||
|
"secid",
|
||||||
|
"emitent_id",
|
||||||
|
"emitent_inn",
|
||||||
|
"name",
|
||||||
|
"isin",
|
||||||
|
"type",
|
||||||
|
"group",
|
||||||
|
"primary_boardid",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _strings(value):
|
||||||
|
if isinstance(value, Decimal):
|
||||||
|
return format(value, "f")
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [_strings(v) for v in value]
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k: _strings(v) for k, v in value.items()}
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _date(value) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise SnapshotValidationError("missing_moex_observation_date")
|
||||||
|
try:
|
||||||
|
return date.fromisoformat(value).isoformat()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise SnapshotValidationError("invalid_moex_observation_date") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def discover_duration_date(html: str) -> str:
|
||||||
|
matches = re.findall(
|
||||||
|
r':init-dt=["\']new Date\([\'\"](\d{4}-\d{2}-\d{2})[\'\"]\)', html
|
||||||
|
)
|
||||||
|
if len(set(matches)) != 1:
|
||||||
|
raise SnapshotValidationError("moex_duration_publication_date_missing")
|
||||||
|
return _date(matches[0])
|
||||||
|
|
||||||
|
|
||||||
|
def iss_rows(document: dict, block: str, required: set[str]) -> list[dict]:
|
||||||
|
value = document.get(block)
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise SnapshotValidationError("missing_moex_block")
|
||||||
|
columns = value.get("columns")
|
||||||
|
rows = value.get("data")
|
||||||
|
if (
|
||||||
|
not isinstance(columns, list)
|
||||||
|
or any(not isinstance(c, str) for c in columns)
|
||||||
|
or len(set(columns)) != len(columns)
|
||||||
|
or not required.issubset(columns)
|
||||||
|
or not isinstance(rows, list)
|
||||||
|
):
|
||||||
|
raise SnapshotValidationError("unknown_moex_columns")
|
||||||
|
result = []
|
||||||
|
for row in rows:
|
||||||
|
if not isinstance(row, list) or len(row) != len(columns):
|
||||||
|
raise SnapshotValidationError("invalid_moex_row_width")
|
||||||
|
result.append(dict(zip(columns, row, strict=False)))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _metric(value, unit, calculated_on):
|
||||||
|
number = None
|
||||||
|
if value is not None:
|
||||||
|
try:
|
||||||
|
decimal = Decimal(str(value))
|
||||||
|
if not decimal.is_finite():
|
||||||
|
raise InvalidOperation
|
||||||
|
number = format(decimal, "f")
|
||||||
|
except InvalidOperation as exc:
|
||||||
|
raise SnapshotValidationError("invalid_moex_metric") from exc
|
||||||
|
return {"value": number, "unit": unit, "calculated_on": calculated_on}
|
||||||
|
|
||||||
|
|
||||||
|
class _Collector:
|
||||||
|
def __init__(self, session, handle):
|
||||||
|
self.session = session
|
||||||
|
self.handle = handle
|
||||||
|
self.requests = 0
|
||||||
|
|
||||||
|
def read(self, url, params=None, *, html=False):
|
||||||
|
if self.requests >= getattr(settings, "EXCHANGE_MOEX_MAX_REQUESTS", 10000):
|
||||||
|
raise SnapshotValidationError("moex_request_limit")
|
||||||
|
self.requests += 1
|
||||||
|
with registry_response(
|
||||||
|
self.session, url, hosts=MOEX_HOSTS, params=params
|
||||||
|
) as response:
|
||||||
|
body = limited_bytes(
|
||||||
|
response,
|
||||||
|
4 * 1024**2
|
||||||
|
if html
|
||||||
|
else getattr(
|
||||||
|
settings, "EXCHANGE_MOEX_MAX_RESPONSE_BYTES", 10 * 1024**2
|
||||||
|
),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
text = body.decode("utf-8")
|
||||||
|
value = text if html else _strings(json.loads(text, parse_float=Decimal))
|
||||||
|
except (UnicodeError, ValueError) as exc:
|
||||||
|
raise SnapshotValidationError("invalid_moex_response") from exc
|
||||||
|
envelope = {
|
||||||
|
"url": url,
|
||||||
|
"params": params,
|
||||||
|
"sha256": hashlib.sha256(body).hexdigest(),
|
||||||
|
"body_utf8": text,
|
||||||
|
}
|
||||||
|
self.handle.write((json.dumps(envelope, ensure_ascii=False) + "\n").encode())
|
||||||
|
if not html and not isinstance(value, dict):
|
||||||
|
raise SnapshotValidationError("invalid_moex_document")
|
||||||
|
return value
|
||||||
|
|
||||||
|
def reference(self, market):
|
||||||
|
seen = set()
|
||||||
|
total = None
|
||||||
|
for page in range(getattr(settings, "EXCHANGE_MOEX_MAX_PAGES", 1000)):
|
||||||
|
start = page * 100
|
||||||
|
document = self.read(
|
||||||
|
ISS_BASE + "securities.json",
|
||||||
|
{
|
||||||
|
"engine": "stock",
|
||||||
|
"market": market,
|
||||||
|
"is_trading": 1,
|
||||||
|
"limit": 100,
|
||||||
|
"start": start,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
rows = iss_rows(document, "securities", REFERENCE_COLUMNS)
|
||||||
|
if len(rows) > 100:
|
||||||
|
raise SnapshotValidationError("moex_page_too_large")
|
||||||
|
if "securities.cursor" in document:
|
||||||
|
cursors = iss_rows(
|
||||||
|
document, "securities.cursor", {"INDEX", "TOTAL", "PAGESIZE"}
|
||||||
|
)
|
||||||
|
if len(cursors) != 1:
|
||||||
|
raise SnapshotValidationError("invalid_moex_cursor")
|
||||||
|
cursor = cursors[0]
|
||||||
|
if (
|
||||||
|
any(
|
||||||
|
not isinstance(cursor[key], int)
|
||||||
|
or isinstance(cursor[key], bool)
|
||||||
|
for key in ("INDEX", "TOTAL", "PAGESIZE")
|
||||||
|
)
|
||||||
|
or cursor["INDEX"] != start
|
||||||
|
or cursor["PAGESIZE"] != 100
|
||||||
|
or cursor["TOTAL"] < start
|
||||||
|
or (total is not None and total != cursor["TOTAL"])
|
||||||
|
or len(rows) != min(100, cursor["TOTAL"] - start)
|
||||||
|
):
|
||||||
|
raise SnapshotValidationError("incomplete_moex_cursor")
|
||||||
|
total = cursor["TOTAL"]
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
for row in rows:
|
||||||
|
secid = row["secid"]
|
||||||
|
if (
|
||||||
|
not isinstance(secid, str)
|
||||||
|
or not secid
|
||||||
|
or len(secid) > 200
|
||||||
|
or secid in seen
|
||||||
|
):
|
||||||
|
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
|
||||||
|
seen.add(secid)
|
||||||
|
yield row
|
||||||
|
raise SnapshotValidationError("moex_page_limit")
|
||||||
|
|
||||||
|
|
||||||
|
def _duration_map(collector):
|
||||||
|
html = collector.read(DURATION_PAGE, html=True)
|
||||||
|
published = discover_duration_date(html)
|
||||||
|
document = collector.read(ISS_BASE + DURATION_ROUTE, {"date": published})
|
||||||
|
rows = iss_rows(
|
||||||
|
document, "durations", {"tradedate", "secid", "effectiveyield", "duration"}
|
||||||
|
)
|
||||||
|
if len(rows) > getattr(settings, "EXCHANGE_MOEX_MAX_DURATION_ROWS", 100000):
|
||||||
|
raise SnapshotValidationError("moex_duration_row_limit")
|
||||||
|
result = {}
|
||||||
|
for row in rows:
|
||||||
|
if (
|
||||||
|
_date(row["tradedate"]) != published
|
||||||
|
or not isinstance(row["secid"], str)
|
||||||
|
or not row["secid"]
|
||||||
|
or row["secid"] in result
|
||||||
|
):
|
||||||
|
raise SnapshotValidationError("conflicting_moex_duration")
|
||||||
|
result[row["secid"]] = row
|
||||||
|
return published, result
|
||||||
|
|
||||||
|
|
||||||
|
def _parameters(collector, reference):
|
||||||
|
code = quote(reference["secid"], safe="")
|
||||||
|
document = collector.read(ISS_BASE + f"securities/{code}.json")
|
||||||
|
rows = iss_rows(document, "description", {"name", "value"})
|
||||||
|
parameters = {}
|
||||||
|
for row in rows:
|
||||||
|
if not isinstance(row["name"], str) or row["name"] in parameters:
|
||||||
|
raise SnapshotValidationError("conflicting_moex_parameters")
|
||||||
|
parameters[row["name"]] = row["value"]
|
||||||
|
if parameters.get("SECID") != reference["secid"] or str(
|
||||||
|
parameters.get("EMITTER_ID")
|
||||||
|
) != str(reference["emitent_id"]):
|
||||||
|
raise SnapshotValidationError("moex_parameter_identity_mismatch")
|
||||||
|
boards = iss_rows(document, "boards", {"boardid"})
|
||||||
|
if reference["primary_boardid"] not in {b["boardid"] for b in boards}:
|
||||||
|
raise SnapshotValidationError("moex_primary_board_missing")
|
||||||
|
return parameters, document
|
||||||
|
|
||||||
|
|
||||||
|
def _trading(collector, reference, market):
|
||||||
|
code = quote(reference["secid"], safe="")
|
||||||
|
document = collector.read(
|
||||||
|
ISS_BASE + f"engines/stock/markets/{market}/securities/{code}.json"
|
||||||
|
)
|
||||||
|
securities = iss_rows(document, "securities", {"SECID", "BOARDID"})
|
||||||
|
observations = iss_rows(document, "marketdata", {"SECID", "BOARDID"})
|
||||||
|
versions = iss_rows(document, "dataversion", {"trade_date"})
|
||||||
|
if any(r["SECID"] != reference["secid"] for r in securities + observations):
|
||||||
|
raise SnapshotValidationError("moex_trading_identity_mismatch")
|
||||||
|
rows = [r for r in observations if r["BOARDID"] == reference["primary_boardid"]]
|
||||||
|
if len(rows) > 1:
|
||||||
|
raise SnapshotValidationError("conflicting_moex_trading")
|
||||||
|
if not rows:
|
||||||
|
return None, document
|
||||||
|
if len(versions) != 1:
|
||||||
|
raise SnapshotValidationError("moex_trading_date_missing")
|
||||||
|
return {
|
||||||
|
"traded_on": _date(versions[0]["trade_date"]),
|
||||||
|
"board_id": reference["primary_boardid"],
|
||||||
|
"upstream": rows[0],
|
||||||
|
}, document
|
||||||
|
|
||||||
|
|
||||||
|
def _collect(collector, index, artifact, counts):
|
||||||
|
published, durations = _duration_map(collector)
|
||||||
|
artifact.source_published_at = date.fromisoformat(published)
|
||||||
|
seen = set()
|
||||||
|
for market in ("shares", "bonds"):
|
||||||
|
for reference in collector.reference(market):
|
||||||
|
counts["parsed"] += 1
|
||||||
|
secid = reference["secid"]
|
||||||
|
if secid in seen:
|
||||||
|
raise SnapshotValidationError("conflicting_moex_market_identity")
|
||||||
|
seen.add(secid)
|
||||||
|
inn = reference["emitent_inn"]
|
||||||
|
if not isinstance(inn, str) or not valid_legal_inn(inn):
|
||||||
|
counts["missing_or_invalid_issuer_inn"] += 1
|
||||||
|
continue
|
||||||
|
organization, reason = index.resolve(inn=inn, ogrn="")
|
||||||
|
if reason == "outside_directory":
|
||||||
|
counts["outside_directory"] += 1
|
||||||
|
continue
|
||||||
|
payload = {
|
||||||
|
"exchange": "moex",
|
||||||
|
"instrument_id": secid,
|
||||||
|
"issuer_inn": inn,
|
||||||
|
"issuer_provider_id": str(reference["emitent_id"])
|
||||||
|
if reference["emitent_id"] is not None
|
||||||
|
else None,
|
||||||
|
"isin": reference["isin"],
|
||||||
|
"instrument_kind": reference["type"],
|
||||||
|
"listing_date": None,
|
||||||
|
"nominal_value": None,
|
||||||
|
"nominal_currency": None,
|
||||||
|
"parameters": {},
|
||||||
|
"trading": None,
|
||||||
|
"duration": None,
|
||||||
|
"yield": None,
|
||||||
|
"upstream": {"reference": reference},
|
||||||
|
"provenance": {
|
||||||
|
"artifact_id": str(artifact.uid),
|
||||||
|
"duration_published_at": published,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if not reason:
|
||||||
|
parameters, raw_parameters = _parameters(collector, reference)
|
||||||
|
trading, raw_trading = _trading(collector, reference, market)
|
||||||
|
payload.update(
|
||||||
|
parameters=parameters,
|
||||||
|
trading=trading,
|
||||||
|
nominal_currency=parameters.get("FACEUNIT"),
|
||||||
|
)
|
||||||
|
nominal = parameters.get("FACEVALUE")
|
||||||
|
if nominal is not None:
|
||||||
|
payload["nominal_value"] = _metric(nominal, None, None)["value"]
|
||||||
|
payload["upstream"].update(
|
||||||
|
parameters=raw_parameters, trading=raw_trading
|
||||||
|
)
|
||||||
|
metrics = durations.get(secid) if market == "bonds" else None
|
||||||
|
if metrics is not None:
|
||||||
|
calculated_on = _date(metrics["tradedate"])
|
||||||
|
payload.update(
|
||||||
|
{
|
||||||
|
"duration": _metric(
|
||||||
|
metrics["duration"], "days", calculated_on
|
||||||
|
),
|
||||||
|
"yield": _metric(
|
||||||
|
metrics["effectiveyield"], None, calculated_on
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
payload["upstream"]["duration"] = metrics
|
||||||
|
payload["provenance"]["yield_kind"] = "effectiveyield"
|
||||||
|
payload["provenance"]["warnings"] = ["yield_unit_unverified"]
|
||||||
|
yield SourceRecordInput(
|
||||||
|
external_id=f"moex:{secid}",
|
||||||
|
title=reference["name"] or secid,
|
||||||
|
organization_name=organization.name if organization else "",
|
||||||
|
organization_uid=organization.uid if organization else None,
|
||||||
|
inn=inn,
|
||||||
|
record_type="exchange_instrument",
|
||||||
|
record_date=payload["trading"]["traded_on"]
|
||||||
|
if payload["trading"]
|
||||||
|
else "",
|
||||||
|
url=ISS_BASE + f'securities/{quote(secid,safe="")}.json',
|
||||||
|
payload=payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def iter_moex_records(
|
||||||
|
*, organization_index: OwnOrganizationIndex, artifact: ParserSourceArtifact
|
||||||
|
) -> Iterator[SourceRecordInput]:
|
||||||
|
counts = Counter()
|
||||||
|
collector = None
|
||||||
|
complete = False
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryFile() as handle, registry_session() as session:
|
||||||
|
collector = _Collector(session, handle)
|
||||||
|
try:
|
||||||
|
yield from _collect(collector, organization_index, artifact, counts)
|
||||||
|
complete = True
|
||||||
|
finally:
|
||||||
|
save_artifact_file(artifact, handle)
|
||||||
|
artifact.parsed_count = counts["parsed"]
|
||||||
|
artifact.metadata = {
|
||||||
|
**artifact.metadata,
|
||||||
|
"http_requests": collector.requests,
|
||||||
|
"diagnostics": dict(counts),
|
||||||
|
}
|
||||||
|
artifact.save()
|
||||||
|
if complete:
|
||||||
|
artifact.status = ParserSourceArtifact.Status.PARSED
|
||||||
|
artifact.save(update_fields=("status",))
|
||||||
|
except Exception as exc:
|
||||||
|
reject_snapshot(artifact, exc)
|
||||||
|
raise
|
||||||
266
tests/apps/parsers/test_exchange_moex.py
Normal file
266
tests/apps/parsers/test_exchange_moex.py
Normal file
@@ -0,0 +1,266 @@
|
|||||||
|
"""Issuer linkage, complete pagination and independent observations."""
|
||||||
|
import importlib
|
||||||
|
import json
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from apps.parsers.models import ParserSourceArtifact
|
||||||
|
from apps.parsers.registry_snapshots import (
|
||||||
|
OwnOrganizationIndex,
|
||||||
|
SnapshotValidationError,
|
||||||
|
)
|
||||||
|
from django.test import override_settings
|
||||||
|
from django.utils import timezone
|
||||||
|
from organizations.models import Organization
|
||||||
|
|
||||||
|
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
||||||
|
|
||||||
|
|
||||||
|
def adapter():
|
||||||
|
return importlib.import_module("apps.parsers.exchange_moex")
|
||||||
|
|
||||||
|
|
||||||
|
def block(columns, rows):
|
||||||
|
return {"columns": columns, "data": rows}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def scenario(db, tmp_path):
|
||||||
|
org = Organization.objects.create(
|
||||||
|
name="Synthetic issuer",
|
||||||
|
inn="7707083893",
|
||||||
|
ogrn="1027700132195",
|
||||||
|
okpo="00032537",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
wrong = Organization.objects.create(
|
||||||
|
name="Wrong issuer",
|
||||||
|
inn="0123456789",
|
||||||
|
ogrn="1027700132194",
|
||||||
|
okpo="01234567",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
artifact = ParserSourceArtifact.objects.create(
|
||||||
|
source="stock_exchange_activity",
|
||||||
|
original_name="moex.ndjson",
|
||||||
|
load_batch=1,
|
||||||
|
metadata={"dataset": "moex", "input_only": True},
|
||||||
|
)
|
||||||
|
reference = json.loads((FIXTURES / "reference.json").read_text())
|
||||||
|
raw = dict(
|
||||||
|
zip(
|
||||||
|
reference["securities"]["columns"],
|
||||||
|
reference["securities"]["data"][0],
|
||||||
|
strict=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
raw.update(emitent_inn=org.inn, emitent_id=wrong.inn)
|
||||||
|
requests = []
|
||||||
|
changes = {}
|
||||||
|
|
||||||
|
def response_value(url, params):
|
||||||
|
if url.endswith("/durations"):
|
||||||
|
return b"<div :init-dt=\"new Date('2026-10-02')\"></div>"
|
||||||
|
if url.endswith("/durations.json"):
|
||||||
|
return {
|
||||||
|
"durations": block(
|
||||||
|
[
|
||||||
|
"tradedate",
|
||||||
|
"secid",
|
||||||
|
"effectiveyield",
|
||||||
|
"duration",
|
||||||
|
"title",
|
||||||
|
"matdate",
|
||||||
|
],
|
||||||
|
[
|
||||||
|
[
|
||||||
|
"2026-10-02",
|
||||||
|
"FIXTURE1",
|
||||||
|
changes.get("yield", None),
|
||||||
|
260,
|
||||||
|
"Fixture",
|
||||||
|
"2027-01-01",
|
||||||
|
]
|
||||||
|
],
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if url.endswith("/iss/securities.json"):
|
||||||
|
rows = (
|
||||||
|
[]
|
||||||
|
if params["start"] or params["market"] == "shares"
|
||||||
|
else [list(raw.values())]
|
||||||
|
)
|
||||||
|
if changes.get("repeat") and params["market"] == "bonds":
|
||||||
|
rows = [list(raw.values())]
|
||||||
|
if changes.get("two") and rows:
|
||||||
|
other = dict(raw, secid="FIXTURE2")
|
||||||
|
rows.append(list(other.values()))
|
||||||
|
value = {"securities": block(list(raw), rows)}
|
||||||
|
if changes.get("cursor"):
|
||||||
|
value["securities.cursor"] = block(
|
||||||
|
["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]]
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
if "/iss/securities/" in url:
|
||||||
|
secid = url.rsplit("/", 1)[1][:-5]
|
||||||
|
return {
|
||||||
|
"description": block(
|
||||||
|
["name", "value"],
|
||||||
|
[
|
||||||
|
["SECID", changes.get("secid", secid)],
|
||||||
|
["EMITTER_ID", changes.get("issuer", raw["emitent_id"])],
|
||||||
|
["FACEVALUE", "1000"],
|
||||||
|
["FACEUNIT", "RUB"],
|
||||||
|
],
|
||||||
|
),
|
||||||
|
"boards": block(["boardid"], [["TQCB"]]),
|
||||||
|
}
|
||||||
|
secid = url.rsplit("/", 1)[1][:-5]
|
||||||
|
return {
|
||||||
|
"securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]),
|
||||||
|
"marketdata": block(
|
||||||
|
["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]]
|
||||||
|
),
|
||||||
|
"dataversion": block(["trade_date"], [["2026-10-05"]]),
|
||||||
|
}
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def response(session, url, params=None, **kwargs):
|
||||||
|
requests.append((url, params))
|
||||||
|
value = response_value(url, params)
|
||||||
|
body = value if isinstance(value, bytes) else json.dumps(value).encode()
|
||||||
|
|
||||||
|
class Response:
|
||||||
|
def iter_content(self, chunk_size):
|
||||||
|
yield body
|
||||||
|
|
||||||
|
yield Response()
|
||||||
|
|
||||||
|
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch(
|
||||||
|
"apps.parsers.exchange_moex.registry_response", response
|
||||||
|
):
|
||||||
|
yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests
|
||||||
|
|
||||||
|
|
||||||
|
def collect(scenario):
|
||||||
|
return list(
|
||||||
|
adapter().iter_moex_records(
|
||||||
|
organization_index=scenario[3], artifact=scenario[2]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_internal_issuer_id_cannot_match_inn(scenario):
|
||||||
|
rows = collect(scenario)
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0].organization_uid == scenario[0].uid
|
||||||
|
assert rows[0].organization_uid != scenario[1].uid
|
||||||
|
assert rows[0].payload["issuer_provider_id"] == scenario[1].inn
|
||||||
|
assert rows[0].amount is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_reference_pages_and_raw_checksums(scenario):
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
rows = collect(scenario)
|
||||||
|
artifact = scenario[2]
|
||||||
|
assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3
|
||||||
|
assert artifact.metadata["http_requests"] == 7
|
||||||
|
with artifact.file.open("rb") as f:
|
||||||
|
entries = [json.loads(line) for line in f]
|
||||||
|
assert len(entries) == 7
|
||||||
|
assert all(
|
||||||
|
hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"]
|
||||||
|
for e in entries
|
||||||
|
)
|
||||||
|
assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid)
|
||||||
|
|
||||||
|
|
||||||
|
def test_multiple_bonds_one_issuer(scenario):
|
||||||
|
scenario[4]["two"] = True
|
||||||
|
assert {r.external_id for r in collect(scenario)} == {
|
||||||
|
"moex:FIXTURE1",
|
||||||
|
"moex:FIXTURE2",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_nullable_yield_and_duration_date_separate_from_trading(scenario):
|
||||||
|
row = collect(scenario)[0]
|
||||||
|
assert row.payload["duration"] == {
|
||||||
|
"value": "260",
|
||||||
|
"unit": "days",
|
||||||
|
"calculated_on": "2026-10-02",
|
||||||
|
}
|
||||||
|
assert row.payload["yield"] == {
|
||||||
|
"value": None,
|
||||||
|
"unit": None,
|
||||||
|
"calculated_on": "2026-10-02",
|
||||||
|
}
|
||||||
|
assert row.payload["trading"]["traded_on"] == "2026-10-05"
|
||||||
|
assert row.payload["trading"]["upstream"]["LAST"] == "101.2300"
|
||||||
|
|
||||||
|
|
||||||
|
def test_zero_yield_not_null(scenario):
|
||||||
|
scenario[4]["yield"] = 0
|
||||||
|
assert collect(scenario)[0].payload["yield"]["value"] == "0"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"key,value",
|
||||||
|
[("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)],
|
||||||
|
)
|
||||||
|
def test_identity_or_incomplete_pagination_rejected(scenario, key, value):
|
||||||
|
scenario[4][key] = value
|
||||||
|
with pytest.raises(SnapshotValidationError):
|
||||||
|
collect(scenario)
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_limit_prevents_unbounded_collection(scenario):
|
||||||
|
with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises(
|
||||||
|
SnapshotValidationError
|
||||||
|
):
|
||||||
|
collect(scenario)
|
||||||
|
assert len(scenario[5]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_dated_export_link_rejected():
|
||||||
|
with pytest.raises(SnapshotValidationError):
|
||||||
|
adapter().discover_duration_date("<div>no date</div>")
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_columns_rejected():
|
||||||
|
with pytest.raises(SnapshotValidationError):
|
||||||
|
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_outside_directory_is_counted_without_parameter_requests(scenario):
|
||||||
|
scenario[0].delete()
|
||||||
|
scenario[3].by_inn.pop("7707083893")
|
||||||
|
assert collect(scenario) == []
|
||||||
|
assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1
|
||||||
|
assert len(scenario[5]) == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario):
|
||||||
|
scenario[3].by_inn.clear()
|
||||||
|
assert collect(scenario) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario):
|
||||||
|
scenario[4]["yield"] = "NaN"
|
||||||
|
with pytest.raises(SnapshotValidationError):
|
||||||
|
collect(scenario)
|
||||||
|
assert scenario[2].file.name
|
||||||
|
scenario[2].refresh_from_db()
|
||||||
|
assert scenario[2].status == "rejected"
|
||||||
|
|
||||||
|
|
||||||
|
def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario):
|
||||||
|
scenario[4]["yield"] = "15.212400"
|
||||||
|
row = collect(scenario)[0]
|
||||||
|
assert row.payload["yield"]["value"] == "15.212400"
|
||||||
|
assert row.payload["yield"]["unit"] is None
|
||||||
|
assert row.payload["provenance"]["yield_kind"] == "effectiveyield"
|
||||||
|
assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]
|
||||||
Reference in New Issue
Block a user