feat: collect MOEX issuer-linked instruments and metrics
This commit is contained in:
378
src/apps/parsers/exchange_moex.py
Normal file
378
src/apps/parsers/exchange_moex.py
Normal file
@@ -0,0 +1,378 @@
|
||||
"""Bounded ISS collection, exact issuer linkage and dated independent observations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from collections.abc import Iterator
|
||||
from datetime import date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from urllib.parse import quote
|
||||
|
||||
from apps.parsers.fssp_csv import valid_legal_inn
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_http import (
|
||||
limited_bytes,
|
||||
registry_response,
|
||||
registry_session,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import (
|
||||
OwnOrganizationIndex,
|
||||
SnapshotValidationError,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
)
|
||||
from django.conf import settings
|
||||
from organizations.source_ingestion import SourceRecordInput
|
||||
|
||||
ISS_BASE = "https://iss.moex.com/iss/"
|
||||
DURATION_PAGE = "https://www.moex.com/ru/bondization/durations"
|
||||
DURATION_ROUTE = "statistics/engines/stock/markets/bonds/durations.json"
|
||||
MOEX_HOSTS = frozenset({"iss.moex.com", "www.moex.com"})
|
||||
REFERENCE_COLUMNS = {
|
||||
"secid",
|
||||
"emitent_id",
|
||||
"emitent_inn",
|
||||
"name",
|
||||
"isin",
|
||||
"type",
|
||||
"group",
|
||||
"primary_boardid",
|
||||
}
|
||||
|
||||
|
||||
def _strings(value):
|
||||
if isinstance(value, Decimal):
|
||||
return format(value, "f")
|
||||
if isinstance(value, list):
|
||||
return [_strings(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
return {k: _strings(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def _date(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise SnapshotValidationError("missing_moex_observation_date")
|
||||
try:
|
||||
return date.fromisoformat(value).isoformat()
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_moex_observation_date") from exc
|
||||
|
||||
|
||||
def discover_duration_date(html: str) -> str:
|
||||
matches = re.findall(
|
||||
r':init-dt=["\']new Date\([\'\"](\d{4}-\d{2}-\d{2})[\'\"]\)', html
|
||||
)
|
||||
if len(set(matches)) != 1:
|
||||
raise SnapshotValidationError("moex_duration_publication_date_missing")
|
||||
return _date(matches[0])
|
||||
|
||||
|
||||
def iss_rows(document: dict, block: str, required: set[str]) -> list[dict]:
|
||||
value = document.get(block)
|
||||
if not isinstance(value, dict):
|
||||
raise SnapshotValidationError("missing_moex_block")
|
||||
columns = value.get("columns")
|
||||
rows = value.get("data")
|
||||
if (
|
||||
not isinstance(columns, list)
|
||||
or any(not isinstance(c, str) for c in columns)
|
||||
or len(set(columns)) != len(columns)
|
||||
or not required.issubset(columns)
|
||||
or not isinstance(rows, list)
|
||||
):
|
||||
raise SnapshotValidationError("unknown_moex_columns")
|
||||
result = []
|
||||
for row in rows:
|
||||
if not isinstance(row, list) or len(row) != len(columns):
|
||||
raise SnapshotValidationError("invalid_moex_row_width")
|
||||
result.append(dict(zip(columns, row, strict=False)))
|
||||
return result
|
||||
|
||||
|
||||
def _metric(value, unit, calculated_on):
|
||||
number = None
|
||||
if value is not None:
|
||||
try:
|
||||
decimal = Decimal(str(value))
|
||||
if not decimal.is_finite():
|
||||
raise InvalidOperation
|
||||
number = format(decimal, "f")
|
||||
except InvalidOperation as exc:
|
||||
raise SnapshotValidationError("invalid_moex_metric") from exc
|
||||
return {"value": number, "unit": unit, "calculated_on": calculated_on}
|
||||
|
||||
|
||||
class _Collector:
|
||||
def __init__(self, session, handle):
|
||||
self.session = session
|
||||
self.handle = handle
|
||||
self.requests = 0
|
||||
|
||||
def read(self, url, params=None, *, html=False):
|
||||
if self.requests >= getattr(settings, "EXCHANGE_MOEX_MAX_REQUESTS", 10000):
|
||||
raise SnapshotValidationError("moex_request_limit")
|
||||
self.requests += 1
|
||||
with registry_response(
|
||||
self.session, url, hosts=MOEX_HOSTS, params=params
|
||||
) as response:
|
||||
body = limited_bytes(
|
||||
response,
|
||||
4 * 1024**2
|
||||
if html
|
||||
else getattr(
|
||||
settings, "EXCHANGE_MOEX_MAX_RESPONSE_BYTES", 10 * 1024**2
|
||||
),
|
||||
)
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
value = text if html else _strings(json.loads(text, parse_float=Decimal))
|
||||
except (UnicodeError, ValueError) as exc:
|
||||
raise SnapshotValidationError("invalid_moex_response") from exc
|
||||
envelope = {
|
||||
"url": url,
|
||||
"params": params,
|
||||
"sha256": hashlib.sha256(body).hexdigest(),
|
||||
"body_utf8": text,
|
||||
}
|
||||
self.handle.write((json.dumps(envelope, ensure_ascii=False) + "\n").encode())
|
||||
if not html and not isinstance(value, dict):
|
||||
raise SnapshotValidationError("invalid_moex_document")
|
||||
return value
|
||||
|
||||
def reference(self, market):
|
||||
seen = set()
|
||||
total = None
|
||||
for page in range(getattr(settings, "EXCHANGE_MOEX_MAX_PAGES", 1000)):
|
||||
start = page * 100
|
||||
document = self.read(
|
||||
ISS_BASE + "securities.json",
|
||||
{
|
||||
"engine": "stock",
|
||||
"market": market,
|
||||
"is_trading": 1,
|
||||
"limit": 100,
|
||||
"start": start,
|
||||
},
|
||||
)
|
||||
rows = iss_rows(document, "securities", REFERENCE_COLUMNS)
|
||||
if len(rows) > 100:
|
||||
raise SnapshotValidationError("moex_page_too_large")
|
||||
if "securities.cursor" in document:
|
||||
cursors = iss_rows(
|
||||
document, "securities.cursor", {"INDEX", "TOTAL", "PAGESIZE"}
|
||||
)
|
||||
if len(cursors) != 1:
|
||||
raise SnapshotValidationError("invalid_moex_cursor")
|
||||
cursor = cursors[0]
|
||||
if (
|
||||
any(
|
||||
not isinstance(cursor[key], int)
|
||||
or isinstance(cursor[key], bool)
|
||||
for key in ("INDEX", "TOTAL", "PAGESIZE")
|
||||
)
|
||||
or cursor["INDEX"] != start
|
||||
or cursor["PAGESIZE"] != 100
|
||||
or cursor["TOTAL"] < start
|
||||
or (total is not None and total != cursor["TOTAL"])
|
||||
or len(rows) != min(100, cursor["TOTAL"] - start)
|
||||
):
|
||||
raise SnapshotValidationError("incomplete_moex_cursor")
|
||||
total = cursor["TOTAL"]
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
secid = row["secid"]
|
||||
if (
|
||||
not isinstance(secid, str)
|
||||
or not secid
|
||||
or len(secid) > 200
|
||||
or secid in seen
|
||||
):
|
||||
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
|
||||
seen.add(secid)
|
||||
yield row
|
||||
raise SnapshotValidationError("moex_page_limit")
|
||||
|
||||
|
||||
def _duration_map(collector):
|
||||
html = collector.read(DURATION_PAGE, html=True)
|
||||
published = discover_duration_date(html)
|
||||
document = collector.read(ISS_BASE + DURATION_ROUTE, {"date": published})
|
||||
rows = iss_rows(
|
||||
document, "durations", {"tradedate", "secid", "effectiveyield", "duration"}
|
||||
)
|
||||
if len(rows) > getattr(settings, "EXCHANGE_MOEX_MAX_DURATION_ROWS", 100000):
|
||||
raise SnapshotValidationError("moex_duration_row_limit")
|
||||
result = {}
|
||||
for row in rows:
|
||||
if (
|
||||
_date(row["tradedate"]) != published
|
||||
or not isinstance(row["secid"], str)
|
||||
or not row["secid"]
|
||||
or row["secid"] in result
|
||||
):
|
||||
raise SnapshotValidationError("conflicting_moex_duration")
|
||||
result[row["secid"]] = row
|
||||
return published, result
|
||||
|
||||
|
||||
def _parameters(collector, reference):
|
||||
code = quote(reference["secid"], safe="")
|
||||
document = collector.read(ISS_BASE + f"securities/{code}.json")
|
||||
rows = iss_rows(document, "description", {"name", "value"})
|
||||
parameters = {}
|
||||
for row in rows:
|
||||
if not isinstance(row["name"], str) or row["name"] in parameters:
|
||||
raise SnapshotValidationError("conflicting_moex_parameters")
|
||||
parameters[row["name"]] = row["value"]
|
||||
if parameters.get("SECID") != reference["secid"] or str(
|
||||
parameters.get("EMITTER_ID")
|
||||
) != str(reference["emitent_id"]):
|
||||
raise SnapshotValidationError("moex_parameter_identity_mismatch")
|
||||
boards = iss_rows(document, "boards", {"boardid"})
|
||||
if reference["primary_boardid"] not in {b["boardid"] for b in boards}:
|
||||
raise SnapshotValidationError("moex_primary_board_missing")
|
||||
return parameters, document
|
||||
|
||||
|
||||
def _trading(collector, reference, market):
|
||||
code = quote(reference["secid"], safe="")
|
||||
document = collector.read(
|
||||
ISS_BASE + f"engines/stock/markets/{market}/securities/{code}.json"
|
||||
)
|
||||
securities = iss_rows(document, "securities", {"SECID", "BOARDID"})
|
||||
observations = iss_rows(document, "marketdata", {"SECID", "BOARDID"})
|
||||
versions = iss_rows(document, "dataversion", {"trade_date"})
|
||||
if any(r["SECID"] != reference["secid"] for r in securities + observations):
|
||||
raise SnapshotValidationError("moex_trading_identity_mismatch")
|
||||
rows = [r for r in observations if r["BOARDID"] == reference["primary_boardid"]]
|
||||
if len(rows) > 1:
|
||||
raise SnapshotValidationError("conflicting_moex_trading")
|
||||
if not rows:
|
||||
return None, document
|
||||
if len(versions) != 1:
|
||||
raise SnapshotValidationError("moex_trading_date_missing")
|
||||
return {
|
||||
"traded_on": _date(versions[0]["trade_date"]),
|
||||
"board_id": reference["primary_boardid"],
|
||||
"upstream": rows[0],
|
||||
}, document
|
||||
|
||||
|
||||
def _collect(collector, index, artifact, counts):
|
||||
published, durations = _duration_map(collector)
|
||||
artifact.source_published_at = date.fromisoformat(published)
|
||||
seen = set()
|
||||
for market in ("shares", "bonds"):
|
||||
for reference in collector.reference(market):
|
||||
counts["parsed"] += 1
|
||||
secid = reference["secid"]
|
||||
if secid in seen:
|
||||
raise SnapshotValidationError("conflicting_moex_market_identity")
|
||||
seen.add(secid)
|
||||
inn = reference["emitent_inn"]
|
||||
if not isinstance(inn, str) or not valid_legal_inn(inn):
|
||||
counts["missing_or_invalid_issuer_inn"] += 1
|
||||
continue
|
||||
organization, reason = index.resolve(inn=inn, ogrn="")
|
||||
if reason == "outside_directory":
|
||||
counts["outside_directory"] += 1
|
||||
continue
|
||||
payload = {
|
||||
"exchange": "moex",
|
||||
"instrument_id": secid,
|
||||
"issuer_inn": inn,
|
||||
"issuer_provider_id": str(reference["emitent_id"])
|
||||
if reference["emitent_id"] is not None
|
||||
else None,
|
||||
"isin": reference["isin"],
|
||||
"instrument_kind": reference["type"],
|
||||
"listing_date": None,
|
||||
"nominal_value": None,
|
||||
"nominal_currency": None,
|
||||
"parameters": {},
|
||||
"trading": None,
|
||||
"duration": None,
|
||||
"yield": None,
|
||||
"upstream": {"reference": reference},
|
||||
"provenance": {
|
||||
"artifact_id": str(artifact.uid),
|
||||
"duration_published_at": published,
|
||||
},
|
||||
}
|
||||
if not reason:
|
||||
parameters, raw_parameters = _parameters(collector, reference)
|
||||
trading, raw_trading = _trading(collector, reference, market)
|
||||
payload.update(
|
||||
parameters=parameters,
|
||||
trading=trading,
|
||||
nominal_currency=parameters.get("FACEUNIT"),
|
||||
)
|
||||
nominal = parameters.get("FACEVALUE")
|
||||
if nominal is not None:
|
||||
payload["nominal_value"] = _metric(nominal, None, None)["value"]
|
||||
payload["upstream"].update(
|
||||
parameters=raw_parameters, trading=raw_trading
|
||||
)
|
||||
metrics = durations.get(secid) if market == "bonds" else None
|
||||
if metrics is not None:
|
||||
calculated_on = _date(metrics["tradedate"])
|
||||
payload.update(
|
||||
{
|
||||
"duration": _metric(
|
||||
metrics["duration"], "days", calculated_on
|
||||
),
|
||||
"yield": _metric(
|
||||
metrics["effectiveyield"], None, calculated_on
|
||||
),
|
||||
}
|
||||
)
|
||||
payload["upstream"]["duration"] = metrics
|
||||
payload["provenance"]["yield_kind"] = "effectiveyield"
|
||||
payload["provenance"]["warnings"] = ["yield_unit_unverified"]
|
||||
yield SourceRecordInput(
|
||||
external_id=f"moex:{secid}",
|
||||
title=reference["name"] or secid,
|
||||
organization_name=organization.name if organization else "",
|
||||
organization_uid=organization.uid if organization else None,
|
||||
inn=inn,
|
||||
record_type="exchange_instrument",
|
||||
record_date=payload["trading"]["traded_on"]
|
||||
if payload["trading"]
|
||||
else "",
|
||||
url=ISS_BASE + f'securities/{quote(secid,safe="")}.json',
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
|
||||
def iter_moex_records(
|
||||
*, organization_index: OwnOrganizationIndex, artifact: ParserSourceArtifact
|
||||
) -> Iterator[SourceRecordInput]:
|
||||
counts = Counter()
|
||||
collector = None
|
||||
complete = False
|
||||
try:
|
||||
with tempfile.TemporaryFile() as handle, registry_session() as session:
|
||||
collector = _Collector(session, handle)
|
||||
try:
|
||||
yield from _collect(collector, organization_index, artifact, counts)
|
||||
complete = True
|
||||
finally:
|
||||
save_artifact_file(artifact, handle)
|
||||
artifact.parsed_count = counts["parsed"]
|
||||
artifact.metadata = {
|
||||
**artifact.metadata,
|
||||
"http_requests": collector.requests,
|
||||
"diagnostics": dict(counts),
|
||||
}
|
||||
artifact.save()
|
||||
if complete:
|
||||
artifact.status = ParserSourceArtifact.Status.PARSED
|
||||
artifact.save(update_fields=("status",))
|
||||
except Exception as exc:
|
||||
reject_snapshot(artifact, exc)
|
||||
raise
|
||||
266
tests/apps/parsers/test_exchange_moex.py
Normal file
266
tests/apps/parsers/test_exchange_moex.py
Normal file
@@ -0,0 +1,266 @@
|
||||
"""Issuer linkage, complete pagination and independent observations."""
|
||||
import importlib
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_snapshots import (
|
||||
OwnOrganizationIndex,
|
||||
SnapshotValidationError,
|
||||
)
|
||||
from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization
|
||||
|
||||
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
||||
|
||||
|
||||
def adapter():
|
||||
return importlib.import_module("apps.parsers.exchange_moex")
|
||||
|
||||
|
||||
def block(columns, rows):
|
||||
return {"columns": columns, "data": rows}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scenario(db, tmp_path):
|
||||
org = Organization.objects.create(
|
||||
name="Synthetic issuer",
|
||||
inn="7707083893",
|
||||
ogrn="1027700132195",
|
||||
okpo="00032537",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
wrong = Organization.objects.create(
|
||||
name="Wrong issuer",
|
||||
inn="0123456789",
|
||||
ogrn="1027700132194",
|
||||
okpo="01234567",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source="stock_exchange_activity",
|
||||
original_name="moex.ndjson",
|
||||
load_batch=1,
|
||||
metadata={"dataset": "moex", "input_only": True},
|
||||
)
|
||||
reference = json.loads((FIXTURES / "reference.json").read_text())
|
||||
raw = dict(
|
||||
zip(
|
||||
reference["securities"]["columns"],
|
||||
reference["securities"]["data"][0],
|
||||
strict=False,
|
||||
)
|
||||
)
|
||||
raw.update(emitent_inn=org.inn, emitent_id=wrong.inn)
|
||||
requests = []
|
||||
changes = {}
|
||||
|
||||
def response_value(url, params):
|
||||
if url.endswith("/durations"):
|
||||
return b"<div :init-dt=\"new Date('2026-10-02')\"></div>"
|
||||
if url.endswith("/durations.json"):
|
||||
return {
|
||||
"durations": block(
|
||||
[
|
||||
"tradedate",
|
||||
"secid",
|
||||
"effectiveyield",
|
||||
"duration",
|
||||
"title",
|
||||
"matdate",
|
||||
],
|
||||
[
|
||||
[
|
||||
"2026-10-02",
|
||||
"FIXTURE1",
|
||||
changes.get("yield", None),
|
||||
260,
|
||||
"Fixture",
|
||||
"2027-01-01",
|
||||
]
|
||||
],
|
||||
)
|
||||
}
|
||||
if url.endswith("/iss/securities.json"):
|
||||
rows = (
|
||||
[]
|
||||
if params["start"] or params["market"] == "shares"
|
||||
else [list(raw.values())]
|
||||
)
|
||||
if changes.get("repeat") and params["market"] == "bonds":
|
||||
rows = [list(raw.values())]
|
||||
if changes.get("two") and rows:
|
||||
other = dict(raw, secid="FIXTURE2")
|
||||
rows.append(list(other.values()))
|
||||
value = {"securities": block(list(raw), rows)}
|
||||
if changes.get("cursor"):
|
||||
value["securities.cursor"] = block(
|
||||
["INDEX", "TOTAL", "PAGESIZE"], [[params["start"], 1000, 100]]
|
||||
)
|
||||
return value
|
||||
if "/iss/securities/" in url:
|
||||
secid = url.rsplit("/", 1)[1][:-5]
|
||||
return {
|
||||
"description": block(
|
||||
["name", "value"],
|
||||
[
|
||||
["SECID", changes.get("secid", secid)],
|
||||
["EMITTER_ID", changes.get("issuer", raw["emitent_id"])],
|
||||
["FACEVALUE", "1000"],
|
||||
["FACEUNIT", "RUB"],
|
||||
],
|
||||
),
|
||||
"boards": block(["boardid"], [["TQCB"]]),
|
||||
}
|
||||
secid = url.rsplit("/", 1)[1][:-5]
|
||||
return {
|
||||
"securities": block(["SECID", "BOARDID"], [[secid, "TQCB"]]),
|
||||
"marketdata": block(
|
||||
["SECID", "BOARDID", "LAST"], [[secid, "TQCB", "101.2300"]]
|
||||
),
|
||||
"dataversion": block(["trade_date"], [["2026-10-05"]]),
|
||||
}
|
||||
|
||||
@contextmanager
|
||||
def response(session, url, params=None, **kwargs):
|
||||
requests.append((url, params))
|
||||
value = response_value(url, params)
|
||||
body = value if isinstance(value, bytes) else json.dumps(value).encode()
|
||||
|
||||
class Response:
|
||||
def iter_content(self, chunk_size):
|
||||
yield body
|
||||
|
||||
yield Response()
|
||||
|
||||
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch(
|
||||
"apps.parsers.exchange_moex.registry_response", response
|
||||
):
|
||||
yield org, wrong, artifact, OwnOrganizationIndex(), changes, requests
|
||||
|
||||
|
||||
def collect(scenario):
|
||||
return list(
|
||||
adapter().iter_moex_records(
|
||||
organization_index=scenario[3], artifact=scenario[2]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_internal_issuer_id_cannot_match_inn(scenario):
|
||||
rows = collect(scenario)
|
||||
assert len(rows) == 1
|
||||
assert rows[0].organization_uid == scenario[0].uid
|
||||
assert rows[0].organization_uid != scenario[1].uid
|
||||
assert rows[0].payload["issuer_provider_id"] == scenario[1].inn
|
||||
assert rows[0].amount is None
|
||||
|
||||
|
||||
def test_all_reference_pages_and_raw_checksums(scenario):
|
||||
import hashlib
|
||||
|
||||
rows = collect(scenario)
|
||||
artifact = scenario[2]
|
||||
assert len([q for u, q in scenario[5] if u.endswith("/iss/securities.json")]) == 3
|
||||
assert artifact.metadata["http_requests"] == 7
|
||||
with artifact.file.open("rb") as f:
|
||||
entries = [json.loads(line) for line in f]
|
||||
assert len(entries) == 7
|
||||
assert all(
|
||||
hashlib.sha256(e["body_utf8"].encode()).hexdigest() == e["sha256"]
|
||||
for e in entries
|
||||
)
|
||||
assert rows[0].payload["provenance"]["artifact_id"] == str(artifact.uid)
|
||||
|
||||
|
||||
def test_multiple_bonds_one_issuer(scenario):
|
||||
scenario[4]["two"] = True
|
||||
assert {r.external_id for r in collect(scenario)} == {
|
||||
"moex:FIXTURE1",
|
||||
"moex:FIXTURE2",
|
||||
}
|
||||
|
||||
|
||||
def test_nullable_yield_and_duration_date_separate_from_trading(scenario):
|
||||
row = collect(scenario)[0]
|
||||
assert row.payload["duration"] == {
|
||||
"value": "260",
|
||||
"unit": "days",
|
||||
"calculated_on": "2026-10-02",
|
||||
}
|
||||
assert row.payload["yield"] == {
|
||||
"value": None,
|
||||
"unit": None,
|
||||
"calculated_on": "2026-10-02",
|
||||
}
|
||||
assert row.payload["trading"]["traded_on"] == "2026-10-05"
|
||||
assert row.payload["trading"]["upstream"]["LAST"] == "101.2300"
|
||||
|
||||
|
||||
def test_zero_yield_not_null(scenario):
|
||||
scenario[4]["yield"] = 0
|
||||
assert collect(scenario)[0].payload["yield"]["value"] == "0"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key,value",
|
||||
[("issuer", "different"), ("secid", "WRONG"), ("repeat", True), ("cursor", True)],
|
||||
)
|
||||
def test_identity_or_incomplete_pagination_rejected(scenario, key, value):
|
||||
scenario[4][key] = value
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
collect(scenario)
|
||||
|
||||
|
||||
def test_request_limit_prevents_unbounded_collection(scenario):
|
||||
with override_settings(EXCHANGE_MOEX_MAX_REQUESTS=1), pytest.raises(
|
||||
SnapshotValidationError
|
||||
):
|
||||
collect(scenario)
|
||||
assert len(scenario[5]) == 1
|
||||
|
||||
|
||||
def test_missing_dated_export_link_rejected():
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
adapter().discover_duration_date("<div>no date</div>")
|
||||
|
||||
|
||||
def test_unknown_columns_rejected():
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
|
||||
|
||||
|
||||
def test_outside_directory_is_counted_without_parameter_requests(scenario):
|
||||
scenario[0].delete()
|
||||
scenario[3].by_inn.pop("7707083893")
|
||||
assert collect(scenario) == []
|
||||
assert scenario[2].metadata["diagnostics"]["outside_directory"] == 1
|
||||
assert len(scenario[5]) == 5
|
||||
|
||||
|
||||
def test_unknown_numeric_issuer_is_never_an_inn_fallback(scenario):
|
||||
scenario[3].by_inn.clear()
|
||||
assert collect(scenario) == []
|
||||
|
||||
|
||||
def test_nonfinite_yield_rejected_and_raw_artifact_retained(scenario):
|
||||
scenario[4]["yield"] = "NaN"
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
collect(scenario)
|
||||
assert scenario[2].file.name
|
||||
scenario[2].refresh_from_db()
|
||||
assert scenario[2].status == "rejected"
|
||||
|
||||
|
||||
def test_unverified_yield_units_explicit_and_decimal_precision_preserved(scenario):
|
||||
scenario[4]["yield"] = "15.212400"
|
||||
row = collect(scenario)[0]
|
||||
assert row.payload["yield"]["value"] == "15.212400"
|
||||
assert row.payload["yield"]["unit"] is None
|
||||
assert row.payload["provenance"]["yield_kind"] == "effectiveyield"
|
||||
assert row.payload["provenance"]["warnings"] == ["yield_unit_unverified"]
|
||||
Reference in New Issue
Block a user