Files
mostovik-backend/src/apps/parsers/exchange_spb.py
Aleksandr Meshchryakov 1a82345ab3
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m3s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
fix: accept nullable message publishers and scoped dev SPB trust
2026-10-05 19:23:03 +02:00

207 lines
7.7 KiB
Python

"""Strict streaming reader for the verified current SPB public CSV export."""
from __future__ import annotations
import csv
import re
from collections.abc import Iterator
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from io import BytesIO, TextIOWrapper
from typing import BinaryIO
from urllib.parse import urljoin, urlparse
from apps.parsers.clients.common.structured import EIS_CA_BUNDLE_PATH
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_http import (
limited_bytes,
registry_response,
registry_session,
)
from apps.parsers.registry_snapshots import (
SnapshotValidationError,
reject_snapshot,
save_artifact_file,
)
from bs4 import BeautifulSoup
from django.conf import settings
from organizations.source_ingestion import SourceRecordInput
EXCHANGE_SOURCE = "stock_exchange_activity"
SPB_INDEX = "https://spbexchange.ru/listing/securities/"
SPB_LIST = "https://spbexchange.ru/listing/securities/list/"
SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export"
SPB_HOSTS = frozenset({"spbexchange.ru"})
SPB_HEADERS = (
"Row",
"s_level_name",
"e_full_name",
"e_INN_code",
"s_sec_type_name_dop",
"s_sec_form_name_full",
"s_RTS_code",
"s_ISIN_code",
"si_gos_reg_num",
"si_gos_reg_date",
"s_face_value",
"s_face_value_currency",
"s_quot_list_in_date",
"s_segment",
"s_date_defolt",
"s_date_technic_defolt",
"f_is_exchanged",
"s_is_org_discl_restriction",
)
def discover_spb_date(html: str) -> date:
dates = []
for link in BeautifulSoup(html, "html.parser").select("a[href]"):
url = urlparse(urljoin(SPB_INDEX, link["href"]))
match = re.fullmatch(
r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})",
link.get_text(" ", strip=True),
)
if (
match
and url.scheme == "https"
and url.hostname == "spbexchange.ru"
and url.path.rstrip("/") == "/listing/securities/list"
and not url.username
and not url.password
and url.port in (None, 443)
):
try:
dates.append(datetime.strptime(match[1], "%d.%m.%Y").date())
except ValueError as exc:
raise SnapshotValidationError("invalid_spb_publication_date") from exc
if not dates:
raise SnapshotValidationError("spb_current_listing_missing")
return max(dates)
def _date(value: str) -> str | None:
if not value or value == "-":
return None
try:
return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat()
except ValueError as exc:
raise SnapshotValidationError("invalid_spb_date") from exc
def _nominal(value: str) -> str | None:
if value in ("", "-", "без номинальной стоимости"):
return None
try:
number = Decimal(value.replace(",", "."))
if not number.is_finite():
raise InvalidOperation
return format(number, "f")
except InvalidOperation as exc:
raise SnapshotValidationError("invalid_spb_nominal") from exc
def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]:
text = TextIOWrapper(handle, encoding="utf-8-sig", newline="")
try:
reader = csv.DictReader(text, delimiter=";", strict=True)
if (
not reader.fieldnames
or len(reader.fieldnames) != len(SPB_HEADERS)
or set(reader.fieldnames) != set(SPB_HEADERS)
):
raise SnapshotValidationError("unknown_spb_headers")
for number, raw in enumerate(reader, 1):
if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000):
raise SnapshotValidationError("spb_row_limit")
if None in raw or any(value is None for value in raw.values()):
raise SnapshotValidationError("invalid_spb_row_width")
fields = {k: v.strip() for k, v in raw.items()}
code = fields["s_RTS_code"]
if not code or len(code) > 200 or not fields["e_full_name"]:
raise SnapshotValidationError("missing_spb_instrument_identity")
listing_date = _date(fields["s_quot_list_in_date"])
payload = {
"exchange": "spb",
"instrument_id": code,
"issuer_inn": fields["e_INN_code"],
"issuer_provider_id": None,
"isin": fields["s_ISIN_code"] or None,
"instrument_kind": fields["s_sec_type_name_dop"],
"listing_date": listing_date,
"nominal_value": _nominal(fields["s_face_value"]),
"nominal_currency": fields["s_face_value_currency"] or None,
"parameters": {
"registration_number": fields["si_gos_reg_num"] or None,
"registered_on": _date(fields["si_gos_reg_date"]),
"listing_level": fields["s_level_name"],
"category": fields["s_sec_form_name_full"],
},
"trading": None,
"duration": None,
"yield": None,
"upstream": raw,
"provenance": {"row_number": number},
}
yield SourceRecordInput(
external_id=f"spb:{code}",
title=code,
organization_name=fields["e_full_name"],
inn=fields["e_INN_code"],
record_type="exchange_instrument",
record_date=listing_date or "",
url=SPB_LIST,
payload=payload,
)
except (UnicodeError, csv.Error) as exc:
raise SnapshotValidationError("invalid_spb_csv") from exc
finally:
text.detach()
def download_spb_artifact(
*, load_batch: int, uploaded_by_id: int | None = None
) -> ParserSourceArtifact:
artifact = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="spb-listing.csv",
content_type="text/csv",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT},
)
try:
with registry_session() as session:
# This session serves only allowlisted SPB requests; MOEX has its own.
if getattr(settings, "EXCHANGE_SPB_RUSSIAN_TRUST_ENABLED", False):
session.verify = str(EIS_CA_BUNDLE_PATH)
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
before = discover_spb_date(
limited_bytes(response, 4 * 1024**2).decode("utf-8")
)
with registry_response(
session,
SPB_EXPORT,
hosts=SPB_HOSTS,
params={"fileName": "ListingSecurityList"},
) as response:
handle = BytesIO(
limited_bytes(
response,
getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2),
)
)
save_artifact_file(artifact, handle)
handle.close()
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
after = discover_spb_date(
limited_bytes(response, 4 * 1024**2).decode("utf-8")
)
if after != before:
raise SnapshotValidationError("spb_publication_changed_during_download")
artifact.source_published_at = before
artifact.save(update_fields=("source_published_at",))
return artifact
except Exception as exc:
reject_snapshot(artifact, exc)
raise