Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m3s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
207 lines
7.7 KiB
Python
207 lines
7.7 KiB
Python
"""Strict streaming reader for the verified current SPB public CSV export."""
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import re
|
|
from collections.abc import Iterator
|
|
from datetime import date, datetime
|
|
from decimal import Decimal, InvalidOperation
|
|
from io import BytesIO, TextIOWrapper
|
|
from typing import BinaryIO
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
from apps.parsers.clients.common.structured import EIS_CA_BUNDLE_PATH
|
|
from apps.parsers.models import ParserSourceArtifact
|
|
from apps.parsers.registry_http import (
|
|
limited_bytes,
|
|
registry_response,
|
|
registry_session,
|
|
)
|
|
from apps.parsers.registry_snapshots import (
|
|
SnapshotValidationError,
|
|
reject_snapshot,
|
|
save_artifact_file,
|
|
)
|
|
from bs4 import BeautifulSoup
|
|
from django.conf import settings
|
|
from organizations.source_ingestion import SourceRecordInput
|
|
|
|
EXCHANGE_SOURCE = "stock_exchange_activity"
|
|
SPB_INDEX = "https://spbexchange.ru/listing/securities/"
|
|
SPB_LIST = "https://spbexchange.ru/listing/securities/list/"
|
|
SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export"
|
|
SPB_HOSTS = frozenset({"spbexchange.ru"})
|
|
SPB_HEADERS = (
|
|
"Row",
|
|
"s_level_name",
|
|
"e_full_name",
|
|
"e_INN_code",
|
|
"s_sec_type_name_dop",
|
|
"s_sec_form_name_full",
|
|
"s_RTS_code",
|
|
"s_ISIN_code",
|
|
"si_gos_reg_num",
|
|
"si_gos_reg_date",
|
|
"s_face_value",
|
|
"s_face_value_currency",
|
|
"s_quot_list_in_date",
|
|
"s_segment",
|
|
"s_date_defolt",
|
|
"s_date_technic_defolt",
|
|
"f_is_exchanged",
|
|
"s_is_org_discl_restriction",
|
|
)
|
|
|
|
|
|
def discover_spb_date(html: str) -> date:
|
|
dates = []
|
|
for link in BeautifulSoup(html, "html.parser").select("a[href]"):
|
|
url = urlparse(urljoin(SPB_INDEX, link["href"]))
|
|
match = re.fullmatch(
|
|
r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})",
|
|
link.get_text(" ", strip=True),
|
|
)
|
|
if (
|
|
match
|
|
and url.scheme == "https"
|
|
and url.hostname == "spbexchange.ru"
|
|
and url.path.rstrip("/") == "/listing/securities/list"
|
|
and not url.username
|
|
and not url.password
|
|
and url.port in (None, 443)
|
|
):
|
|
try:
|
|
dates.append(datetime.strptime(match[1], "%d.%m.%Y").date())
|
|
except ValueError as exc:
|
|
raise SnapshotValidationError("invalid_spb_publication_date") from exc
|
|
if not dates:
|
|
raise SnapshotValidationError("spb_current_listing_missing")
|
|
return max(dates)
|
|
|
|
|
|
def _date(value: str) -> str | None:
|
|
if not value or value == "-":
|
|
return None
|
|
try:
|
|
return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat()
|
|
except ValueError as exc:
|
|
raise SnapshotValidationError("invalid_spb_date") from exc
|
|
|
|
|
|
def _nominal(value: str) -> str | None:
|
|
if value in ("", "-", "без номинальной стоимости"):
|
|
return None
|
|
try:
|
|
number = Decimal(value.replace(",", "."))
|
|
if not number.is_finite():
|
|
raise InvalidOperation
|
|
return format(number, "f")
|
|
except InvalidOperation as exc:
|
|
raise SnapshotValidationError("invalid_spb_nominal") from exc
|
|
|
|
|
|
def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]:
|
|
text = TextIOWrapper(handle, encoding="utf-8-sig", newline="")
|
|
try:
|
|
reader = csv.DictReader(text, delimiter=";", strict=True)
|
|
if (
|
|
not reader.fieldnames
|
|
or len(reader.fieldnames) != len(SPB_HEADERS)
|
|
or set(reader.fieldnames) != set(SPB_HEADERS)
|
|
):
|
|
raise SnapshotValidationError("unknown_spb_headers")
|
|
for number, raw in enumerate(reader, 1):
|
|
if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000):
|
|
raise SnapshotValidationError("spb_row_limit")
|
|
if None in raw or any(value is None for value in raw.values()):
|
|
raise SnapshotValidationError("invalid_spb_row_width")
|
|
fields = {k: v.strip() for k, v in raw.items()}
|
|
code = fields["s_RTS_code"]
|
|
if not code or len(code) > 200 or not fields["e_full_name"]:
|
|
raise SnapshotValidationError("missing_spb_instrument_identity")
|
|
listing_date = _date(fields["s_quot_list_in_date"])
|
|
payload = {
|
|
"exchange": "spb",
|
|
"instrument_id": code,
|
|
"issuer_inn": fields["e_INN_code"],
|
|
"issuer_provider_id": None,
|
|
"isin": fields["s_ISIN_code"] or None,
|
|
"instrument_kind": fields["s_sec_type_name_dop"],
|
|
"listing_date": listing_date,
|
|
"nominal_value": _nominal(fields["s_face_value"]),
|
|
"nominal_currency": fields["s_face_value_currency"] or None,
|
|
"parameters": {
|
|
"registration_number": fields["si_gos_reg_num"] or None,
|
|
"registered_on": _date(fields["si_gos_reg_date"]),
|
|
"listing_level": fields["s_level_name"],
|
|
"category": fields["s_sec_form_name_full"],
|
|
},
|
|
"trading": None,
|
|
"duration": None,
|
|
"yield": None,
|
|
"upstream": raw,
|
|
"provenance": {"row_number": number},
|
|
}
|
|
yield SourceRecordInput(
|
|
external_id=f"spb:{code}",
|
|
title=code,
|
|
organization_name=fields["e_full_name"],
|
|
inn=fields["e_INN_code"],
|
|
record_type="exchange_instrument",
|
|
record_date=listing_date or "",
|
|
url=SPB_LIST,
|
|
payload=payload,
|
|
)
|
|
except (UnicodeError, csv.Error) as exc:
|
|
raise SnapshotValidationError("invalid_spb_csv") from exc
|
|
finally:
|
|
text.detach()
|
|
|
|
|
|
def download_spb_artifact(
|
|
*, load_batch: int, uploaded_by_id: int | None = None
|
|
) -> ParserSourceArtifact:
|
|
artifact = ParserSourceArtifact.objects.create(
|
|
source=EXCHANGE_SOURCE,
|
|
original_name="spb-listing.csv",
|
|
content_type="text/csv",
|
|
load_batch=load_batch,
|
|
uploaded_by_id=uploaded_by_id,
|
|
metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT},
|
|
)
|
|
try:
|
|
with registry_session() as session:
|
|
# This session serves only allowlisted SPB requests; MOEX has its own.
|
|
if getattr(settings, "EXCHANGE_SPB_RUSSIAN_TRUST_ENABLED", False):
|
|
session.verify = str(EIS_CA_BUNDLE_PATH)
|
|
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
|
|
before = discover_spb_date(
|
|
limited_bytes(response, 4 * 1024**2).decode("utf-8")
|
|
)
|
|
with registry_response(
|
|
session,
|
|
SPB_EXPORT,
|
|
hosts=SPB_HOSTS,
|
|
params={"fileName": "ListingSecurityList"},
|
|
) as response:
|
|
handle = BytesIO(
|
|
limited_bytes(
|
|
response,
|
|
getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2),
|
|
)
|
|
)
|
|
save_artifact_file(artifact, handle)
|
|
handle.close()
|
|
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
|
|
after = discover_spb_date(
|
|
limited_bytes(response, 4 * 1024**2).decode("utf-8")
|
|
)
|
|
if after != before:
|
|
raise SnapshotValidationError("spb_publication_changed_during_download")
|
|
artifact.source_published_at = before
|
|
artifact.save(update_fields=("source_published_at",))
|
|
return artifact
|
|
except Exception as exc:
|
|
reject_snapshot(artifact, exc)
|
|
raise
|