feat: ingest verified SPB listing CSV

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:04:11 +02:00
parent 648d995bcc
commit 084965b16d
3 changed files with 384 additions and 0 deletions

View File

@@ -8,6 +8,7 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag
from django.utils.deconstruct import deconstructible
PRIVATE_ARTIFACT_PREFIXES = (
"parser-artifacts/stock_exchange_activity/",
"parser-artifacts/checko_leasing/",
"parser-artifacts/checko_messages/",
"parser-artifacts/checko_licenses/",

View File

@@ -0,0 +1,203 @@
"""Strict streaming reader for the verified current SPB public CSV export."""
from __future__ import annotations
import csv
import re
from collections.abc import Iterator
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from io import BytesIO, TextIOWrapper
from typing import BinaryIO
from urllib.parse import urljoin, urlparse
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_http import (
limited_bytes,
registry_response,
registry_session,
)
from apps.parsers.registry_snapshots import (
SnapshotValidationError,
reject_snapshot,
save_artifact_file,
)
from bs4 import BeautifulSoup
from django.conf import settings
from organizations.source_ingestion import SourceRecordInput
EXCHANGE_SOURCE = "stock_exchange_activity"
SPB_INDEX = "https://spbexchange.ru/listing/securities/"
SPB_LIST = "https://spbexchange.ru/listing/securities/list/"
SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export"
SPB_HOSTS = frozenset({"spbexchange.ru"})
SPB_HEADERS = (
"Row",
"s_level_name",
"e_full_name",
"e_INN_code",
"s_sec_type_name_dop",
"s_sec_form_name_full",
"s_RTS_code",
"s_ISIN_code",
"si_gos_reg_num",
"si_gos_reg_date",
"s_face_value",
"s_face_value_currency",
"s_quot_list_in_date",
"s_segment",
"s_date_defolt",
"s_date_technic_defolt",
"f_is_exchanged",
"s_is_org_discl_restriction",
)
def discover_spb_date(html: str) -> date:
dates = []
for link in BeautifulSoup(html, "html.parser").select("a[href]"):
url = urlparse(urljoin(SPB_INDEX, link["href"]))
match = re.fullmatch(
r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})",
link.get_text(" ", strip=True),
)
if (
match
and url.scheme == "https"
and url.hostname == "spbexchange.ru"
and url.path.rstrip("/") == "/listing/securities/list"
and not url.username
and not url.password
and url.port in (None, 443)
):
try:
dates.append(datetime.strptime(match[1], "%d.%m.%Y").date())
except ValueError as exc:
raise SnapshotValidationError("invalid_spb_publication_date") from exc
if not dates:
raise SnapshotValidationError("spb_current_listing_missing")
return max(dates)
def _date(value: str) -> str | None:
if not value or value == "-":
return None
try:
return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat()
except ValueError as exc:
raise SnapshotValidationError("invalid_spb_date") from exc
def _nominal(value: str) -> str | None:
if value in ("", "-", "без номинальной стоимости"):
return None
try:
number = Decimal(value.replace(",", "."))
if not number.is_finite():
raise InvalidOperation
return format(number, "f")
except InvalidOperation as exc:
raise SnapshotValidationError("invalid_spb_nominal") from exc
def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]:
text = TextIOWrapper(handle, encoding="utf-8-sig", newline="")
try:
reader = csv.DictReader(text, delimiter=";", strict=True)
if (
not reader.fieldnames
or len(reader.fieldnames) != len(SPB_HEADERS)
or set(reader.fieldnames) != set(SPB_HEADERS)
):
raise SnapshotValidationError("unknown_spb_headers")
for number, raw in enumerate(reader, 1):
if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000):
raise SnapshotValidationError("spb_row_limit")
if None in raw or any(value is None for value in raw.values()):
raise SnapshotValidationError("invalid_spb_row_width")
fields = {k: v.strip() for k, v in raw.items()}
code = fields["s_RTS_code"]
if not code or len(code) > 200 or not fields["e_full_name"]:
raise SnapshotValidationError("missing_spb_instrument_identity")
listing_date = _date(fields["s_quot_list_in_date"])
payload = {
"exchange": "spb",
"instrument_id": code,
"issuer_inn": fields["e_INN_code"],
"issuer_provider_id": None,
"isin": fields["s_ISIN_code"] or None,
"instrument_kind": fields["s_sec_type_name_dop"],
"listing_date": listing_date,
"nominal_value": _nominal(fields["s_face_value"]),
"nominal_currency": fields["s_face_value_currency"] or None,
"parameters": {
"registration_number": fields["si_gos_reg_num"] or None,
"registered_on": _date(fields["si_gos_reg_date"]),
"listing_level": fields["s_level_name"],
"category": fields["s_sec_form_name_full"],
},
"trading": None,
"duration": None,
"yield": None,
"upstream": raw,
"provenance": {"row_number": number},
}
yield SourceRecordInput(
external_id=f"spb:{code}",
title=code,
organization_name=fields["e_full_name"],
inn=fields["e_INN_code"],
record_type="exchange_instrument",
record_date=listing_date or "",
url=SPB_LIST,
payload=payload,
)
except (UnicodeError, csv.Error) as exc:
raise SnapshotValidationError("invalid_spb_csv") from exc
finally:
text.detach()
def download_spb_artifact(
*, load_batch: int, uploaded_by_id: int | None = None
) -> ParserSourceArtifact:
artifact = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="spb-listing.csv",
content_type="text/csv",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT},
)
try:
with registry_session() as session:
# Existing EIS trust is deliberately NOT reused: approval was probe-only.
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
before = discover_spb_date(
limited_bytes(response, 4 * 1024**2).decode("utf-8")
)
with registry_response(
session,
SPB_EXPORT,
hosts=SPB_HOSTS,
params={"fileName": "ListingSecurityList"},
) as response:
handle = BytesIO(
limited_bytes(
response,
getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2),
)
)
save_artifact_file(artifact, handle)
handle.close()
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
after = discover_spb_date(
limited_bytes(response, 4 * 1024**2).decode("utf-8")
)
if after != before:
raise SnapshotValidationError("spb_publication_changed_during_download")
artifact.source_published_at = before
artifact.save(update_fields=("source_published_at",))
return artifact
except Exception as exc:
reject_snapshot(artifact, exc)
raise