feat: ingest verified SPB listing CSV
This commit is contained in:
@@ -8,6 +8,7 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag
|
||||
from django.utils.deconstruct import deconstructible
|
||||
|
||||
PRIVATE_ARTIFACT_PREFIXES = (
|
||||
"parser-artifacts/stock_exchange_activity/",
|
||||
"parser-artifacts/checko_leasing/",
|
||||
"parser-artifacts/checko_messages/",
|
||||
"parser-artifacts/checko_licenses/",
|
||||
|
||||
203
src/apps/parsers/exchange_spb.py
Normal file
203
src/apps/parsers/exchange_spb.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Strict streaming reader for the verified current SPB public CSV export."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from io import BytesIO, TextIOWrapper
|
||||
from typing import BinaryIO
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_http import (
|
||||
limited_bytes,
|
||||
registry_response,
|
||||
registry_session,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import (
|
||||
SnapshotValidationError,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
)
|
||||
from bs4 import BeautifulSoup
|
||||
from django.conf import settings
|
||||
from organizations.source_ingestion import SourceRecordInput
|
||||
|
||||
EXCHANGE_SOURCE = "stock_exchange_activity"
|
||||
SPB_INDEX = "https://spbexchange.ru/listing/securities/"
|
||||
SPB_LIST = "https://spbexchange.ru/listing/securities/list/"
|
||||
SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export"
|
||||
SPB_HOSTS = frozenset({"spbexchange.ru"})
|
||||
SPB_HEADERS = (
|
||||
"Row",
|
||||
"s_level_name",
|
||||
"e_full_name",
|
||||
"e_INN_code",
|
||||
"s_sec_type_name_dop",
|
||||
"s_sec_form_name_full",
|
||||
"s_RTS_code",
|
||||
"s_ISIN_code",
|
||||
"si_gos_reg_num",
|
||||
"si_gos_reg_date",
|
||||
"s_face_value",
|
||||
"s_face_value_currency",
|
||||
"s_quot_list_in_date",
|
||||
"s_segment",
|
||||
"s_date_defolt",
|
||||
"s_date_technic_defolt",
|
||||
"f_is_exchanged",
|
||||
"s_is_org_discl_restriction",
|
||||
)
|
||||
|
||||
|
||||
def discover_spb_date(html: str) -> date:
|
||||
dates = []
|
||||
for link in BeautifulSoup(html, "html.parser").select("a[href]"):
|
||||
url = urlparse(urljoin(SPB_INDEX, link["href"]))
|
||||
match = re.fullmatch(
|
||||
r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})",
|
||||
link.get_text(" ", strip=True),
|
||||
)
|
||||
if (
|
||||
match
|
||||
and url.scheme == "https"
|
||||
and url.hostname == "spbexchange.ru"
|
||||
and url.path.rstrip("/") == "/listing/securities/list"
|
||||
and not url.username
|
||||
and not url.password
|
||||
and url.port in (None, 443)
|
||||
):
|
||||
try:
|
||||
dates.append(datetime.strptime(match[1], "%d.%m.%Y").date())
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_spb_publication_date") from exc
|
||||
if not dates:
|
||||
raise SnapshotValidationError("spb_current_listing_missing")
|
||||
return max(dates)
|
||||
|
||||
|
||||
def _date(value: str) -> str | None:
|
||||
if not value or value == "-":
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat()
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_spb_date") from exc
|
||||
|
||||
|
||||
def _nominal(value: str) -> str | None:
|
||||
if value in ("", "-", "без номинальной стоимости"):
|
||||
return None
|
||||
try:
|
||||
number = Decimal(value.replace(",", "."))
|
||||
if not number.is_finite():
|
||||
raise InvalidOperation
|
||||
return format(number, "f")
|
||||
except InvalidOperation as exc:
|
||||
raise SnapshotValidationError("invalid_spb_nominal") from exc
|
||||
|
||||
|
||||
def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]:
|
||||
text = TextIOWrapper(handle, encoding="utf-8-sig", newline="")
|
||||
try:
|
||||
reader = csv.DictReader(text, delimiter=";", strict=True)
|
||||
if (
|
||||
not reader.fieldnames
|
||||
or len(reader.fieldnames) != len(SPB_HEADERS)
|
||||
or set(reader.fieldnames) != set(SPB_HEADERS)
|
||||
):
|
||||
raise SnapshotValidationError("unknown_spb_headers")
|
||||
for number, raw in enumerate(reader, 1):
|
||||
if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000):
|
||||
raise SnapshotValidationError("spb_row_limit")
|
||||
if None in raw or any(value is None for value in raw.values()):
|
||||
raise SnapshotValidationError("invalid_spb_row_width")
|
||||
fields = {k: v.strip() for k, v in raw.items()}
|
||||
code = fields["s_RTS_code"]
|
||||
if not code or len(code) > 200 or not fields["e_full_name"]:
|
||||
raise SnapshotValidationError("missing_spb_instrument_identity")
|
||||
listing_date = _date(fields["s_quot_list_in_date"])
|
||||
payload = {
|
||||
"exchange": "spb",
|
||||
"instrument_id": code,
|
||||
"issuer_inn": fields["e_INN_code"],
|
||||
"issuer_provider_id": None,
|
||||
"isin": fields["s_ISIN_code"] or None,
|
||||
"instrument_kind": fields["s_sec_type_name_dop"],
|
||||
"listing_date": listing_date,
|
||||
"nominal_value": _nominal(fields["s_face_value"]),
|
||||
"nominal_currency": fields["s_face_value_currency"] or None,
|
||||
"parameters": {
|
||||
"registration_number": fields["si_gos_reg_num"] or None,
|
||||
"registered_on": _date(fields["si_gos_reg_date"]),
|
||||
"listing_level": fields["s_level_name"],
|
||||
"category": fields["s_sec_form_name_full"],
|
||||
},
|
||||
"trading": None,
|
||||
"duration": None,
|
||||
"yield": None,
|
||||
"upstream": raw,
|
||||
"provenance": {"row_number": number},
|
||||
}
|
||||
yield SourceRecordInput(
|
||||
external_id=f"spb:{code}",
|
||||
title=code,
|
||||
organization_name=fields["e_full_name"],
|
||||
inn=fields["e_INN_code"],
|
||||
record_type="exchange_instrument",
|
||||
record_date=listing_date or "",
|
||||
url=SPB_LIST,
|
||||
payload=payload,
|
||||
)
|
||||
except (UnicodeError, csv.Error) as exc:
|
||||
raise SnapshotValidationError("invalid_spb_csv") from exc
|
||||
finally:
|
||||
text.detach()
|
||||
|
||||
|
||||
def download_spb_artifact(
|
||||
*, load_batch: int, uploaded_by_id: int | None = None
|
||||
) -> ParserSourceArtifact:
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source=EXCHANGE_SOURCE,
|
||||
original_name="spb-listing.csv",
|
||||
content_type="text/csv",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT},
|
||||
)
|
||||
try:
|
||||
with registry_session() as session:
|
||||
# Existing EIS trust is deliberately NOT reused: approval was probe-only.
|
||||
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
|
||||
before = discover_spb_date(
|
||||
limited_bytes(response, 4 * 1024**2).decode("utf-8")
|
||||
)
|
||||
with registry_response(
|
||||
session,
|
||||
SPB_EXPORT,
|
||||
hosts=SPB_HOSTS,
|
||||
params={"fileName": "ListingSecurityList"},
|
||||
) as response:
|
||||
handle = BytesIO(
|
||||
limited_bytes(
|
||||
response,
|
||||
getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2),
|
||||
)
|
||||
)
|
||||
save_artifact_file(artifact, handle)
|
||||
handle.close()
|
||||
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
|
||||
after = discover_spb_date(
|
||||
limited_bytes(response, 4 * 1024**2).decode("utf-8")
|
||||
)
|
||||
if after != before:
|
||||
raise SnapshotValidationError("spb_publication_changed_during_download")
|
||||
artifact.source_published_at = before
|
||||
artifact.save(update_fields=("source_published_at",))
|
||||
return artifact
|
||||
except Exception as exc:
|
||||
reject_snapshot(artifact, exc)
|
||||
raise
|
||||
Reference in New Issue
Block a user