"""Strict streaming reader for the verified current SPB public CSV export.""" from __future__ import annotations import csv import re from collections.abc import Iterator from datetime import date, datetime from decimal import Decimal, InvalidOperation from io import BytesIO, TextIOWrapper from typing import BinaryIO from urllib.parse import urljoin, urlparse from apps.parsers.clients.common.structured import EIS_CA_BUNDLE_PATH from apps.parsers.models import ParserSourceArtifact from apps.parsers.registry_http import ( limited_bytes, registry_response, registry_session, ) from apps.parsers.registry_snapshots import ( SnapshotValidationError, reject_snapshot, save_artifact_file, ) from bs4 import BeautifulSoup from django.conf import settings from organizations.source_ingestion import SourceRecordInput EXCHANGE_SOURCE = "stock_exchange_activity" SPB_INDEX = "https://spbexchange.ru/listing/securities/" SPB_LIST = "https://spbexchange.ru/listing/securities/list/" SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export" SPB_HOSTS = frozenset({"spbexchange.ru"}) SPB_HEADERS = ( "Row", "s_level_name", "e_full_name", "e_INN_code", "s_sec_type_name_dop", "s_sec_form_name_full", "s_RTS_code", "s_ISIN_code", "si_gos_reg_num", "si_gos_reg_date", "s_face_value", "s_face_value_currency", "s_quot_list_in_date", "s_segment", "s_date_defolt", "s_date_technic_defolt", "f_is_exchanged", "s_is_org_discl_restriction", ) def discover_spb_date(html: str) -> date: dates = [] for link in BeautifulSoup(html, "html.parser").select("a[href]"): url = urlparse(urljoin(SPB_INDEX, link["href"])) match = re.fullmatch( r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})", link.get_text(" ", strip=True), ) if ( match and url.scheme == "https" and url.hostname == "spbexchange.ru" and url.path.rstrip("/") == "/listing/securities/list" and not url.username and not url.password and url.port in (None, 443) ): try: dates.append(datetime.strptime(match[1], "%d.%m.%Y").date()) except ValueError as exc: raise SnapshotValidationError("invalid_spb_publication_date") from exc if not dates: raise SnapshotValidationError("spb_current_listing_missing") return max(dates) def _date(value: str) -> str | None: if not value or value == "-": return None try: return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat() except ValueError as exc: raise SnapshotValidationError("invalid_spb_date") from exc def _nominal(value: str) -> str | None: if value in ("", "-", "без номинальной стоимости"): return None try: number = Decimal(value.replace(",", ".")) if not number.is_finite(): raise InvalidOperation return format(number, "f") except InvalidOperation as exc: raise SnapshotValidationError("invalid_spb_nominal") from exc def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]: text = TextIOWrapper(handle, encoding="utf-8-sig", newline="") try: reader = csv.DictReader(text, delimiter=";", strict=True) if ( not reader.fieldnames or len(reader.fieldnames) != len(SPB_HEADERS) or set(reader.fieldnames) != set(SPB_HEADERS) ): raise SnapshotValidationError("unknown_spb_headers") for number, raw in enumerate(reader, 1): if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000): raise SnapshotValidationError("spb_row_limit") if None in raw or any(value is None for value in raw.values()): raise SnapshotValidationError("invalid_spb_row_width") fields = {k: v.strip() for k, v in raw.items()} code = fields["s_RTS_code"] if not code or len(code) > 200 or not fields["e_full_name"]: raise SnapshotValidationError("missing_spb_instrument_identity") listing_date = _date(fields["s_quot_list_in_date"]) payload = { "exchange": "spb", "instrument_id": code, "issuer_inn": fields["e_INN_code"], "issuer_provider_id": None, "isin": fields["s_ISIN_code"] or None, "instrument_kind": fields["s_sec_type_name_dop"], "listing_date": listing_date, "nominal_value": _nominal(fields["s_face_value"]), "nominal_currency": fields["s_face_value_currency"] or None, "parameters": { "registration_number": fields["si_gos_reg_num"] or None, "registered_on": _date(fields["si_gos_reg_date"]), "listing_level": fields["s_level_name"], "category": fields["s_sec_form_name_full"], }, "trading": None, "duration": None, "yield": None, "upstream": raw, "provenance": {"row_number": number}, } yield SourceRecordInput( external_id=f"spb:{code}", title=code, organization_name=fields["e_full_name"], inn=fields["e_INN_code"], record_type="exchange_instrument", record_date=listing_date or "", url=SPB_LIST, payload=payload, ) except (UnicodeError, csv.Error) as exc: raise SnapshotValidationError("invalid_spb_csv") from exc finally: text.detach() def download_spb_artifact( *, load_batch: int, uploaded_by_id: int | None = None ) -> ParserSourceArtifact: artifact = ParserSourceArtifact.objects.create( source=EXCHANGE_SOURCE, original_name="spb-listing.csv", content_type="text/csv", load_batch=load_batch, uploaded_by_id=uploaded_by_id, metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT}, ) try: with registry_session() as session: # This session serves only allowlisted SPB requests; MOEX has its own. if getattr(settings, "EXCHANGE_SPB_RUSSIAN_TRUST_ENABLED", False): session.verify = str(EIS_CA_BUNDLE_PATH) with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response: before = discover_spb_date( limited_bytes(response, 4 * 1024**2).decode("utf-8") ) with registry_response( session, SPB_EXPORT, hosts=SPB_HOSTS, params={"fileName": "ListingSecurityList"}, ) as response: handle = BytesIO( limited_bytes( response, getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2), ) ) save_artifact_file(artifact, handle) handle.close() with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response: after = discover_spb_date( limited_bytes(response, 4 * 1024**2).decode("utf-8") ) if after != before: raise SnapshotValidationError("spb_publication_changed_during_download") artifact.source_published_at = before artifact.save(update_fields=("source_published_at",)) return artifact except Exception as exc: reject_snapshot(artifact, exc) raise