feat: ingest verified SPB listing CSV

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:04:11 +02:00
parent 648d995bcc
commit 084965b16d
3 changed files with 384 additions and 0 deletions

View File

@@ -8,6 +8,7 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag
from django.utils.deconstruct import deconstructible
PRIVATE_ARTIFACT_PREFIXES = (
"parser-artifacts/stock_exchange_activity/",
"parser-artifacts/checko_leasing/",
"parser-artifacts/checko_messages/",
"parser-artifacts/checko_licenses/",

View File

@@ -0,0 +1,203 @@
"""Strict streaming reader for the verified current SPB public CSV export."""
from __future__ import annotations
import csv
import re
from collections.abc import Iterator
from datetime import date, datetime
from decimal import Decimal, InvalidOperation
from io import BytesIO, TextIOWrapper
from typing import BinaryIO
from urllib.parse import urljoin, urlparse
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_http import (
limited_bytes,
registry_response,
registry_session,
)
from apps.parsers.registry_snapshots import (
SnapshotValidationError,
reject_snapshot,
save_artifact_file,
)
from bs4 import BeautifulSoup
from django.conf import settings
from organizations.source_ingestion import SourceRecordInput
EXCHANGE_SOURCE = "stock_exchange_activity"
SPB_INDEX = "https://spbexchange.ru/listing/securities/"
SPB_LIST = "https://spbexchange.ru/listing/securities/list/"
SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export"
SPB_HOSTS = frozenset({"spbexchange.ru"})
SPB_HEADERS = (
"Row",
"s_level_name",
"e_full_name",
"e_INN_code",
"s_sec_type_name_dop",
"s_sec_form_name_full",
"s_RTS_code",
"s_ISIN_code",
"si_gos_reg_num",
"si_gos_reg_date",
"s_face_value",
"s_face_value_currency",
"s_quot_list_in_date",
"s_segment",
"s_date_defolt",
"s_date_technic_defolt",
"f_is_exchanged",
"s_is_org_discl_restriction",
)
def discover_spb_date(html: str) -> date:
dates = []
for link in BeautifulSoup(html, "html.parser").select("a[href]"):
url = urlparse(urljoin(SPB_INDEX, link["href"]))
match = re.fullmatch(
r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})",
link.get_text(" ", strip=True),
)
if (
match
and url.scheme == "https"
and url.hostname == "spbexchange.ru"
and url.path.rstrip("/") == "/listing/securities/list"
and not url.username
and not url.password
and url.port in (None, 443)
):
try:
dates.append(datetime.strptime(match[1], "%d.%m.%Y").date())
except ValueError as exc:
raise SnapshotValidationError("invalid_spb_publication_date") from exc
if not dates:
raise SnapshotValidationError("spb_current_listing_missing")
return max(dates)
def _date(value: str) -> str | None:
if not value or value == "-":
return None
try:
return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat()
except ValueError as exc:
raise SnapshotValidationError("invalid_spb_date") from exc
def _nominal(value: str) -> str | None:
if value in ("", "-", "без номинальной стоимости"):
return None
try:
number = Decimal(value.replace(",", "."))
if not number.is_finite():
raise InvalidOperation
return format(number, "f")
except InvalidOperation as exc:
raise SnapshotValidationError("invalid_spb_nominal") from exc
def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]:
text = TextIOWrapper(handle, encoding="utf-8-sig", newline="")
try:
reader = csv.DictReader(text, delimiter=";", strict=True)
if (
not reader.fieldnames
or len(reader.fieldnames) != len(SPB_HEADERS)
or set(reader.fieldnames) != set(SPB_HEADERS)
):
raise SnapshotValidationError("unknown_spb_headers")
for number, raw in enumerate(reader, 1):
if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000):
raise SnapshotValidationError("spb_row_limit")
if None in raw or any(value is None for value in raw.values()):
raise SnapshotValidationError("invalid_spb_row_width")
fields = {k: v.strip() for k, v in raw.items()}
code = fields["s_RTS_code"]
if not code or len(code) > 200 or not fields["e_full_name"]:
raise SnapshotValidationError("missing_spb_instrument_identity")
listing_date = _date(fields["s_quot_list_in_date"])
payload = {
"exchange": "spb",
"instrument_id": code,
"issuer_inn": fields["e_INN_code"],
"issuer_provider_id": None,
"isin": fields["s_ISIN_code"] or None,
"instrument_kind": fields["s_sec_type_name_dop"],
"listing_date": listing_date,
"nominal_value": _nominal(fields["s_face_value"]),
"nominal_currency": fields["s_face_value_currency"] or None,
"parameters": {
"registration_number": fields["si_gos_reg_num"] or None,
"registered_on": _date(fields["si_gos_reg_date"]),
"listing_level": fields["s_level_name"],
"category": fields["s_sec_form_name_full"],
},
"trading": None,
"duration": None,
"yield": None,
"upstream": raw,
"provenance": {"row_number": number},
}
yield SourceRecordInput(
external_id=f"spb:{code}",
title=code,
organization_name=fields["e_full_name"],
inn=fields["e_INN_code"],
record_type="exchange_instrument",
record_date=listing_date or "",
url=SPB_LIST,
payload=payload,
)
except (UnicodeError, csv.Error) as exc:
raise SnapshotValidationError("invalid_spb_csv") from exc
finally:
text.detach()
def download_spb_artifact(
*, load_batch: int, uploaded_by_id: int | None = None
) -> ParserSourceArtifact:
artifact = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="spb-listing.csv",
content_type="text/csv",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT},
)
try:
with registry_session() as session:
# Existing EIS trust is deliberately NOT reused: approval was probe-only.
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
before = discover_spb_date(
limited_bytes(response, 4 * 1024**2).decode("utf-8")
)
with registry_response(
session,
SPB_EXPORT,
hosts=SPB_HOSTS,
params={"fileName": "ListingSecurityList"},
) as response:
handle = BytesIO(
limited_bytes(
response,
getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2),
)
)
save_artifact_file(artifact, handle)
handle.close()
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
after = discover_spb_date(
limited_bytes(response, 4 * 1024**2).decode("utf-8")
)
if after != before:
raise SnapshotValidationError("spb_publication_changed_during_download")
artifact.source_published_at = before
artifact.save(update_fields=("source_published_at",))
return artifact
except Exception as exc:
reject_snapshot(artifact, exc)
raise

View File

@@ -0,0 +1,180 @@
"""Public CSV contract derived from the current SPB listing export."""
import csv
import importlib
from io import BytesIO, StringIO
from pathlib import Path
import pytest
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.test import override_settings
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
def adapter():
return importlib.import_module("apps.parsers.exchange_spb")
def data(**changes):
rows = list(
csv.reader(
StringIO((FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")),
delimiter=";",
)
)
for key, value in changes.items():
rows[1][rows[0].index(key)] = value
stream = StringIO()
csv.writer(stream, delimiter=";").writerows(rows)
return BytesIO(stream.getvalue().encode())
def test_preserves_inn_vendor_identity_and_decimal():
row = next(adapter().iter_spb_records(data()))
assert row.inn == "0123456789"
assert row.external_id == "spb:fixture-security-1"
assert row.payload["exchange"] == "spb"
assert row.amount is None
assert row.payload["nominal_value"] == "0.1"
assert row.payload["listing_date"] == "2024-11-06"
assert row.payload["duration"] is None
assert row.payload["yield"] is None
@pytest.mark.parametrize("name", ["unknown_headers.csv"])
def test_unknown_headers_rejected(name):
with (FIXTURES / name).open("rb") as f, pytest.raises(SnapshotValidationError):
list(adapter().iter_spb_records(f))
@pytest.mark.parametrize("value", [b"not csv", b"PK\x03\x04invalid", b""])
def test_invalid_input_rejected(value):
with pytest.raises(SnapshotValidationError):
list(adapter().iter_spb_records(BytesIO(value)))
def test_genuine_empty_with_valid_headers():
with (FIXTURES / "empty.csv").open("rb") as f:
assert list(adapter().iter_spb_records(f)) == []
def test_missing_inn_stays_missing_and_name_is_not_identity():
row = next(adapter().iter_spb_records(data(e_INN_code="")))
assert row.inn == ""
assert row.organization_uid is None
@pytest.mark.parametrize(
"field,value",
[("s_RTS_code", ""), ("s_quot_list_in_date", "tomorrow"), ("s_face_value", "NaN")],
)
def test_malformed_required_or_numeric_fields_rejected(field, value):
with pytest.raises(SnapshotValidationError):
list(adapter().iter_spb_records(data(**{field: value})))
def test_bounded_rows():
with override_settings(EXCHANGE_SPB_MAX_ROWS=0), pytest.raises(
SnapshotValidationError
):
list(adapter().iter_spb_records(data()))
def test_latest_date_not_link_order():
html = '<a href="/listing/securities/list">Список ценных бумаг на 05.10.2026</a><a href="/listing/securities/list/">Список ценных бумаг на 02.10.2026</a>'
assert adapter().discover_spb_date(html).isoformat() == "2026-10-05"
def test_no_dated_current_link_rejected():
with pytest.raises(SnapshotValidationError):
adapter().discover_spb_date(
'<a href="https://evil.test/list">Список ценных бумаг на 05.10.2026</a>'
)
def test_handle_left_open_and_text_inn_not_coerced():
handle = data(e_INN_code="0010000025")
assert next(adapter().iter_spb_records(handle)).inn == "0010000025"
assert not handle.closed
def test_tls_trust_stays_default():
session = adapter().registry_session()
assert session.verify is True
session.close()
@pytest.mark.django_db
def test_download_checksum_date_private_file_and_three_bounded_reads(tmp_path):
import hashlib
from contextlib import contextmanager
from unittest.mock import patch
module = adapter()
body = (FIXTURES / "listing.csv").read_bytes()
html = b'<a href="/listing/securities/list">\xd0\xa1\xd0\xbf\xd0\xb8\xd1\x81\xd0\xbe\xd0\xba \xd1\x86\xd0\xb5\xd0\xbd\xd0\xbd\xd1\x8b\xd1\x85 \xd0\xb1\xd1\x83\xd0\xbc\xd0\xb0\xd0\xb3 \xd0\xbd\xd0\xb0 05.10.2026</a>'
urls = []
@contextmanager
def response(session, url, **kwargs):
urls.append(url)
class Response:
def iter_content(self, chunk_size):
yield body if url == module.SPB_EXPORT else html
yield Response()
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch.object(
module, "registry_response", response
):
artifact = module.download_spb_artifact(load_batch=1)
assert artifact.sha256 == hashlib.sha256(body).hexdigest()
assert artifact.source_published_at.isoformat() == "2026-10-05"
assert artifact.metadata["input_only"] is True
assert len(urls) == 3
with pytest.raises(ValueError):
_ = artifact.file.url
@pytest.mark.django_db
def test_download_size_limit_rejects_artifact(tmp_path):
from contextlib import contextmanager
from unittest.mock import patch
from apps.parsers.models import ParserSourceArtifact
module = adapter()
html = '<a href="/listing/securities/list">Список ценных бумаг на 05.10.2026</a>'.encode()
@contextmanager
def response(session, url, **kwargs):
class Response:
def iter_content(self, chunk_size):
yield b"oversized" if url == module.SPB_EXPORT else html
yield Response()
with override_settings(
PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path), EXCHANGE_SPB_MAX_BYTES=1
), patch.object(module, "registry_response", response), pytest.raises(
SnapshotValidationError
):
module.download_spb_artifact(load_batch=1)
assert ParserSourceArtifact.objects.get().status == "rejected"
def test_redirect_host_rejected_before_second_request():
from unittest.mock import Mock
module = adapter()
session = Mock()
response = Mock(
is_redirect=True, headers={"Location": "https://evil.test/export.csv"}
)
session.get.return_value = response
with pytest.raises(SnapshotValidationError), module.registry_response(
session, module.SPB_EXPORT, hosts=module.SPB_HOSTS
):
pytest.fail("unsafe redirect yielded")
assert session.get.call_count == 1