feat: ingest verified SPB listing CSV
This commit is contained in:
@@ -8,6 +8,7 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag
|
||||
from django.utils.deconstruct import deconstructible
|
||||
|
||||
PRIVATE_ARTIFACT_PREFIXES = (
|
||||
"parser-artifacts/stock_exchange_activity/",
|
||||
"parser-artifacts/checko_leasing/",
|
||||
"parser-artifacts/checko_messages/",
|
||||
"parser-artifacts/checko_licenses/",
|
||||
|
||||
203
src/apps/parsers/exchange_spb.py
Normal file
203
src/apps/parsers/exchange_spb.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Strict streaming reader for the verified current SPB public CSV export."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from io import BytesIO, TextIOWrapper
|
||||
from typing import BinaryIO
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_http import (
|
||||
limited_bytes,
|
||||
registry_response,
|
||||
registry_session,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import (
|
||||
SnapshotValidationError,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
)
|
||||
from bs4 import BeautifulSoup
|
||||
from django.conf import settings
|
||||
from organizations.source_ingestion import SourceRecordInput
|
||||
|
||||
EXCHANGE_SOURCE = "stock_exchange_activity"
|
||||
SPB_INDEX = "https://spbexchange.ru/listing/securities/"
|
||||
SPB_LIST = "https://spbexchange.ru/listing/securities/list/"
|
||||
SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export"
|
||||
SPB_HOSTS = frozenset({"spbexchange.ru"})
|
||||
SPB_HEADERS = (
|
||||
"Row",
|
||||
"s_level_name",
|
||||
"e_full_name",
|
||||
"e_INN_code",
|
||||
"s_sec_type_name_dop",
|
||||
"s_sec_form_name_full",
|
||||
"s_RTS_code",
|
||||
"s_ISIN_code",
|
||||
"si_gos_reg_num",
|
||||
"si_gos_reg_date",
|
||||
"s_face_value",
|
||||
"s_face_value_currency",
|
||||
"s_quot_list_in_date",
|
||||
"s_segment",
|
||||
"s_date_defolt",
|
||||
"s_date_technic_defolt",
|
||||
"f_is_exchanged",
|
||||
"s_is_org_discl_restriction",
|
||||
)
|
||||
|
||||
|
||||
def discover_spb_date(html: str) -> date:
|
||||
dates = []
|
||||
for link in BeautifulSoup(html, "html.parser").select("a[href]"):
|
||||
url = urlparse(urljoin(SPB_INDEX, link["href"]))
|
||||
match = re.fullmatch(
|
||||
r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})",
|
||||
link.get_text(" ", strip=True),
|
||||
)
|
||||
if (
|
||||
match
|
||||
and url.scheme == "https"
|
||||
and url.hostname == "spbexchange.ru"
|
||||
and url.path.rstrip("/") == "/listing/securities/list"
|
||||
and not url.username
|
||||
and not url.password
|
||||
and url.port in (None, 443)
|
||||
):
|
||||
try:
|
||||
dates.append(datetime.strptime(match[1], "%d.%m.%Y").date())
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_spb_publication_date") from exc
|
||||
if not dates:
|
||||
raise SnapshotValidationError("spb_current_listing_missing")
|
||||
return max(dates)
|
||||
|
||||
|
||||
def _date(value: str) -> str | None:
|
||||
if not value or value == "-":
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat()
|
||||
except ValueError as exc:
|
||||
raise SnapshotValidationError("invalid_spb_date") from exc
|
||||
|
||||
|
||||
def _nominal(value: str) -> str | None:
|
||||
if value in ("", "-", "без номинальной стоимости"):
|
||||
return None
|
||||
try:
|
||||
number = Decimal(value.replace(",", "."))
|
||||
if not number.is_finite():
|
||||
raise InvalidOperation
|
||||
return format(number, "f")
|
||||
except InvalidOperation as exc:
|
||||
raise SnapshotValidationError("invalid_spb_nominal") from exc
|
||||
|
||||
|
||||
def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]:
|
||||
text = TextIOWrapper(handle, encoding="utf-8-sig", newline="")
|
||||
try:
|
||||
reader = csv.DictReader(text, delimiter=";", strict=True)
|
||||
if (
|
||||
not reader.fieldnames
|
||||
or len(reader.fieldnames) != len(SPB_HEADERS)
|
||||
or set(reader.fieldnames) != set(SPB_HEADERS)
|
||||
):
|
||||
raise SnapshotValidationError("unknown_spb_headers")
|
||||
for number, raw in enumerate(reader, 1):
|
||||
if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000):
|
||||
raise SnapshotValidationError("spb_row_limit")
|
||||
if None in raw or any(value is None for value in raw.values()):
|
||||
raise SnapshotValidationError("invalid_spb_row_width")
|
||||
fields = {k: v.strip() for k, v in raw.items()}
|
||||
code = fields["s_RTS_code"]
|
||||
if not code or len(code) > 200 or not fields["e_full_name"]:
|
||||
raise SnapshotValidationError("missing_spb_instrument_identity")
|
||||
listing_date = _date(fields["s_quot_list_in_date"])
|
||||
payload = {
|
||||
"exchange": "spb",
|
||||
"instrument_id": code,
|
||||
"issuer_inn": fields["e_INN_code"],
|
||||
"issuer_provider_id": None,
|
||||
"isin": fields["s_ISIN_code"] or None,
|
||||
"instrument_kind": fields["s_sec_type_name_dop"],
|
||||
"listing_date": listing_date,
|
||||
"nominal_value": _nominal(fields["s_face_value"]),
|
||||
"nominal_currency": fields["s_face_value_currency"] or None,
|
||||
"parameters": {
|
||||
"registration_number": fields["si_gos_reg_num"] or None,
|
||||
"registered_on": _date(fields["si_gos_reg_date"]),
|
||||
"listing_level": fields["s_level_name"],
|
||||
"category": fields["s_sec_form_name_full"],
|
||||
},
|
||||
"trading": None,
|
||||
"duration": None,
|
||||
"yield": None,
|
||||
"upstream": raw,
|
||||
"provenance": {"row_number": number},
|
||||
}
|
||||
yield SourceRecordInput(
|
||||
external_id=f"spb:{code}",
|
||||
title=code,
|
||||
organization_name=fields["e_full_name"],
|
||||
inn=fields["e_INN_code"],
|
||||
record_type="exchange_instrument",
|
||||
record_date=listing_date or "",
|
||||
url=SPB_LIST,
|
||||
payload=payload,
|
||||
)
|
||||
except (UnicodeError, csv.Error) as exc:
|
||||
raise SnapshotValidationError("invalid_spb_csv") from exc
|
||||
finally:
|
||||
text.detach()
|
||||
|
||||
|
||||
def download_spb_artifact(
|
||||
*, load_batch: int, uploaded_by_id: int | None = None
|
||||
) -> ParserSourceArtifact:
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source=EXCHANGE_SOURCE,
|
||||
original_name="spb-listing.csv",
|
||||
content_type="text/csv",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT},
|
||||
)
|
||||
try:
|
||||
with registry_session() as session:
|
||||
# Existing EIS trust is deliberately NOT reused: approval was probe-only.
|
||||
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
|
||||
before = discover_spb_date(
|
||||
limited_bytes(response, 4 * 1024**2).decode("utf-8")
|
||||
)
|
||||
with registry_response(
|
||||
session,
|
||||
SPB_EXPORT,
|
||||
hosts=SPB_HOSTS,
|
||||
params={"fileName": "ListingSecurityList"},
|
||||
) as response:
|
||||
handle = BytesIO(
|
||||
limited_bytes(
|
||||
response,
|
||||
getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2),
|
||||
)
|
||||
)
|
||||
save_artifact_file(artifact, handle)
|
||||
handle.close()
|
||||
with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response:
|
||||
after = discover_spb_date(
|
||||
limited_bytes(response, 4 * 1024**2).decode("utf-8")
|
||||
)
|
||||
if after != before:
|
||||
raise SnapshotValidationError("spb_publication_changed_during_download")
|
||||
artifact.source_published_at = before
|
||||
artifact.save(update_fields=("source_published_at",))
|
||||
return artifact
|
||||
except Exception as exc:
|
||||
reject_snapshot(artifact, exc)
|
||||
raise
|
||||
180
tests/apps/parsers/test_exchange_spb.py
Normal file
180
tests/apps/parsers/test_exchange_spb.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""Public CSV contract derived from the current SPB listing export."""
|
||||
import csv
|
||||
import importlib
|
||||
from io import BytesIO, StringIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from django.test import override_settings
|
||||
|
||||
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
||||
|
||||
|
||||
def adapter():
|
||||
return importlib.import_module("apps.parsers.exchange_spb")
|
||||
|
||||
|
||||
def data(**changes):
|
||||
rows = list(
|
||||
csv.reader(
|
||||
StringIO((FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")),
|
||||
delimiter=";",
|
||||
)
|
||||
)
|
||||
for key, value in changes.items():
|
||||
rows[1][rows[0].index(key)] = value
|
||||
stream = StringIO()
|
||||
csv.writer(stream, delimiter=";").writerows(rows)
|
||||
return BytesIO(stream.getvalue().encode())
|
||||
|
||||
|
||||
def test_preserves_inn_vendor_identity_and_decimal():
|
||||
row = next(adapter().iter_spb_records(data()))
|
||||
assert row.inn == "0123456789"
|
||||
assert row.external_id == "spb:fixture-security-1"
|
||||
assert row.payload["exchange"] == "spb"
|
||||
assert row.amount is None
|
||||
assert row.payload["nominal_value"] == "0.1"
|
||||
assert row.payload["listing_date"] == "2024-11-06"
|
||||
assert row.payload["duration"] is None
|
||||
assert row.payload["yield"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["unknown_headers.csv"])
|
||||
def test_unknown_headers_rejected(name):
|
||||
with (FIXTURES / name).open("rb") as f, pytest.raises(SnapshotValidationError):
|
||||
list(adapter().iter_spb_records(f))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [b"not csv", b"PK\x03\x04invalid", b""])
|
||||
def test_invalid_input_rejected(value):
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
list(adapter().iter_spb_records(BytesIO(value)))
|
||||
|
||||
|
||||
def test_genuine_empty_with_valid_headers():
|
||||
with (FIXTURES / "empty.csv").open("rb") as f:
|
||||
assert list(adapter().iter_spb_records(f)) == []
|
||||
|
||||
|
||||
def test_missing_inn_stays_missing_and_name_is_not_identity():
|
||||
row = next(adapter().iter_spb_records(data(e_INN_code="")))
|
||||
assert row.inn == ""
|
||||
assert row.organization_uid is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[("s_RTS_code", ""), ("s_quot_list_in_date", "tomorrow"), ("s_face_value", "NaN")],
|
||||
)
|
||||
def test_malformed_required_or_numeric_fields_rejected(field, value):
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
list(adapter().iter_spb_records(data(**{field: value})))
|
||||
|
||||
|
||||
def test_bounded_rows():
|
||||
with override_settings(EXCHANGE_SPB_MAX_ROWS=0), pytest.raises(
|
||||
SnapshotValidationError
|
||||
):
|
||||
list(adapter().iter_spb_records(data()))
|
||||
|
||||
|
||||
def test_latest_date_not_link_order():
|
||||
html = '<a href="/listing/securities/list">Список ценных бумаг на 05.10.2026</a><a href="/listing/securities/list/">Список ценных бумаг на 02.10.2026</a>'
|
||||
assert adapter().discover_spb_date(html).isoformat() == "2026-10-05"
|
||||
|
||||
|
||||
def test_no_dated_current_link_rejected():
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
adapter().discover_spb_date(
|
||||
'<a href="https://evil.test/list">Список ценных бумаг на 05.10.2026</a>'
|
||||
)
|
||||
|
||||
|
||||
def test_handle_left_open_and_text_inn_not_coerced():
|
||||
handle = data(e_INN_code="0010000025")
|
||||
assert next(adapter().iter_spb_records(handle)).inn == "0010000025"
|
||||
assert not handle.closed
|
||||
|
||||
|
||||
def test_tls_trust_stays_default():
|
||||
session = adapter().registry_session()
|
||||
assert session.verify is True
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_download_checksum_date_private_file_and_three_bounded_reads(tmp_path):
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
module = adapter()
|
||||
body = (FIXTURES / "listing.csv").read_bytes()
|
||||
html = b'<a href="/listing/securities/list">\xd0\xa1\xd0\xbf\xd0\xb8\xd1\x81\xd0\xbe\xd0\xba \xd1\x86\xd0\xb5\xd0\xbd\xd0\xbd\xd1\x8b\xd1\x85 \xd0\xb1\xd1\x83\xd0\xbc\xd0\xb0\xd0\xb3 \xd0\xbd\xd0\xb0 05.10.2026</a>'
|
||||
urls = []
|
||||
|
||||
@contextmanager
|
||||
def response(session, url, **kwargs):
|
||||
urls.append(url)
|
||||
|
||||
class Response:
|
||||
def iter_content(self, chunk_size):
|
||||
yield body if url == module.SPB_EXPORT else html
|
||||
|
||||
yield Response()
|
||||
|
||||
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch.object(
|
||||
module, "registry_response", response
|
||||
):
|
||||
artifact = module.download_spb_artifact(load_batch=1)
|
||||
assert artifact.sha256 == hashlib.sha256(body).hexdigest()
|
||||
assert artifact.source_published_at.isoformat() == "2026-10-05"
|
||||
assert artifact.metadata["input_only"] is True
|
||||
assert len(urls) == 3
|
||||
with pytest.raises(ValueError):
|
||||
_ = artifact.file.url
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_download_size_limit_rejects_artifact(tmp_path):
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
|
||||
module = adapter()
|
||||
html = '<a href="/listing/securities/list">Список ценных бумаг на 05.10.2026</a>'.encode()
|
||||
|
||||
@contextmanager
|
||||
def response(session, url, **kwargs):
|
||||
class Response:
|
||||
def iter_content(self, chunk_size):
|
||||
yield b"oversized" if url == module.SPB_EXPORT else html
|
||||
|
||||
yield Response()
|
||||
|
||||
with override_settings(
|
||||
PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path), EXCHANGE_SPB_MAX_BYTES=1
|
||||
), patch.object(module, "registry_response", response), pytest.raises(
|
||||
SnapshotValidationError
|
||||
):
|
||||
module.download_spb_artifact(load_batch=1)
|
||||
assert ParserSourceArtifact.objects.get().status == "rejected"
|
||||
|
||||
|
||||
def test_redirect_host_rejected_before_second_request():
|
||||
from unittest.mock import Mock
|
||||
|
||||
module = adapter()
|
||||
session = Mock()
|
||||
response = Mock(
|
||||
is_redirect=True, headers={"Location": "https://evil.test/export.csv"}
|
||||
)
|
||||
session.get.return_value = response
|
||||
with pytest.raises(SnapshotValidationError), module.registry_response(
|
||||
session, module.SPB_EXPORT, hosts=module.SPB_HOSTS
|
||||
):
|
||||
pytest.fail("unsafe redirect yielded")
|
||||
assert session.get.call_count == 1
|
||||
Reference in New Issue
Block a user