From 084965b16d30ef188a81134afbc918822bb9c5ff Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 17:04:11 +0200 Subject: [PATCH] feat: ingest verified SPB listing CSV --- src/apps/parsers/artifact_storage.py | 1 + src/apps/parsers/exchange_spb.py | 203 ++++++++++++++++++++++++ tests/apps/parsers/test_exchange_spb.py | 180 +++++++++++++++++++++ 3 files changed, 384 insertions(+) create mode 100644 src/apps/parsers/exchange_spb.py create mode 100644 tests/apps/parsers/test_exchange_spb.py diff --git a/src/apps/parsers/artifact_storage.py b/src/apps/parsers/artifact_storage.py index bc5ff60..3c04904 100644 --- a/src/apps/parsers/artifact_storage.py +++ b/src/apps/parsers/artifact_storage.py @@ -8,6 +8,7 @@ from django.core.files.storage import FileSystemStorage, Storage, default_storag from django.utils.deconstruct import deconstructible PRIVATE_ARTIFACT_PREFIXES = ( + "parser-artifacts/stock_exchange_activity/", "parser-artifacts/checko_leasing/", "parser-artifacts/checko_messages/", "parser-artifacts/checko_licenses/", diff --git a/src/apps/parsers/exchange_spb.py b/src/apps/parsers/exchange_spb.py new file mode 100644 index 0000000..ad68a03 --- /dev/null +++ b/src/apps/parsers/exchange_spb.py @@ -0,0 +1,203 @@ +"""Strict streaming reader for the verified current SPB public CSV export.""" +from __future__ import annotations + +import csv +import re +from collections.abc import Iterator +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from io import BytesIO, TextIOWrapper +from typing import BinaryIO +from urllib.parse import urljoin, urlparse + +from apps.parsers.models import ParserSourceArtifact +from apps.parsers.registry_http import ( + limited_bytes, + registry_response, + registry_session, +) +from apps.parsers.registry_snapshots import ( + SnapshotValidationError, + reject_snapshot, + save_artifact_file, +) +from bs4 import BeautifulSoup +from django.conf import settings +from organizations.source_ingestion import SourceRecordInput + +EXCHANGE_SOURCE = "stock_exchange_activity" +SPB_INDEX = "https://spbexchange.ru/listing/securities/" +SPB_LIST = "https://spbexchange.ru/listing/securities/list/" +SPB_EXPORT = "https://spbexchange.ru/api/listing/v1/list/export" +SPB_HOSTS = frozenset({"spbexchange.ru"}) +SPB_HEADERS = ( + "Row", + "s_level_name", + "e_full_name", + "e_INN_code", + "s_sec_type_name_dop", + "s_sec_form_name_full", + "s_RTS_code", + "s_ISIN_code", + "si_gos_reg_num", + "si_gos_reg_date", + "s_face_value", + "s_face_value_currency", + "s_quot_list_in_date", + "s_segment", + "s_date_defolt", + "s_date_technic_defolt", + "f_is_exchanged", + "s_is_org_discl_restriction", +) + + +def discover_spb_date(html: str) -> date: + dates = [] + for link in BeautifulSoup(html, "html.parser").select("a[href]"): + url = urlparse(urljoin(SPB_INDEX, link["href"])) + match = re.fullmatch( + r"Список ценных бумаг на (\d{2}\.\d{2}\.\d{4})", + link.get_text(" ", strip=True), + ) + if ( + match + and url.scheme == "https" + and url.hostname == "spbexchange.ru" + and url.path.rstrip("/") == "/listing/securities/list" + and not url.username + and not url.password + and url.port in (None, 443) + ): + try: + dates.append(datetime.strptime(match[1], "%d.%m.%Y").date()) + except ValueError as exc: + raise SnapshotValidationError("invalid_spb_publication_date") from exc + if not dates: + raise SnapshotValidationError("spb_current_listing_missing") + return max(dates) + + +def _date(value: str) -> str | None: + if not value or value == "-": + return None + try: + return datetime.strptime(value.split(" ")[0], "%d.%m.%Y").date().isoformat() + except ValueError as exc: + raise SnapshotValidationError("invalid_spb_date") from exc + + +def _nominal(value: str) -> str | None: + if value in ("", "-", "без номинальной стоимости"): + return None + try: + number = Decimal(value.replace(",", ".")) + if not number.is_finite(): + raise InvalidOperation + return format(number, "f") + except InvalidOperation as exc: + raise SnapshotValidationError("invalid_spb_nominal") from exc + + +def iter_spb_records(handle: BinaryIO) -> Iterator[SourceRecordInput]: + text = TextIOWrapper(handle, encoding="utf-8-sig", newline="") + try: + reader = csv.DictReader(text, delimiter=";", strict=True) + if ( + not reader.fieldnames + or len(reader.fieldnames) != len(SPB_HEADERS) + or set(reader.fieldnames) != set(SPB_HEADERS) + ): + raise SnapshotValidationError("unknown_spb_headers") + for number, raw in enumerate(reader, 1): + if number > getattr(settings, "EXCHANGE_SPB_MAX_ROWS", 100000): + raise SnapshotValidationError("spb_row_limit") + if None in raw or any(value is None for value in raw.values()): + raise SnapshotValidationError("invalid_spb_row_width") + fields = {k: v.strip() for k, v in raw.items()} + code = fields["s_RTS_code"] + if not code or len(code) > 200 or not fields["e_full_name"]: + raise SnapshotValidationError("missing_spb_instrument_identity") + listing_date = _date(fields["s_quot_list_in_date"]) + payload = { + "exchange": "spb", + "instrument_id": code, + "issuer_inn": fields["e_INN_code"], + "issuer_provider_id": None, + "isin": fields["s_ISIN_code"] or None, + "instrument_kind": fields["s_sec_type_name_dop"], + "listing_date": listing_date, + "nominal_value": _nominal(fields["s_face_value"]), + "nominal_currency": fields["s_face_value_currency"] or None, + "parameters": { + "registration_number": fields["si_gos_reg_num"] or None, + "registered_on": _date(fields["si_gos_reg_date"]), + "listing_level": fields["s_level_name"], + "category": fields["s_sec_form_name_full"], + }, + "trading": None, + "duration": None, + "yield": None, + "upstream": raw, + "provenance": {"row_number": number}, + } + yield SourceRecordInput( + external_id=f"spb:{code}", + title=code, + organization_name=fields["e_full_name"], + inn=fields["e_INN_code"], + record_type="exchange_instrument", + record_date=listing_date or "", + url=SPB_LIST, + payload=payload, + ) + except (UnicodeError, csv.Error) as exc: + raise SnapshotValidationError("invalid_spb_csv") from exc + finally: + text.detach() + + +def download_spb_artifact( + *, load_batch: int, uploaded_by_id: int | None = None +) -> ParserSourceArtifact: + artifact = ParserSourceArtifact.objects.create( + source=EXCHANGE_SOURCE, + original_name="spb-listing.csv", + content_type="text/csv", + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + metadata={"dataset": "spb", "input_only": True, "source_url": SPB_EXPORT}, + ) + try: + with registry_session() as session: + # Existing EIS trust is deliberately NOT reused: approval was probe-only. + with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response: + before = discover_spb_date( + limited_bytes(response, 4 * 1024**2).decode("utf-8") + ) + with registry_response( + session, + SPB_EXPORT, + hosts=SPB_HOSTS, + params={"fileName": "ListingSecurityList"}, + ) as response: + handle = BytesIO( + limited_bytes( + response, + getattr(settings, "EXCHANGE_SPB_MAX_BYTES", 50 * 1024**2), + ) + ) + save_artifact_file(artifact, handle) + handle.close() + with registry_response(session, SPB_INDEX, hosts=SPB_HOSTS) as response: + after = discover_spb_date( + limited_bytes(response, 4 * 1024**2).decode("utf-8") + ) + if after != before: + raise SnapshotValidationError("spb_publication_changed_during_download") + artifact.source_published_at = before + artifact.save(update_fields=("source_published_at",)) + return artifact + except Exception as exc: + reject_snapshot(artifact, exc) + raise diff --git a/tests/apps/parsers/test_exchange_spb.py b/tests/apps/parsers/test_exchange_spb.py new file mode 100644 index 0000000..61b5838 --- /dev/null +++ b/tests/apps/parsers/test_exchange_spb.py @@ -0,0 +1,180 @@ +"""Public CSV contract derived from the current SPB listing export.""" +import csv +import importlib +from io import BytesIO, StringIO +from pathlib import Path + +import pytest +from apps.parsers.registry_snapshots import SnapshotValidationError +from django.test import override_settings + +FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges" + + +def adapter(): + return importlib.import_module("apps.parsers.exchange_spb") + + +def data(**changes): + rows = list( + csv.reader( + StringIO((FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")), + delimiter=";", + ) + ) + for key, value in changes.items(): + rows[1][rows[0].index(key)] = value + stream = StringIO() + csv.writer(stream, delimiter=";").writerows(rows) + return BytesIO(stream.getvalue().encode()) + + +def test_preserves_inn_vendor_identity_and_decimal(): + row = next(adapter().iter_spb_records(data())) + assert row.inn == "0123456789" + assert row.external_id == "spb:fixture-security-1" + assert row.payload["exchange"] == "spb" + assert row.amount is None + assert row.payload["nominal_value"] == "0.1" + assert row.payload["listing_date"] == "2024-11-06" + assert row.payload["duration"] is None + assert row.payload["yield"] is None + + +@pytest.mark.parametrize("name", ["unknown_headers.csv"]) +def test_unknown_headers_rejected(name): + with (FIXTURES / name).open("rb") as f, pytest.raises(SnapshotValidationError): + list(adapter().iter_spb_records(f)) + + +@pytest.mark.parametrize("value", [b"not csv", b"PK\x03\x04invalid", b""]) +def test_invalid_input_rejected(value): + with pytest.raises(SnapshotValidationError): + list(adapter().iter_spb_records(BytesIO(value))) + + +def test_genuine_empty_with_valid_headers(): + with (FIXTURES / "empty.csv").open("rb") as f: + assert list(adapter().iter_spb_records(f)) == [] + + +def test_missing_inn_stays_missing_and_name_is_not_identity(): + row = next(adapter().iter_spb_records(data(e_INN_code=""))) + assert row.inn == "" + assert row.organization_uid is None + + +@pytest.mark.parametrize( + "field,value", + [("s_RTS_code", ""), ("s_quot_list_in_date", "tomorrow"), ("s_face_value", "NaN")], +) +def test_malformed_required_or_numeric_fields_rejected(field, value): + with pytest.raises(SnapshotValidationError): + list(adapter().iter_spb_records(data(**{field: value}))) + + +def test_bounded_rows(): + with override_settings(EXCHANGE_SPB_MAX_ROWS=0), pytest.raises( + SnapshotValidationError + ): + list(adapter().iter_spb_records(data())) + + +def test_latest_date_not_link_order(): + html = 'Список ценных бумаг на 05.10.2026Список ценных бумаг на 02.10.2026' + assert adapter().discover_spb_date(html).isoformat() == "2026-10-05" + + +def test_no_dated_current_link_rejected(): + with pytest.raises(SnapshotValidationError): + adapter().discover_spb_date( + 'Список ценных бумаг на 05.10.2026' + ) + + +def test_handle_left_open_and_text_inn_not_coerced(): + handle = data(e_INN_code="0010000025") + assert next(adapter().iter_spb_records(handle)).inn == "0010000025" + assert not handle.closed + + +def test_tls_trust_stays_default(): + session = adapter().registry_session() + assert session.verify is True + session.close() + + +@pytest.mark.django_db +def test_download_checksum_date_private_file_and_three_bounded_reads(tmp_path): + import hashlib + from contextlib import contextmanager + from unittest.mock import patch + + module = adapter() + body = (FIXTURES / "listing.csv").read_bytes() + html = b'\xd0\xa1\xd0\xbf\xd0\xb8\xd1\x81\xd0\xbe\xd0\xba \xd1\x86\xd0\xb5\xd0\xbd\xd0\xbd\xd1\x8b\xd1\x85 \xd0\xb1\xd1\x83\xd0\xbc\xd0\xb0\xd0\xb3 \xd0\xbd\xd0\xb0 05.10.2026' + urls = [] + + @contextmanager + def response(session, url, **kwargs): + urls.append(url) + + class Response: + def iter_content(self, chunk_size): + yield body if url == module.SPB_EXPORT else html + + yield Response() + + with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch.object( + module, "registry_response", response + ): + artifact = module.download_spb_artifact(load_batch=1) + assert artifact.sha256 == hashlib.sha256(body).hexdigest() + assert artifact.source_published_at.isoformat() == "2026-10-05" + assert artifact.metadata["input_only"] is True + assert len(urls) == 3 + with pytest.raises(ValueError): + _ = artifact.file.url + + +@pytest.mark.django_db +def test_download_size_limit_rejects_artifact(tmp_path): + from contextlib import contextmanager + from unittest.mock import patch + + from apps.parsers.models import ParserSourceArtifact + + module = adapter() + html = 'Список ценных бумаг на 05.10.2026'.encode() + + @contextmanager + def response(session, url, **kwargs): + class Response: + def iter_content(self, chunk_size): + yield b"oversized" if url == module.SPB_EXPORT else html + + yield Response() + + with override_settings( + PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path), EXCHANGE_SPB_MAX_BYTES=1 + ), patch.object(module, "registry_response", response), pytest.raises( + SnapshotValidationError + ): + module.download_spb_artifact(load_batch=1) + assert ParserSourceArtifact.objects.get().status == "rejected" + + +def test_redirect_host_rejected_before_second_request(): + from unittest.mock import Mock + + module = adapter() + session = Mock() + response = Mock( + is_redirect=True, headers={"Location": "https://evil.test/export.csv"} + ) + session.get.return_value = response + with pytest.raises(SnapshotValidationError), module.registry_response( + session, module.SPB_EXPORT, hosts=module.SPB_HOSTS + ): + pytest.fail("unsafe redirect yielded") + assert session.get.call_count == 1