feat: ingest verified SPB listing CSV
This commit is contained in:
180
tests/apps/parsers/test_exchange_spb.py
Normal file
180
tests/apps/parsers/test_exchange_spb.py
Normal file
@@ -0,0 +1,180 @@
|
||||
"""Public CSV contract derived from the current SPB listing export."""
|
||||
import csv
|
||||
import importlib
|
||||
from io import BytesIO, StringIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from django.test import override_settings
|
||||
|
||||
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
||||
|
||||
|
||||
def adapter():
|
||||
return importlib.import_module("apps.parsers.exchange_spb")
|
||||
|
||||
|
||||
def data(**changes):
|
||||
rows = list(
|
||||
csv.reader(
|
||||
StringIO((FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")),
|
||||
delimiter=";",
|
||||
)
|
||||
)
|
||||
for key, value in changes.items():
|
||||
rows[1][rows[0].index(key)] = value
|
||||
stream = StringIO()
|
||||
csv.writer(stream, delimiter=";").writerows(rows)
|
||||
return BytesIO(stream.getvalue().encode())
|
||||
|
||||
|
||||
def test_preserves_inn_vendor_identity_and_decimal():
|
||||
row = next(adapter().iter_spb_records(data()))
|
||||
assert row.inn == "0123456789"
|
||||
assert row.external_id == "spb:fixture-security-1"
|
||||
assert row.payload["exchange"] == "spb"
|
||||
assert row.amount is None
|
||||
assert row.payload["nominal_value"] == "0.1"
|
||||
assert row.payload["listing_date"] == "2024-11-06"
|
||||
assert row.payload["duration"] is None
|
||||
assert row.payload["yield"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["unknown_headers.csv"])
|
||||
def test_unknown_headers_rejected(name):
|
||||
with (FIXTURES / name).open("rb") as f, pytest.raises(SnapshotValidationError):
|
||||
list(adapter().iter_spb_records(f))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [b"not csv", b"PK\x03\x04invalid", b""])
|
||||
def test_invalid_input_rejected(value):
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
list(adapter().iter_spb_records(BytesIO(value)))
|
||||
|
||||
|
||||
def test_genuine_empty_with_valid_headers():
|
||||
with (FIXTURES / "empty.csv").open("rb") as f:
|
||||
assert list(adapter().iter_spb_records(f)) == []
|
||||
|
||||
|
||||
def test_missing_inn_stays_missing_and_name_is_not_identity():
|
||||
row = next(adapter().iter_spb_records(data(e_INN_code="")))
|
||||
assert row.inn == ""
|
||||
assert row.organization_uid is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[("s_RTS_code", ""), ("s_quot_list_in_date", "tomorrow"), ("s_face_value", "NaN")],
|
||||
)
|
||||
def test_malformed_required_or_numeric_fields_rejected(field, value):
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
list(adapter().iter_spb_records(data(**{field: value})))
|
||||
|
||||
|
||||
def test_bounded_rows():
|
||||
with override_settings(EXCHANGE_SPB_MAX_ROWS=0), pytest.raises(
|
||||
SnapshotValidationError
|
||||
):
|
||||
list(adapter().iter_spb_records(data()))
|
||||
|
||||
|
||||
def test_latest_date_not_link_order():
|
||||
html = '<a href="/listing/securities/list">Список ценных бумаг на 05.10.2026</a><a href="/listing/securities/list/">Список ценных бумаг на 02.10.2026</a>'
|
||||
assert adapter().discover_spb_date(html).isoformat() == "2026-10-05"
|
||||
|
||||
|
||||
def test_no_dated_current_link_rejected():
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
adapter().discover_spb_date(
|
||||
'<a href="https://evil.test/list">Список ценных бумаг на 05.10.2026</a>'
|
||||
)
|
||||
|
||||
|
||||
def test_handle_left_open_and_text_inn_not_coerced():
|
||||
handle = data(e_INN_code="0010000025")
|
||||
assert next(adapter().iter_spb_records(handle)).inn == "0010000025"
|
||||
assert not handle.closed
|
||||
|
||||
|
||||
def test_tls_trust_stays_default():
|
||||
session = adapter().registry_session()
|
||||
assert session.verify is True
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_download_checksum_date_private_file_and_three_bounded_reads(tmp_path):
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
module = adapter()
|
||||
body = (FIXTURES / "listing.csv").read_bytes()
|
||||
html = b'<a href="/listing/securities/list">\xd0\xa1\xd0\xbf\xd0\xb8\xd1\x81\xd0\xbe\xd0\xba \xd1\x86\xd0\xb5\xd0\xbd\xd0\xbd\xd1\x8b\xd1\x85 \xd0\xb1\xd1\x83\xd0\xbc\xd0\xb0\xd0\xb3 \xd0\xbd\xd0\xb0 05.10.2026</a>'
|
||||
urls = []
|
||||
|
||||
@contextmanager
|
||||
def response(session, url, **kwargs):
|
||||
urls.append(url)
|
||||
|
||||
class Response:
|
||||
def iter_content(self, chunk_size):
|
||||
yield body if url == module.SPB_EXPORT else html
|
||||
|
||||
yield Response()
|
||||
|
||||
with override_settings(PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path)), patch.object(
|
||||
module, "registry_response", response
|
||||
):
|
||||
artifact = module.download_spb_artifact(load_batch=1)
|
||||
assert artifact.sha256 == hashlib.sha256(body).hexdigest()
|
||||
assert artifact.source_published_at.isoformat() == "2026-10-05"
|
||||
assert artifact.metadata["input_only"] is True
|
||||
assert len(urls) == 3
|
||||
with pytest.raises(ValueError):
|
||||
_ = artifact.file.url
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_download_size_limit_rejects_artifact(tmp_path):
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
|
||||
module = adapter()
|
||||
html = '<a href="/listing/securities/list">Список ценных бумаг на 05.10.2026</a>'.encode()
|
||||
|
||||
@contextmanager
|
||||
def response(session, url, **kwargs):
|
||||
class Response:
|
||||
def iter_content(self, chunk_size):
|
||||
yield b"oversized" if url == module.SPB_EXPORT else html
|
||||
|
||||
yield Response()
|
||||
|
||||
with override_settings(
|
||||
PARSER_PRIVATE_ARTIFACT_ROOT=str(tmp_path), EXCHANGE_SPB_MAX_BYTES=1
|
||||
), patch.object(module, "registry_response", response), pytest.raises(
|
||||
SnapshotValidationError
|
||||
):
|
||||
module.download_spb_artifact(load_batch=1)
|
||||
assert ParserSourceArtifact.objects.get().status == "rejected"
|
||||
|
||||
|
||||
def test_redirect_host_rejected_before_second_request():
|
||||
from unittest.mock import Mock
|
||||
|
||||
module = adapter()
|
||||
session = Mock()
|
||||
response = Mock(
|
||||
is_redirect=True, headers={"Location": "https://evil.test/export.csv"}
|
||||
)
|
||||
session.get.return_value = response
|
||||
with pytest.raises(SnapshotValidationError), module.registry_response(
|
||||
session, module.SPB_EXPORT, hosts=module.SPB_HOSTS
|
||||
):
|
||||
pytest.fail("unsafe redirect yielded")
|
||||
assert session.get.call_count == 1
|
||||
Reference in New Issue
Block a user