feat: expose unified exchange activity APIs and exports

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:33:33 +02:00
parent d69ba600d4
commit 3ce9898c0e
30 changed files with 1324 additions and 85 deletions

View File

@@ -36,7 +36,7 @@ def test_discriminated_records_keep_named_ordering_and_old_operations(contract):
schemas = contract["components"]["schemas"]
for name in ("OrganizationSourceRecord", "OrganizationSourceRecordList"):
record = schemas[name]
assert len(record["oneOf"]) == 10
assert len(record["oneOf"]) == 11
assert record["discriminator"]["propertyName"] == "source_group"
for group, record_type in (
("budget_process_registry", "budget_registry_organization"),

View File

@@ -0,0 +1,294 @@
"""Unified exchange card, typed observations and source API/export parity."""
import importlib
from unittest.mock import patch
import pytest
from apps.core.models import BackgroundJob
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact
from apps.parsers.snapshot_metadata import get_snapshot_artifact
from apps.parsers.source_cards import SOURCE_CARD_DEFINITIONS, SourceCardService
from core.celery import app as celery_app
from django.core.cache import cache
from django.test import TestCase
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_groups import get_source_group_descriptor
from organizations.source_record_export import _build_record_row, _source_group_queryset
from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
SOURCE = "stock_exchange_activity"
def exchange_payload(exchange="moex"):
return {
"exchange": exchange,
"instrument_id": "FIXTURE1",
"issuer_inn": "7707083893",
"issuer_provider_id": "484" if exchange == "moex" else None,
"isin": "RU0000000001",
"instrument_kind": "exchange_bond",
"listing_date": "2020-01-01",
"nominal_value": "1000.00",
"nominal_currency": "RUB",
"parameters": {"unknown": {"keep": [None, False, 0, "001"]}},
"trading": {
"traded_on": "2026-10-05",
"board_id": "TQCB",
"upstream": {"LAST": "101.2300"},
}
if exchange == "moex"
else None,
"duration": {"value": "260", "unit": "days", "calculated_on": "2026-10-02"}
if exchange == "moex"
else None,
"yield": {"value": None, "unit": None, "calculated_on": "2026-10-02"}
if exchange == "moex"
else None,
"provenance": {"artifact_id": "00000000-0000-0000-0000-000000000001"},
"upstream": {"unknown": [0, None, False, "0007"]},
}
class ExchangeContractTests(TestCase):
def setUp(self):
cache.clear()
self.org = Organization.objects.create(
name="Canonical issuer",
inn="7707083893",
ogrn="1027700132195",
okpo="00032537",
directory_imported_at=timezone.now(),
)
self.extension = get_source_group_descriptor(
SOURCE
).extension_model.objects.create(organization=self.org)
ParserSourceArtifact.objects.create(
source=SOURCE,
load_batch=1,
status="published",
original_name="exchange-manifest.json",
metadata={"snapshot_organizations_count": 1},
published_count=2,
)
self.rows = [
OrganizationSourceRecord.objects.create(
extension=self.extension,
source=SOURCE,
record_type="exchange_instrument",
external_id=exchange + ":FIXTURE1",
title="Fixture",
load_batch=1,
payload=exchange_payload(exchange),
)
for exchange in ("moex", "spb")
]
self.client = APIClient()
self.client.force_authenticate(UserFactory.create_user())
self.url = "/api/v2/organization-source-records/"
def test_exact_unified_source_card_and_list_detail_export(self):
descriptor = get_source_group_descriptor(SOURCE)
assert descriptor.source_group == SOURCE
assert descriptor.record_type == "exchange_instrument"
assert (
len(
[
c
for c in SOURCE_CARD_DEFINITIONS
if c.slug == "stock-exchange-activity"
]
)
== 1
)
response = self.client.get(self.url, {"source_group": SOURCE})
assert response.status_code == 200, response.data
assert len(response.data["data"]) == 2
for row in response.data["data"]:
assert row["amount"] is None
assert row["organization"]["okpo"] == "00032537"
detail = self.client.get(self.url + row["uid"] + "/")
assert detail.data["payload"] == row["payload"]
record = _source_group_queryset(SOURCE, export_year=2026).get(
uid=row["uid"]
)
exported = _build_record_row(record, include_financial_lines=False)
assert exported["payload.exchange"] == row["payload"]["exchange"]
assert exported["payload.nominal_value"] == "1000.00"
assert exported["amount"] is None
if row["payload"]["exchange"] == "moex":
assert exported["payload.duration.value"] == "260"
assert exported["payload.duration.unit"] == "days"
assert row["payload"]["yield"]["value"] is None
def test_exchange_filters_ordering_and_invalid_exchange(self):
response = self.client.get(
self.url,
{
"source_group": SOURCE,
"exchange": "moex",
"instrument_kind": "exchange_bond",
"ordering": "-payload__instrument_id",
},
)
assert response.status_code == 200, response.data
assert len(response.data["data"]) == 1
assert response.data["data"][0]["payload"]["exchange"] == "moex"
assert (
self.client.get(
self.url, {"source_group": SOURCE, "exchange": "unknown"}
).status_code
== 400
)
def test_input_artifact_never_overrides_aggregate_provenance(self):
log = ParserLoadLog.objects.create(source=SOURCE, batch_id=1, status="success")
child = ParserSourceArtifact.objects.create(
source=SOURCE,
load_batch=1,
status="parsed",
original_name="moex.ndjson",
metadata={"dataset": "moex", "input_only": True},
)
assert get_snapshot_artifact(log).uid != child.uid
def test_invalid_exchange_rejected_by_named_serializer():
module = importlib.import_module("organizations.exchange_payload_serializers")
serializer = module.ExchangeInstrumentPayloadSerializer(
data=exchange_payload("invalid")
)
assert not serializer.is_valid()
assert "exchange" in serializer.errors
@pytest.mark.django_db
def test_one_job_refresh_run_shared_guard_permissions():
importlib.import_module("apps.parsers.tasks")
task = celery_app.tasks["parsers.stock_exchange_activity.refresh"]
client = APIClient()
client.force_authenticate(UserFactory.create_user())
urls = [
"/api/v1/sources/stock-exchange-activity/refresh/",
"/api/v1/parsers/run/stock_exchange_activity/",
]
assert all(client.post(url, {}, format="json").status_code == 403 for url in urls)
client.force_authenticate(UserFactory.create_user(is_staff=True))
with patch.object(task, "apply_async"):
assert client.post(urls[0], {}, format="json").status_code == 202
assert client.post(urls[1], {}, format="json").status_code == 409
assert BackgroundJob.objects.count() == 1
@pytest.mark.django_db
def test_worker_success_propagates_combined_manifest_counters_and_failure_keeps_success():
from apps.parsers import exchange_snapshot
from apps.parsers.registry_snapshots import SnapshotResult, SnapshotValidationError
importlib.import_module("apps.parsers.tasks")
task = celery_app.tasks["parsers.stock_exchange_activity.refresh"]
cache.clear()
def refresh(**kwargs):
artifact = ParserSourceArtifact.objects.create(
source=SOURCE,
load_batch=kwargs["load_batch"],
status="published",
original_name="exchange-manifest.json",
parsed_count=5,
published_count=2,
quarantined_count=1,
metadata={
"snapshot_organizations_count": 1,
"adapter_counters": {"spb": {"parsed": 3}, "moex": {"parsed": 2}},
"inputs": [{"exchange": "spb"}, {"exchange": "moex"}],
},
)
result = SnapshotResult(parsed=5, published=2, quarantined=1, skipped=2)
kwargs["on_publish"](artifact, result)
return artifact, result
with patch.object(exchange_snapshot, "refresh_stock_exchange_activity", refresh):
result = task.apply(throw=True).result
assert result["published_records_count"] == 2
assert result["adapter_counters"]["spb"]["parsed"] == 3
assert len(result["inputs"]) == 2
before = SourceCardService.get_card("stock-exchange-activity")[
"latest_success_load"
]["batch_id"]
def fail(**kwargs):
ParserSourceArtifact.objects.create(
source=SOURCE,
load_batch=kwargs["load_batch"],
status="rejected",
original_name="exchange-manifest.json",
metadata={
"rejection_reason": "synthetic_failure",
"inputs": [{"exchange": "spb"}, {"exchange": "moex"}],
},
)
ParserSourceArtifact.objects.create(
source=SOURCE,
load_batch=kwargs["load_batch"],
status="rejected",
original_name="moex.ndjson",
metadata={"dataset": "moex", "input_only": True},
)
raise SnapshotValidationError("synthetic_failure")
with patch.object(
exchange_snapshot, "refresh_stock_exchange_activity", fail
), pytest.raises(SnapshotValidationError):
task.apply(throw=True)
card = SourceCardService.get_card("stock-exchange-activity")
assert card["latest_success_load"]["batch_id"] == before
job = BackgroundJob.objects.order_by("-created_at").first()
assert job.result["rejection_reason"] == "synthetic_failure"
assert len(job.result["inputs"]) == 2
def test_exchange_discriminator_and_nullable_metrics_in_generated_contract():
import json
from pathlib import Path
document = json.loads(
(Path(__file__).parents[3] / "src/core/openapi.json").read_text()
)
schemas = document["components"]["schemas"]
for name in ("OrganizationSourceRecord", "OrganizationSourceRecordList"):
mapping = schemas[name]["discriminator"]["mapping"]
variant = schemas[mapping[SOURCE].rsplit("/", 1)[1]]
assert variant["properties"]["record_type"]["enum"] == ["exchange_instrument"]
payload = schemas["ExchangeInstrumentListPayload"]
assert payload["properties"]["duration"]["nullable"] is True
assert payload["properties"]["yield"]["nullable"] is True
@pytest.mark.django_db
@pytest.mark.parametrize(
"module_name",
[
"apps.parsers.migrations.0043_fssp_disabled_schedule",
"apps.parsers.migrations.0045_checko_disabled_schedules",
"apps.parsers.migrations.0047_exchange_disabled_schedule",
],
)
def test_new_schedule_seed_always_disabled(module_name):
from django.apps import apps
from django_celery_beat.models import PeriodicTask
module = importlib.import_module(module_name)
module.seed(apps, None)
tasks = PeriodicTask.objects.filter(
task__in=[
"parsers.fssp_enforcement_proceedings.refresh",
"parsers.checko_leasing.refresh",
"parsers.checko_messages.refresh",
"parsers.checko_licenses.refresh",
"parsers.stock_exchange_activity.refresh",
]
)
assert tasks.exists()
assert not tasks.filter(enabled=True).exists()

View File

@@ -2,8 +2,10 @@
import json
import os
from importlib import import_module
from io import StringIO
from pathlib import Path
from unittest.mock import patch
from uuid import uuid4
import pytest
@@ -20,6 +22,7 @@ from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_groups import get_source_group_descriptor
from rest_framework.test import APIClient
from tests.apps.organizations.test_exchange_contract import exchange_payload
from tests.apps.parsers.test_checko_sections import client_with, fixture
from tests.apps.parsers.test_registry_payload_contract import full_budget_fixture
from tests.apps.parsers.test_registry_snapshots import _xml
@@ -163,7 +166,29 @@ def _canonical_records():
},
)
)
return [budget, sme, sro, fssp, *sections], raw_budget
exchange = _create_record(
fssp_org,
"stock_exchange_activity",
{
"title": "Synthetic instrument",
"record_date": "2026-10-05",
"amount": None,
"status": "",
"url": "https://iss.moex.com/iss/securities/FIXTURE1.json",
"payload": exchange_payload(),
},
)
spb = OrganizationSourceRecord.objects.create(
extension=exchange.extension,
source="stock_exchange_activity",
record_type="exchange_instrument",
external_id="spb:FIXTURE1",
load_batch=9001,
title="Synthetic SPB instrument",
url="https://spbexchange.ru/listing/securities/list/",
payload=exchange_payload("spb"),
)
return [budget, sme, sro, fssp, *sections, exchange, spb], raw_budget
@pytest.mark.django_db
@@ -184,22 +209,31 @@ def test_source_record_runtime_fixtures():
return body
mixed = capture("mixed-list", "list", client.get(list_url))
assert len(mixed["data"]) == 7
assert mixed["meta"]["pagination"]["total_count"] == 7
assert len(mixed["data"]) == 9
assert mixed["meta"]["pagination"]["total_count"] == 9
details = {}
listed_sources = set()
for record in records:
listing = capture(
f"{record.source}-list",
"list",
client.get(list_url, {"source": record.source}),
)
assert len(listing["data"]) == 1
if record.source not in listed_sources:
listing = capture(
f"{record.source}-list",
"list",
client.get(list_url, {"source": record.source}),
)
assert len(listing["data"]) == (
2 if record.source == "stock_exchange_activity" else 1
)
listed_sources.add(record.source)
detail_url = reverse(
"api_v2:organizations:organization-source-records-detail",
kwargs={"uid": record.uid},
)
details[record.source] = capture(
f"{record.source}-detail", "detail", client.get(detail_url)
f"{record.source}-spb-detail"
if record is records[-1]
else f"{record.source}-detail",
"detail",
client.get(detail_url),
)
budget_payload = details["budget_ubpandnubp"]["payload"]
assert budget_payload["upstream"] == raw_budget
@@ -238,6 +272,57 @@ def test_source_record_runtime_fixtures():
status,
)
from core.celery import app as celery_app
import_module("apps.parsers.tasks")
sources = [
("fssp_enforcement_proceedings", "fssp-enforcement-proceedings"),
("checko_leasing", "checko-leasing"),
("checko_messages", "checko-messages"),
("checko_licenses", "checko-licenses"),
("stock_exchange_activity", "stock-exchange-activity"),
]
for source, slug in sources:
capture(
f"{source}-refresh-forbidden",
"refresh",
client.post(f"/api/v1/sources/{slug}/refresh/", {}, format="json"),
403,
)
client.force_authenticate(UserFactory.create_user(is_staff=True))
for source, slug in sources:
refresh_url = f"/api/v1/sources/{slug}/refresh/"
capture(
f"{source}-refresh-invalid",
"refresh",
client.post(
refresh_url,
{"params": {"file_url": "https://foreign.invalid"}},
format="json",
),
400,
)
task = celery_app.tasks[f"parsers.{source}.refresh"]
with patch.object(task, "apply_async"):
capture(
f"{source}-refresh-accepted",
"refresh",
client.post(refresh_url, {}, format="json"),
202,
)
capture(
f"{source}-refresh-conflict",
"refresh",
client.post(refresh_url, {}, format="json"),
409,
)
capture(
"missing-refresh",
"refresh",
client.post("/api/v1/sources/unknown-source/refresh/", {}, format="json"),
404,
)
# Ordinary pytest runs create no artifact. CI opts in with an explicit path.
destination = os.environ.get("OPENAPI_RUNTIME_FIXTURES")
if destination:

View File

@@ -69,6 +69,7 @@ def issue(client, group="budget_process_registry", file_format="json"):
@pytest.mark.parametrize(
"source,group",
[
("stock_exchange_activity", "stock_exchange_activity"),
("checko_leasing", "leasing"),
("checko_messages", "company_messages"),
("checko_licenses", "company_licenses"),
@@ -220,7 +221,7 @@ def test_sro_export_keeps_all_years_and_identifier_strings(export_client, file_f
record_date="2001-01-01",
)
generation = exports.build_source_record_export_artifacts()
assert generation.artifacts_count == 61
assert generation.artifacts_count == 64
response = issue(export_client, "sro_membership", file_format)
assert response.status_code == 201
package = exports.consume_source_record_export_download_ticket(

View File

@@ -128,9 +128,9 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
call_command("build_source_record_exports", stdout=command_output)
generation = load_current_source_record_export_generation()
self.assertEqual(generation.artifacts_count, 61)
self.assertEqual(generation.artifacts_count, 64)
self.assertEqual(generation.export_year, timezone.localdate().year)
self.assertIn('"artifacts_count": 61', command_output.getvalue())
self.assertIn('"artifacts_count": 64', command_output.getvalue())
def test_sanctions_xlsx_export_uses_source_contract_columns_and_boolean_flags(self):
organization = Organization.objects.create(
@@ -641,7 +641,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertEqual(
response["X-Source-Export-Generated-At"], generation.generated_at
)
self.assertEqual(generation.artifacts_count, 61)
self.assertEqual(generation.artifacts_count, 64)
self.assertIn(
'filename="planned-inspections__financial-indicators_',
response["Content-Disposition"],
@@ -840,8 +840,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
key=lambda item: item.file_name,
)
self.assertEqual(generation.artifacts_count, 61)
self.assertEqual(generation.files_count, 62)
self.assertEqual(generation.artifacts_count, 64)
self.assertEqual(generation.files_count, 65)
self.assertEqual(
[item.file_name for item in artifacts],
[
@@ -1065,7 +1065,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
first_generation = build_source_record_export_artifacts()
current_generation = load_current_source_record_export_generation()
self.assertEqual(first_generation.artifacts_count, 61)
self.assertEqual(first_generation.artifacts_count, 64)
self.assertEqual(
current_generation.generation_id, first_generation.generation_id
)

View File

@@ -147,7 +147,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase):
result = refresh_source_record_export_artifacts()
self.assertEqual(result["status"], "success")
self.assertEqual(result["artifacts_count"], 61)
self.assertEqual(result["artifacts_count"], 64)
self.assertEqual(result["export_year"], timezone.localdate().year)
self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))

View File

@@ -83,6 +83,7 @@ class TestCompaniesCommandsTest(TestCase):
ParserLoadLog.Source.CHECKO_LEASING,
ParserLoadLog.Source.CHECKO_MESSAGES,
ParserLoadLog.Source.CHECKO_LICENSES,
ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
}
}
for company in companies:
@@ -277,7 +278,7 @@ class TestCompaniesCommandsTest(TestCase):
)
self.assertEqual(
OrganizationSourceRecord.objects.count(),
20 * (len(ParserLoadLog.Source) - 7),
20 * (len(ParserLoadLog.Source) - 8),
)
def test_create_replaces_stale_source_record_for_same_legacy_row(self):

View File

@@ -31,6 +31,7 @@ def _parallel(actions):
@pytest.mark.parametrize(
"slug,source",
[
("stock-exchange-activity", "stock_exchange_activity"),
("checko-leasing", "checko_leasing"),
("checko-messages", "checko_messages"),
("checko-licenses", "checko_licenses"),

View File

@@ -100,6 +100,7 @@ class SourceCardServiceUnitTest(TestCase):
"checko-leasing",
"checko-messages",
"checko-licenses",
"stock-exchange-activity",
"fssp-enforcement-proceedings",
],
)
@@ -126,6 +127,7 @@ class SourceCardServiceUnitTest(TestCase):
"Лизинг",
"Сообщения организаций",
"Лицензии организаций",
"Биржевая активность",
"ФССП: Банк данных исполнительных производств",
],
)