feat: expose unified exchange activity APIs and exports

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:33:33 +02:00
parent d69ba600d4
commit 3ce9898c0e
30 changed files with 1324 additions and 85 deletions

View File

@@ -61,6 +61,7 @@ def _stage(
row.payload["provenance"] = {
**row.payload.get("provenance", {}),
"artifact_id": str(raw.uid),
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
"source_published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
@@ -107,6 +108,7 @@ def _save_manifest(aggregate, raws, counters):
{
"exchange": raw.metadata["dataset"],
"artifact_id": str(raw.uid),
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
"checksum_sha256": raw.sha256,
"size_bytes": raw.size_bytes,
"published_at": raw.source_published_at.isoformat()

View File

@@ -0,0 +1,33 @@
"""Keep the public combined source disabled pending runtime/dev acceptance."""
from django.db import migrations
def seed(apps, schema_editor):
interval, _ = apps.get_model(
"django_celery_beat", "IntervalSchedule"
).objects.get_or_create(every=1, period="days")
apps.get_model("django_celery_beat", "PeriodicTask").objects.update_or_create(
name="parser:stock-exchange-activity:daily",
defaults={
"task": "parsers.stock_exchange_activity.refresh",
"interval": interval,
"enabled": False,
"args": "[]",
"kwargs": "{}",
"description": "Enable only after verified SPB runtime TLS and full combined dev acceptance.",
},
)
def unseed(apps, schema_editor):
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
name="parser:stock-exchange-activity:daily"
).delete()
class Migration(migrations.Migration):
dependencies = [
("parsers", "0046_stock_exchange_activity"),
("django_celery_beat", "0018_improve_crontab_helptext"),
]
operations = [migrations.RunPython(seed, unseed)]

View File

@@ -1242,6 +1242,19 @@ class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
status = serializers.ChoiceField(choices=("queued", "accepted"), read_only=True)
class SourceRefreshErrorItemSerializer(serializers.Serializer):
code = serializers.CharField(read_only=True)
message = serializers.CharField(read_only=True)
details = serializers.JSONField(read_only=True, required=False)
class SourceRefreshErrorSerializer(serializers.Serializer):
success = serializers.BooleanField(read_only=True)
data = serializers.JSONField(read_only=True, allow_null=True)
errors = SourceRefreshErrorItemSerializer(many=True, read_only=True)
meta = serializers.DictField(read_only=True, allow_null=True)
class SourceRefreshConflictItemSerializer(serializers.Serializer):
code = serializers.ChoiceField(
choices=("refresh_already_running", "upstream_access_not_approved"),

View File

@@ -16,7 +16,7 @@ def attach_snapshot_artifacts(logs) -> None:
load_batch__in={log.batch_id for log in logs},
)
.exclude(
Q(source="fssp_enforcement_proceedings")
Q(source__in=("fssp_enforcement_proceedings", "stock_exchange_activity"))
& Q(metadata__dataset__isnull=False)
)
.order_by("-created_at", "-uid")

View File

@@ -46,6 +46,7 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60
SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60
SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4
SNAPSHOT_CARD_SLUGS = {
"stock_exchange_activity": "stock-exchange-activity",
"checko_licenses": "checko-licenses",
"checko_messages": "checko-messages",
"checko_leasing": "checko-leasing",
@@ -56,6 +57,7 @@ SNAPSHOT_CARD_SLUGS = {
"fedresurs_company_profile": "fedresurs-company-profile",
}
SNAPSHOT_SOURCE_GROUPS = (
"stock_exchange_activity",
"company_licenses",
"company_messages",
"leasing",
@@ -562,6 +564,23 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
),
refresh_interval=timedelta(days=30),
),
SourceCardDefinition(
slug="stock-exchange-activity",
title="Биржевая активность",
description="Инструменты канонических эмитентов СПБ и MOEX.",
order=200,
task_names=("parsers.stock_exchange_activity.refresh",),
source_items=(
SourceItemDefinition(
code="stock_exchange_activity",
title="Биржевая активность",
description="Сведения двух бирж с датами наблюдений.",
parser_source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
refresh_key="stock_exchange_activity",
),
),
refresh_interval=timedelta(days=1),
),
SourceCardDefinition(
slug="fssp-enforcement-proceedings",
title="ФССП: Банк данных исполнительных производств",
@@ -589,6 +608,7 @@ SOURCE_CARD_BY_PARSER_SOURCE = {
if source_item.parser_source
}
GENERIC_RECORD_SOURCES_BY_ITEM_CODE = {
"stock_exchange_activity": ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
"checko_licenses": ParserLoadLog.Source.CHECKO_LICENSES,
"checko_messages": ParserLoadLog.Source.CHECKO_MESSAGES,
"checko_leasing": ParserLoadLog.Source.CHECKO_LEASING,
@@ -1417,9 +1437,18 @@ class SourceCardService:
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
parse_stock_exchange_activity,
)
if definition.slug == "checko-leasing":
if definition.slug == "stock-exchange-activity":
task_specs = (
(
parse_stock_exchange_activity,
"parsers.stock_exchange_activity.refresh",
"stock_exchange_activity",
),
)
elif definition.slug == "checko-leasing":
task_specs = (
(
parse_checko_leasing,

View File

@@ -56,6 +56,18 @@ class ParserSourceDescriptor:
PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
"stock_exchange_activity": ParserSourceDescriptor(
key="stock_exchange_activity",
source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
title="Биржевая активность",
agency="СПБ Биржа и Московская биржа",
data_scope="Инструменты канонических эмитентов двух бирж",
task_name="parsers.stock_exchange_activity.refresh",
mode="public_registry",
access_method="verified_https",
parser_strategy="two_exchange_snapshot",
admin_only=True,
),
"checko_licenses": ParserSourceDescriptor(
key="checko_licenses",
source=ParserLoadLog.Source.CHECKO_LICENSES,

View File

@@ -81,6 +81,7 @@ from apps.parsers.tasks_registry_snapshots import ( # noqa: F401
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
parse_stock_exchange_activity,
)
from celery import shared_task
from django.conf import settings

View File

@@ -65,6 +65,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
if artifact.source in (
"sro_membership_check",
"fssp_enforcement_proceedings",
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
@@ -89,6 +90,7 @@ def _claim_snapshot_job(job, source, task_name):
"sro_membership_check",
"fedresurs_company_profile",
"fssp_enforcement_proceedings",
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
@@ -161,7 +163,13 @@ def _snapshot_error_message(exc: Exception) -> str:
def _attach_rejected_section(job, source, load_log):
if (
source in ("checko_leasing", "checko_messages", "checko_licenses")
source
in (
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
)
and load_log is not None
):
rejected = (
@@ -169,6 +177,7 @@ def _attach_rejected_section(job, source, load_log):
source=source,
load_batch=load_log.batch_id,
status=ParserSourceArtifact.Status.REJECTED,
metadata__dataset__isnull=True,
)
.order_by("-created_at")
.first()
@@ -416,3 +425,21 @@ def parse_checko_licenses(self, *, requested_by_id: int | None = None) -> dict:
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.stock_exchange_activity.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_stock_exchange_activity(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.exchange_snapshot import refresh_stock_exchange_activity
return _run_snapshot(
self,
source="stock_exchange_activity",
refresh=refresh_stock_exchange_activity,
requested_by_id=requested_by_id,
track_progress=True,
)

View File

@@ -73,6 +73,7 @@ from apps.parsers.serializers import (
SourceCardRefreshResponseSerializer,
SourceCardSerializer,
SourceRefreshConflictSerializer,
SourceRefreshErrorSerializer,
SourceTaskStatusListResponseSerializer,
SourceTaskStatusSerializer,
SroParserRunRequestSerializer,
@@ -146,6 +147,7 @@ SOURCE_RESULT_TAGS = {
}
TASKS_BY_NAME = {
"parsers.stock_exchange_activity.refresh": tasks.parse_stock_exchange_activity,
"parsers.checko_licenses.refresh": tasks.parse_checko_licenses,
"parsers.checko_messages.refresh": tasks.parse_checko_messages,
"parsers.checko_leasing.refresh": tasks.parse_checko_leasing,
@@ -1535,9 +1537,11 @@ class SourceCardRefreshView(APIView):
request_body=SourceCardRefreshRequestSerializer,
responses={
202: SourceCardRefreshFrontendResponseSerializer,
400: CommonResponses.BAD_REQUEST,
409: SourceRefreshConflictSerializer,
**ErrorResponses.ADMIN_NOT_FOUND,
400: SourceRefreshErrorSerializer,
403: SourceRefreshErrorSerializer,
404: SourceRefreshErrorSerializer,
409: SourceRefreshConflictSerializer,
},
)
def post(self, request, slug: str):

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,38 @@
"""Typed instrument observations shared by the unified list/detail contract."""
from rest_framework import serializers
class ExchangeMetricSerializer(serializers.Serializer):
value = serializers.CharField(allow_null=True)
unit = serializers.CharField(allow_null=True)
calculated_on = serializers.DateField(allow_null=True)
class ExchangeTradingSerializer(serializers.Serializer):
traded_on = serializers.DateField()
board_id = serializers.CharField()
upstream = serializers.JSONField()
class ExchangeInstrumentPayloadSerializer(serializers.Serializer):
exchange = serializers.ChoiceField(choices=("moex", "spb"))
instrument_id = serializers.CharField()
issuer_inn = serializers.CharField()
issuer_provider_id = serializers.CharField(allow_null=True)
isin = serializers.CharField(allow_null=True)
instrument_kind = serializers.CharField()
listing_date = serializers.DateField(allow_null=True)
nominal_value = serializers.CharField(allow_null=True)
nominal_currency = serializers.CharField(allow_null=True)
parameters = serializers.JSONField()
trading = ExchangeTradingSerializer(allow_null=True)
duration = ExchangeMetricSerializer(allow_null=True)
yield_value = ExchangeMetricSerializer(source="yield", allow_null=True)
provenance = serializers.JSONField()
upstream = serializers.JSONField()
def get_fields(self):
fields = super().get_fields()
fields["yield"] = fields.pop("yield_value")
fields["yield"].source = None
return fields

View File

@@ -326,6 +326,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
CHECKO_PAYLOAD_SERIALIZERS,
)
if obj.source == "stock_exchange_activity":
from organizations.exchange_payload_serializers import (
ExchangeInstrumentPayloadSerializer,
)
return ExchangeInstrumentPayloadSerializer(payload).data
if obj.source in CHECKO_PAYLOAD_SERIALIZERS:
return CHECKO_PAYLOAD_SERIALIZERS[obj.source](payload).data
if obj.source == "fssp_enforcement_proceedings":

View File

@@ -49,6 +49,7 @@ EXPORT_FORMAT_JSON = "json"
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
ALL_HISTORY_SOURCE_GROUPS = {
"stock_exchange_activity",
"leasing",
"company_messages",
"company_licenses",
@@ -83,6 +84,7 @@ EXPORT_PROVIDER_NAME_PATTERN = re.compile(
)
SNAPSHOT_EXPORT_SOURCES = {
"stock_exchange_activity": "stock_exchange_activity",
"company_licenses": "checko_licenses",
"company_messages": "checko_messages",
"leasing": "checko_leasing",
@@ -96,6 +98,7 @@ SNAPSHOT_EXPORT_SOURCES = {
GENERATION_ID_PATTERN = re.compile(r"[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}")
SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = {
"stock_exchange_activity": "stock-exchange-activity",
"company_licenses": "checko-licenses",
"company_messages": "checko-messages",
"leasing": "checko-leasing",
@@ -1300,7 +1303,13 @@ def _build_record_row(
}
public_row = (
row
if record.source in ("checko_leasing", "checko_messages", "checko_licenses")
if record.source
in (
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
)
else _sanitize_export_provider_value(row)
)
serialized_row = {

View File

@@ -27,6 +27,7 @@ from drf_yasg import openapi
from rest_framework.exceptions import NotFound
REGISTRY_GROUPS = (
"stock_exchange_activity",
"company_licenses",
"company_messages",
"leasing",
@@ -38,6 +39,7 @@ REGISTRY_GROUPS = (
"fedresurs_company_profile",
)
REGISTRY_SOURCE_GROUPS = {
"stock_exchange_activity": "stock_exchange_activity",
"checko_licenses": "company_licenses",
"checko_messages": "company_messages",
"checko_leasing": "leasing",
@@ -50,6 +52,11 @@ REGISTRY_SOURCE_GROUPS = {
}
REGISTRY_ORDERING_FIELDS = (
"payload__exchange",
"payload__instrument_id",
"payload__instrument_kind",
"payload__listing_date",
"payload__trading__traded_on",
"payload__publication_date",
"payload__message_type",
"payload__license_number",
@@ -87,6 +94,8 @@ REGISTRY_ORDERING_FIELDS = (
"payload__captured_at",
)
EXACT_FILTERS = {
"exchange": "payload__exchange",
"instrument_kind": "payload__instrument_kind",
"message_type": "payload__message_type",
"license_number": "payload__license_number",
"license_authority": "payload__authority",
@@ -273,6 +282,8 @@ def _validate_aliases_and_booleans(values, error):
def _validate_choices(values, error):
if "exchange" in values and values["exchange"] not in ("moex", "spb"):
error("exchange")
group = values.get("source_group") or REGISTRY_SOURCE_GROUPS.get(
values.get("source")
)

View File

@@ -44,6 +44,7 @@ CANONICAL_ONLY_TEST_SOURCES = {
ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK,
}
FILE_UPLOAD_ONLY_TEST_SOURCES = {
ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
"checko_leasing",
"checko_messages",
"checko_licenses",