feat: expose unified exchange activity APIs and exports
This commit is contained in:
@@ -61,6 +61,7 @@ def _stage(
|
||||
row.payload["provenance"] = {
|
||||
**row.payload.get("provenance", {}),
|
||||
"artifact_id": str(raw.uid),
|
||||
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
|
||||
"source_published_at": raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
@@ -107,6 +108,7 @@ def _save_manifest(aggregate, raws, counters):
|
||||
{
|
||||
"exchange": raw.metadata["dataset"],
|
||||
"artifact_id": str(raw.uid),
|
||||
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
|
||||
"checksum_sha256": raw.sha256,
|
||||
"size_bytes": raw.size_bytes,
|
||||
"published_at": raw.source_published_at.isoformat()
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Keep the public combined source disabled pending runtime/dev acceptance."""
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
def seed(apps, schema_editor):
|
||||
interval, _ = apps.get_model(
|
||||
"django_celery_beat", "IntervalSchedule"
|
||||
).objects.get_or_create(every=1, period="days")
|
||||
apps.get_model("django_celery_beat", "PeriodicTask").objects.update_or_create(
|
||||
name="parser:stock-exchange-activity:daily",
|
||||
defaults={
|
||||
"task": "parsers.stock_exchange_activity.refresh",
|
||||
"interval": interval,
|
||||
"enabled": False,
|
||||
"args": "[]",
|
||||
"kwargs": "{}",
|
||||
"description": "Enable only after verified SPB runtime TLS and full combined dev acceptance.",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def unseed(apps, schema_editor):
|
||||
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
|
||||
name="parser:stock-exchange-activity:daily"
|
||||
).delete()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("parsers", "0046_stock_exchange_activity"),
|
||||
("django_celery_beat", "0018_improve_crontab_helptext"),
|
||||
]
|
||||
operations = [migrations.RunPython(seed, unseed)]
|
||||
@@ -1242,6 +1242,19 @@ class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
|
||||
status = serializers.ChoiceField(choices=("queued", "accepted"), read_only=True)
|
||||
|
||||
|
||||
class SourceRefreshErrorItemSerializer(serializers.Serializer):
|
||||
code = serializers.CharField(read_only=True)
|
||||
message = serializers.CharField(read_only=True)
|
||||
details = serializers.JSONField(read_only=True, required=False)
|
||||
|
||||
|
||||
class SourceRefreshErrorSerializer(serializers.Serializer):
|
||||
success = serializers.BooleanField(read_only=True)
|
||||
data = serializers.JSONField(read_only=True, allow_null=True)
|
||||
errors = SourceRefreshErrorItemSerializer(many=True, read_only=True)
|
||||
meta = serializers.DictField(read_only=True, allow_null=True)
|
||||
|
||||
|
||||
class SourceRefreshConflictItemSerializer(serializers.Serializer):
|
||||
code = serializers.ChoiceField(
|
||||
choices=("refresh_already_running", "upstream_access_not_approved"),
|
||||
|
||||
@@ -16,7 +16,7 @@ def attach_snapshot_artifacts(logs) -> None:
|
||||
load_batch__in={log.batch_id for log in logs},
|
||||
)
|
||||
.exclude(
|
||||
Q(source="fssp_enforcement_proceedings")
|
||||
Q(source__in=("fssp_enforcement_proceedings", "stock_exchange_activity"))
|
||||
& Q(metadata__dataset__isnull=False)
|
||||
)
|
||||
.order_by("-created_at", "-uid")
|
||||
|
||||
@@ -46,6 +46,7 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60
|
||||
SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60
|
||||
SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4
|
||||
SNAPSHOT_CARD_SLUGS = {
|
||||
"stock_exchange_activity": "stock-exchange-activity",
|
||||
"checko_licenses": "checko-licenses",
|
||||
"checko_messages": "checko-messages",
|
||||
"checko_leasing": "checko-leasing",
|
||||
@@ -56,6 +57,7 @@ SNAPSHOT_CARD_SLUGS = {
|
||||
"fedresurs_company_profile": "fedresurs-company-profile",
|
||||
}
|
||||
SNAPSHOT_SOURCE_GROUPS = (
|
||||
"stock_exchange_activity",
|
||||
"company_licenses",
|
||||
"company_messages",
|
||||
"leasing",
|
||||
@@ -562,6 +564,23 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
),
|
||||
refresh_interval=timedelta(days=30),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="stock-exchange-activity",
|
||||
title="Биржевая активность",
|
||||
description="Инструменты канонических эмитентов СПБ и MOEX.",
|
||||
order=200,
|
||||
task_names=("parsers.stock_exchange_activity.refresh",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="stock_exchange_activity",
|
||||
title="Биржевая активность",
|
||||
description="Сведения двух бирж с датами наблюдений.",
|
||||
parser_source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
|
||||
refresh_key="stock_exchange_activity",
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=1),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="fssp-enforcement-proceedings",
|
||||
title="ФССП: Банк данных исполнительных производств",
|
||||
@@ -589,6 +608,7 @@ SOURCE_CARD_BY_PARSER_SOURCE = {
|
||||
if source_item.parser_source
|
||||
}
|
||||
GENERIC_RECORD_SOURCES_BY_ITEM_CODE = {
|
||||
"stock_exchange_activity": ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
|
||||
"checko_licenses": ParserLoadLog.Source.CHECKO_LICENSES,
|
||||
"checko_messages": ParserLoadLog.Source.CHECKO_MESSAGES,
|
||||
"checko_leasing": ParserLoadLog.Source.CHECKO_LEASING,
|
||||
@@ -1417,9 +1437,18 @@ class SourceCardService:
|
||||
parse_fssp_enforcement_proceedings,
|
||||
parse_sme_support_recipients,
|
||||
parse_sro_membership,
|
||||
parse_stock_exchange_activity,
|
||||
)
|
||||
|
||||
if definition.slug == "checko-leasing":
|
||||
if definition.slug == "stock-exchange-activity":
|
||||
task_specs = (
|
||||
(
|
||||
parse_stock_exchange_activity,
|
||||
"parsers.stock_exchange_activity.refresh",
|
||||
"stock_exchange_activity",
|
||||
),
|
||||
)
|
||||
elif definition.slug == "checko-leasing":
|
||||
task_specs = (
|
||||
(
|
||||
parse_checko_leasing,
|
||||
|
||||
@@ -56,6 +56,18 @@ class ParserSourceDescriptor:
|
||||
|
||||
|
||||
PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
"stock_exchange_activity": ParserSourceDescriptor(
|
||||
key="stock_exchange_activity",
|
||||
source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
|
||||
title="Биржевая активность",
|
||||
agency="СПБ Биржа и Московская биржа",
|
||||
data_scope="Инструменты канонических эмитентов двух бирж",
|
||||
task_name="parsers.stock_exchange_activity.refresh",
|
||||
mode="public_registry",
|
||||
access_method="verified_https",
|
||||
parser_strategy="two_exchange_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
"checko_licenses": ParserSourceDescriptor(
|
||||
key="checko_licenses",
|
||||
source=ParserLoadLog.Source.CHECKO_LICENSES,
|
||||
|
||||
@@ -81,6 +81,7 @@ from apps.parsers.tasks_registry_snapshots import ( # noqa: F401
|
||||
parse_fssp_enforcement_proceedings,
|
||||
parse_sme_support_recipients,
|
||||
parse_sro_membership,
|
||||
parse_stock_exchange_activity,
|
||||
)
|
||||
from celery import shared_task
|
||||
from django.conf import settings
|
||||
|
||||
@@ -65,6 +65,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||||
if artifact.source in (
|
||||
"sro_membership_check",
|
||||
"fssp_enforcement_proceedings",
|
||||
"stock_exchange_activity",
|
||||
"checko_leasing",
|
||||
"checko_messages",
|
||||
"checko_licenses",
|
||||
@@ -89,6 +90,7 @@ def _claim_snapshot_job(job, source, task_name):
|
||||
"sro_membership_check",
|
||||
"fedresurs_company_profile",
|
||||
"fssp_enforcement_proceedings",
|
||||
"stock_exchange_activity",
|
||||
"checko_leasing",
|
||||
"checko_messages",
|
||||
"checko_licenses",
|
||||
@@ -161,7 +163,13 @@ def _snapshot_error_message(exc: Exception) -> str:
|
||||
|
||||
def _attach_rejected_section(job, source, load_log):
|
||||
if (
|
||||
source in ("checko_leasing", "checko_messages", "checko_licenses")
|
||||
source
|
||||
in (
|
||||
"stock_exchange_activity",
|
||||
"checko_leasing",
|
||||
"checko_messages",
|
||||
"checko_licenses",
|
||||
)
|
||||
and load_log is not None
|
||||
):
|
||||
rejected = (
|
||||
@@ -169,6 +177,7 @@ def _attach_rejected_section(job, source, load_log):
|
||||
source=source,
|
||||
load_batch=load_log.batch_id,
|
||||
status=ParserSourceArtifact.Status.REJECTED,
|
||||
metadata__dataset__isnull=True,
|
||||
)
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
@@ -416,3 +425,21 @@ def parse_checko_licenses(self, *, requested_by_id: int | None = None) -> dict:
|
||||
requested_by_id=requested_by_id,
|
||||
track_progress=True,
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
name="parsers.stock_exchange_activity.refresh",
|
||||
soft_time_limit=8 * 3600,
|
||||
time_limit=8 * 3600 + 300,
|
||||
)
|
||||
def parse_stock_exchange_activity(self, *, requested_by_id: int | None = None) -> dict:
|
||||
from apps.parsers.exchange_snapshot import refresh_stock_exchange_activity
|
||||
|
||||
return _run_snapshot(
|
||||
self,
|
||||
source="stock_exchange_activity",
|
||||
refresh=refresh_stock_exchange_activity,
|
||||
requested_by_id=requested_by_id,
|
||||
track_progress=True,
|
||||
)
|
||||
|
||||
@@ -73,6 +73,7 @@ from apps.parsers.serializers import (
|
||||
SourceCardRefreshResponseSerializer,
|
||||
SourceCardSerializer,
|
||||
SourceRefreshConflictSerializer,
|
||||
SourceRefreshErrorSerializer,
|
||||
SourceTaskStatusListResponseSerializer,
|
||||
SourceTaskStatusSerializer,
|
||||
SroParserRunRequestSerializer,
|
||||
@@ -146,6 +147,7 @@ SOURCE_RESULT_TAGS = {
|
||||
}
|
||||
|
||||
TASKS_BY_NAME = {
|
||||
"parsers.stock_exchange_activity.refresh": tasks.parse_stock_exchange_activity,
|
||||
"parsers.checko_licenses.refresh": tasks.parse_checko_licenses,
|
||||
"parsers.checko_messages.refresh": tasks.parse_checko_messages,
|
||||
"parsers.checko_leasing.refresh": tasks.parse_checko_leasing,
|
||||
@@ -1535,9 +1537,11 @@ class SourceCardRefreshView(APIView):
|
||||
request_body=SourceCardRefreshRequestSerializer,
|
||||
responses={
|
||||
202: SourceCardRefreshFrontendResponseSerializer,
|
||||
400: CommonResponses.BAD_REQUEST,
|
||||
409: SourceRefreshConflictSerializer,
|
||||
**ErrorResponses.ADMIN_NOT_FOUND,
|
||||
400: SourceRefreshErrorSerializer,
|
||||
403: SourceRefreshErrorSerializer,
|
||||
404: SourceRefreshErrorSerializer,
|
||||
409: SourceRefreshConflictSerializer,
|
||||
},
|
||||
)
|
||||
def post(self, request, slug: str):
|
||||
|
||||
File diff suppressed because one or more lines are too long
38
src/organizations/exchange_payload_serializers.py
Normal file
38
src/organizations/exchange_payload_serializers.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""Typed instrument observations shared by the unified list/detail contract."""
|
||||
from rest_framework import serializers
|
||||
|
||||
|
||||
class ExchangeMetricSerializer(serializers.Serializer):
|
||||
value = serializers.CharField(allow_null=True)
|
||||
unit = serializers.CharField(allow_null=True)
|
||||
calculated_on = serializers.DateField(allow_null=True)
|
||||
|
||||
|
||||
class ExchangeTradingSerializer(serializers.Serializer):
|
||||
traded_on = serializers.DateField()
|
||||
board_id = serializers.CharField()
|
||||
upstream = serializers.JSONField()
|
||||
|
||||
|
||||
class ExchangeInstrumentPayloadSerializer(serializers.Serializer):
|
||||
exchange = serializers.ChoiceField(choices=("moex", "spb"))
|
||||
instrument_id = serializers.CharField()
|
||||
issuer_inn = serializers.CharField()
|
||||
issuer_provider_id = serializers.CharField(allow_null=True)
|
||||
isin = serializers.CharField(allow_null=True)
|
||||
instrument_kind = serializers.CharField()
|
||||
listing_date = serializers.DateField(allow_null=True)
|
||||
nominal_value = serializers.CharField(allow_null=True)
|
||||
nominal_currency = serializers.CharField(allow_null=True)
|
||||
parameters = serializers.JSONField()
|
||||
trading = ExchangeTradingSerializer(allow_null=True)
|
||||
duration = ExchangeMetricSerializer(allow_null=True)
|
||||
yield_value = ExchangeMetricSerializer(source="yield", allow_null=True)
|
||||
provenance = serializers.JSONField()
|
||||
upstream = serializers.JSONField()
|
||||
|
||||
def get_fields(self):
|
||||
fields = super().get_fields()
|
||||
fields["yield"] = fields.pop("yield_value")
|
||||
fields["yield"].source = None
|
||||
return fields
|
||||
@@ -326,6 +326,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
|
||||
CHECKO_PAYLOAD_SERIALIZERS,
|
||||
)
|
||||
|
||||
if obj.source == "stock_exchange_activity":
|
||||
from organizations.exchange_payload_serializers import (
|
||||
ExchangeInstrumentPayloadSerializer,
|
||||
)
|
||||
|
||||
return ExchangeInstrumentPayloadSerializer(payload).data
|
||||
if obj.source in CHECKO_PAYLOAD_SERIALIZERS:
|
||||
return CHECKO_PAYLOAD_SERIALIZERS[obj.source](payload).data
|
||||
if obj.source == "fssp_enforcement_proceedings":
|
||||
|
||||
@@ -49,6 +49,7 @@ EXPORT_FORMAT_JSON = "json"
|
||||
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
|
||||
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
|
||||
ALL_HISTORY_SOURCE_GROUPS = {
|
||||
"stock_exchange_activity",
|
||||
"leasing",
|
||||
"company_messages",
|
||||
"company_licenses",
|
||||
@@ -83,6 +84,7 @@ EXPORT_PROVIDER_NAME_PATTERN = re.compile(
|
||||
)
|
||||
|
||||
SNAPSHOT_EXPORT_SOURCES = {
|
||||
"stock_exchange_activity": "stock_exchange_activity",
|
||||
"company_licenses": "checko_licenses",
|
||||
"company_messages": "checko_messages",
|
||||
"leasing": "checko_leasing",
|
||||
@@ -96,6 +98,7 @@ SNAPSHOT_EXPORT_SOURCES = {
|
||||
GENERATION_ID_PATTERN = re.compile(r"[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}")
|
||||
|
||||
SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = {
|
||||
"stock_exchange_activity": "stock-exchange-activity",
|
||||
"company_licenses": "checko-licenses",
|
||||
"company_messages": "checko-messages",
|
||||
"leasing": "checko-leasing",
|
||||
@@ -1300,7 +1303,13 @@ def _build_record_row(
|
||||
}
|
||||
public_row = (
|
||||
row
|
||||
if record.source in ("checko_leasing", "checko_messages", "checko_licenses")
|
||||
if record.source
|
||||
in (
|
||||
"stock_exchange_activity",
|
||||
"checko_leasing",
|
||||
"checko_messages",
|
||||
"checko_licenses",
|
||||
)
|
||||
else _sanitize_export_provider_value(row)
|
||||
)
|
||||
serialized_row = {
|
||||
|
||||
@@ -27,6 +27,7 @@ from drf_yasg import openapi
|
||||
from rest_framework.exceptions import NotFound
|
||||
|
||||
REGISTRY_GROUPS = (
|
||||
"stock_exchange_activity",
|
||||
"company_licenses",
|
||||
"company_messages",
|
||||
"leasing",
|
||||
@@ -38,6 +39,7 @@ REGISTRY_GROUPS = (
|
||||
"fedresurs_company_profile",
|
||||
)
|
||||
REGISTRY_SOURCE_GROUPS = {
|
||||
"stock_exchange_activity": "stock_exchange_activity",
|
||||
"checko_licenses": "company_licenses",
|
||||
"checko_messages": "company_messages",
|
||||
"checko_leasing": "leasing",
|
||||
@@ -50,6 +52,11 @@ REGISTRY_SOURCE_GROUPS = {
|
||||
}
|
||||
|
||||
REGISTRY_ORDERING_FIELDS = (
|
||||
"payload__exchange",
|
||||
"payload__instrument_id",
|
||||
"payload__instrument_kind",
|
||||
"payload__listing_date",
|
||||
"payload__trading__traded_on",
|
||||
"payload__publication_date",
|
||||
"payload__message_type",
|
||||
"payload__license_number",
|
||||
@@ -87,6 +94,8 @@ REGISTRY_ORDERING_FIELDS = (
|
||||
"payload__captured_at",
|
||||
)
|
||||
EXACT_FILTERS = {
|
||||
"exchange": "payload__exchange",
|
||||
"instrument_kind": "payload__instrument_kind",
|
||||
"message_type": "payload__message_type",
|
||||
"license_number": "payload__license_number",
|
||||
"license_authority": "payload__authority",
|
||||
@@ -273,6 +282,8 @@ def _validate_aliases_and_booleans(values, error):
|
||||
|
||||
|
||||
def _validate_choices(values, error):
|
||||
if "exchange" in values and values["exchange"] not in ("moex", "spb"):
|
||||
error("exchange")
|
||||
group = values.get("source_group") or REGISTRY_SOURCE_GROUPS.get(
|
||||
values.get("source")
|
||||
)
|
||||
|
||||
@@ -44,6 +44,7 @@ CANONICAL_ONLY_TEST_SOURCES = {
|
||||
ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK,
|
||||
}
|
||||
FILE_UPLOAD_ONLY_TEST_SOURCES = {
|
||||
ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
|
||||
"checko_leasing",
|
||||
"checko_messages",
|
||||
"checko_licenses",
|
||||
|
||||
Reference in New Issue
Block a user