feat: add versioned all-available source export catalog

This commit is contained in:
Aleksandr Meshchryakov
2026-10-22 21:30:18 +02:00
parent 1ffdcd0aaf
commit 63872e86fe
28 changed files with 5754 additions and 240 deletions

View File

@@ -15,6 +15,7 @@ from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.services import ParserLoadLogService
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
from celery import shared_task
from django.conf import settings
from django.db import transaction
from django.utils import timezone
@@ -26,6 +27,10 @@ def _refresh_published_exports() -> None:
try:
refresh_source_record_export_artifacts.delay()
if settings.SOURCE_EXPORT_AUTOBUILD:
from organizations.tasks import refresh_source_export_catalog
refresh_source_export_catalog.delay()
except Exception as exc:
# Publication already committed; export readiness remains separately visible.
logger.error("Published source export enqueue failed (%s)", type(exc).__name__)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,47 @@
"""Prepare new all-available export generations without changing annual exports."""
from django.core.management.base import BaseCommand, CommandError
from organizations.models import SourceExportGeneration
from organizations.source_export_catalog import (
build_source,
cleanup,
registry,
verify_generation,
)
class Command(BaseCommand):
help = "Собрать all_available каталог и файлы существующих источников"
def add_arguments(self, parser):
parser.add_argument("--source-group", choices=list(registry()))
modes = parser.add_mutually_exclusive_group()
modes.add_argument("--cleanup-only", action="store_true")
modes.add_argument("--verify-only", action="store_true")
def handle(self, *args, **options):
failed = []
for group, entry in registry().items():
if (
options["source_group"] and options["source_group"] != group
) or not entry["export_enabled"]:
continue
try:
if options["verify_only"]:
for generation in SourceExportGeneration.objects.filter(
source_group=group, deleted_at=None
).iterator():
if not verify_generation(generation):
raise CommandError("Artifact integrity failure")
elif not options["cleanup_only"]:
generation = build_source(group)
self.stdout.write(
f"{group}: {generation.pk} records={generation.manifest['records_count']}"
)
if not options["verify_only"]:
cleanup(group)
except Exception as exc:
failed.append(group)
self.stderr.write(f"{group}: failed ({type(exc).__name__})")
if failed:
raise CommandError("Export groups failed: " + ", ".join(failed))

View File

@@ -0,0 +1,103 @@
# Generated by Django 3.2.25 on 2026-10-22 19:05
import uuid
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("organizations", "0018_stock_exchange_activity"),
]
operations = [
migrations.CreateModel(
name="SourceExportGeneration",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("source_group", models.CharField(db_index=True, max_length=64)),
("manifest", models.JSONField(default=dict)),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("deleted_at", models.DateTimeField(null=True)),
("storage_available", models.BooleanField(default=True)),
("data_revision", models.CharField(max_length=64)),
],
),
migrations.CreateModel(
name="SourceExportLease",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("issuer_id", models.CharField(db_index=True, max_length=128)),
("kind", models.CharField(default="ticket", max_length=8)),
("expires_at", models.DateTimeField(db_index=True)),
(
"generation",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="organizations.sourceexportgeneration",
),
),
],
),
migrations.CreateModel(
name="SourceExportLimiter",
fields=[
(
"key",
models.CharField(max_length=180, primary_key=True, serialize=False),
),
("window", models.BigIntegerField(default=0)),
("count", models.PositiveIntegerField(default=0)),
],
),
migrations.CreateModel(
name="SourceExportState",
fields=[
(
"source_group",
models.CharField(max_length=64, primary_key=True, serialize=False),
),
("current_generation", models.UUIDField(null=True)),
("last_build", models.JSONField(default=dict)),
("lock_owner", models.UUIDField(null=True)),
("lock_expires_at", models.DateTimeField(null=True)),
],
),
migrations.CreateModel(
name="SourceExportTicket",
fields=[
(
"token_hash",
models.CharField(max_length=64, primary_key=True, serialize=False),
),
("selection", models.JSONField()),
("expires_at", models.DateTimeField(db_index=True)),
("consumed_at", models.DateTimeField(null=True)),
(
"lease",
models.OneToOneField(
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="organizations.sourceexportlease",
),
),
],
),
]

View File

@@ -0,0 +1,40 @@
"""Register an opt-in schedule; capacity/shared-volume acceptance precedes enablement."""
from django.db import migrations
NAME = "organizations:source-export-catalog:nightly-msk"
def seed(apps, schema_editor):
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
Task = apps.get_model("django_celery_beat", "PeriodicTask")
cron, _ = Cron.objects.get_or_create(
minute="30",
hour="6",
day_of_week="*",
day_of_month="*",
month_of_year="*",
timezone="Europe/Moscow",
)
Task.objects.get_or_create(
name=NAME,
defaults={
"task": "organizations.tasks.refresh_source_export_catalog",
"crontab": cron,
"enabled": False,
"description": "Enable after all-available storage capacity and shared-volume acceptance.",
},
)
def remove(apps, schema_editor):
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
name=NAME
).delete()
class Migration(migrations.Migration):
dependencies = [
("organizations", "0019_source_export_catalog"),
("django_celery_beat", "0018_improve_crontab_helptext"),
]
operations = [migrations.RunPython(seed, reverse_code=remove)]

View File

@@ -1083,3 +1083,70 @@ class StockExchangeActivityExtension(OrganizationSourceExtension):
db_table = "organizations_stock_exchange_activity_extension"
verbose_name = _("биржевая активность")
verbose_name_plural = _("биржевая активность")
class SourceExportState(models.Model):
"""Current pointer and renewable owner lock for one all-available source."""
def __str__(self):
return "SourceExportState"
source_group = models.CharField(max_length=64, primary_key=True)
current_generation = models.UUIDField(null=True)
last_build = models.JSONField(default=dict)
lock_owner = models.UUIDField(null=True)
lock_expires_at = models.DateTimeField(null=True)
class SourceExportGeneration(models.Model):
"""Immutable public metadata; deleted_at retains generation tombstones."""
def __str__(self):
return "SourceExportGeneration"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
source_group = models.CharField(max_length=64, db_index=True)
manifest = models.JSONField(default=dict)
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
deleted_at = models.DateTimeField(null=True)
storage_available = models.BooleanField(default=True)
data_revision = models.CharField(max_length=64)
class SourceExportLease(models.Model):
"""Ticket/stream retention protection, shared by worker and web processes."""
def __str__(self):
return "SourceExportLease"
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
generation = models.ForeignKey(SourceExportGeneration, on_delete=models.CASCADE)
issuer_id = models.CharField(max_length=128, db_index=True)
kind = models.CharField(max_length=8, default="ticket")
expires_at = models.DateTimeField(db_index=True)
class SourceExportTicket(models.Model):
"""Only a SHA-256 token digest is persisted; claim uses a conditional update."""
def __str__(self):
return "SourceExportTicket"
token_hash = models.CharField(max_length=64, primary_key=True)
lease = models.OneToOneField(
SourceExportLease, on_delete=models.SET_NULL, null=True
)
selection = models.JSONField()
expires_at = models.DateTimeField(db_index=True)
consumed_at = models.DateTimeField(null=True)
class SourceExportLimiter(models.Model):
"""Shared fixed-window request counter and stream admission mutex."""
def __str__(self):
return "SourceExportLimiter"
key = models.CharField(max_length=180, primary_key=True)
window = models.BigIntegerField(default=0)
count = models.PositiveIntegerField(default=0)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,18 @@
"""Strict selectors for the new export scope (legacy DTOs stay unchanged)."""
from rest_framework import serializers
class SourceExportSelectionSerializer(serializers.Serializer):
selection_kind = serializers.ChoiceField(choices=("file", "source_archive"))
source_group = serializers.CharField(max_length=64)
generation_id = serializers.UUIDField()
part_id = serializers.UUIDField(required=False)
format = serializers.ChoiceField(choices=("xlsx", "csv", "json"))
def validate(self, attrs):
expected = {"selection_kind", "source_group", "generation_id", "format"}
if attrs["selection_kind"] == "file":
expected.add("part_id")
if set(self.initial_data) != expected:
raise serializers.ValidationError("Некорректный состав selector.")
return {key: str(value) for key, value in attrs.items()}

View File

@@ -124,3 +124,64 @@ def refresh_organization_data_snapshots_for_parser_batch(
batch_id=batch_id,
batch_size=batch_size,
)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_export_catalog(self, source_group=None):
"""Independent group publication; contention retries the latest canonical data."""
from organizations.source_export_catalog import (
ExportError,
build_source,
cleanup,
mark_queued,
registry,
)
groups = [source_group] if source_group else list(registry())
if source_group is None:
queued, failed = 0, []
for group in groups:
if not registry()[group]["export_enabled"] or not mark_queued(group):
continue
try:
refresh_source_export_catalog.delay(source_group=group)
queued += 1
except Exception:
from django.utils import timezone
from organizations.models import SourceExportState
SourceExportState.objects.filter(
pk=group, lock_owner=None, last_build__state="queued"
).update(
lock_expires_at=None,
last_build={
"state": "failed",
"started_at": None,
"finished_at": timezone.now()
.isoformat()
.replace("+00:00", "Z"),
"error_code": "source_export_enqueue_failed",
},
)
logger.exception("Source export enqueue failed group=%s", group)
failed.append(group)
if failed:
raise self.retry(countdown=30, max_retries=1000)
return {"status": "queued", "groups": queued}
try:
generation = build_source(source_group)
except ExportError as exc:
if exc.code == "source_export_not_ready":
raise self.retry(countdown=30, max_retries=1000) from exc
raise
cleanup(source_group)
from organizations.source_cache import get_source_data_cache_version
if generation.data_revision != get_source_data_cache_version():
raise self.retry(countdown=30, max_retries=1000)
return {
"status": "success",
"source_group": source_group,
"generation_id": str(generation.pk),
}

View File

@@ -3,15 +3,18 @@
from __future__ import annotations
import hashlib
import logging
import os
from collections.abc import Mapping
from contextlib import suppress
from tempfile import NamedTemporaryFile
from typing import Any
from uuid import UUID
from uuid import UUID, uuid4
from apps.core.openapi import swagger_tag
from django.conf import settings
from django.core.cache import cache
from django.db import DatabaseError
from django.db.models import Case, CharField, Count, F, Max, Q, Value, When
from django.db.models.fields.json import KeyTextTransform
from django.db.models.functions import Cast, Coalesce, NullIf
@@ -21,13 +24,14 @@ from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from rest_framework import status
from rest_framework.decorators import action
from rest_framework.exceptions import NotFound, ValidationError
from rest_framework.exceptions import NotFound, ParseError, ValidationError
from rest_framework.filters import OrderingFilter, SearchFilter
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated
from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet
from organizations import source_export_catalog as export_catalog_service
from organizations.cache import (
DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS,
ORGANIZATION_API_CACHE_CONTRACT_VERSION,
@@ -62,6 +66,7 @@ from organizations.serializers import (
SourceRecordExportUnavailableSerializer,
SourceRecordFilterOptionsResponseSerializer,
)
from organizations.source_export_serializers import SourceExportSelectionSerializer
from organizations.source_record_export import (
SourceRecordExportArchive,
SourceRecordExportArtifactsUnavailable,
@@ -599,7 +604,12 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
def get_permissions(self):
if self.action == "integrated_attachment":
return [IsAuthenticated()]
if self.action in {"export", "export_ticket"}:
if self.action in {
"export",
"export_ticket",
"export_catalog",
"export_catalog_detail",
}:
return [IsAdminUser()]
if self.action == "export_download":
return [AllowAny()]
@@ -976,6 +986,13 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
*args: Any,
**kwargs: Any,
) -> StreamingHttpResponse | Response:
if not isinstance(request.data, Mapping) or set(request.data) != {
"sources",
"format",
}:
return self._catalog_error(
export_catalog_service.ExportError("invalid_request")
)
serializer = OrganizationSourceRecordExportRequestSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
@@ -997,6 +1014,127 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
return self._source_record_export_response(package)
def finalize_response(self, request, response, *args, **kwargs):
response = super().finalize_response(request, response, *args, **kwargs)
if self.action in {"export_catalog", "export_catalog_detail", "export_ticket"}:
response.setdefault("Cache-Control", "private, no-store")
response.setdefault("X-Request-ID", str(uuid4()))
return response
def handle_exception(self, exc):
if getattr(exc, "status_code", None) == 401:
return super().handle_exception(exc)
is_new = self.action in {"export_catalog", "export_catalog_detail"}
if self.action in {"export_ticket", "export_download"}:
try:
data = self.request.data
except ParseError:
return self._catalog_error(
export_catalog_service.ExportError("invalid_request")
)
is_new = (
(
not isinstance(data, Mapping)
or "selection_kind" in data
or "sources" not in data
)
if self.action == "export_ticket"
else (
not isinstance(data, Mapping)
or str(data.get("ticket", "")).startswith("se1_")
)
)
if is_new:
if getattr(exc, "status_code", None) == 403:
return self._catalog_error(
export_catalog_service.ExportError("forbidden", 403)
)
if isinstance(exc, DatabaseError | OSError):
code = (
"ticket_unavailable"
if self.action in {"export_ticket", "export_download"}
else "metadata_unavailable"
)
return self._catalog_error(
export_catalog_service.ExportError(code, 503)
)
if not getattr(exc, "status_code", None):
logging.getLogger(__name__).exception("Source export API failed")
return self._catalog_error(
export_catalog_service.ExportError("internal_error", 500)
)
return super().handle_exception(exc)
@staticmethod
def _catalog_error(error):
request_id = str(uuid4())
headers = {"Cache-Control": "private, no-store", "X-Request-ID": request_id}
if error.retryable:
headers["Retry-After"] = (
"10"
if error.status == 429
else ("5" if error.code.endswith("ticket_unavailable") else "30")
)
return Response(
{
"detail": "Не удалось выполнить запрос выгрузки.",
"code": error.code,
"field": error.field,
"retryable": error.retryable,
"request_id": request_id,
},
status=error.status,
headers=headers,
)
@staticmethod
def _catalog_response(data, pagination=None):
return Response(
{
"success": True,
"data": data,
"errors": None,
"meta": {"pagination": pagination} if pagination is not None else None,
},
headers={
"Cache-Control": "private, no-store",
"X-Request-ID": str(uuid4()),
},
)
@swagger_auto_schema(
tags=[ORGANIZATIONS_TAG],
operation_id="v2_organization_source_records_export_catalog_list",
)
@action(detail=False, methods=["get"], url_path="export-catalog")
def export_catalog(self, request, *args, **kwargs):
try:
export_catalog_service.rate_limit(request.user.pk, "metadata")
if request.query_params:
raise export_catalog_service.ExportError("invalid_request")
return self._catalog_response(export_catalog_service.catalog())
except export_catalog_service.ExportError as error:
return self._catalog_error(error)
@swagger_auto_schema(
tags=[ORGANIZATIONS_TAG],
operation_id="v2_organization_source_records_export_catalog_read",
)
@action(
detail=False,
methods=["get"],
url_path=r"export-catalog/(?P<source_group>[^/.]+)",
)
def export_catalog_detail(self, request, source_group, *args, **kwargs):
try:
export_catalog_service.rate_limit(request.user.pk, "metadata")
data, pagination = export_catalog_service.detail(
source_group, request.query_params
)
return self._catalog_response(data, pagination)
except export_catalog_service.ExportError as error:
return self._catalog_error(error)
@swagger_auto_schema(
tags=[ORGANIZATIONS_TAG],
operation_id="v2_organization_source_records_export_ticket",
@@ -1016,6 +1154,33 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
)
@action(detail=False, methods=["post"], url_path="export-ticket")
def export_ticket(self, request, *args: Any, **kwargs: Any) -> Response:
if not isinstance(request.data, Mapping):
return self._catalog_error(
export_catalog_service.ExportError("invalid_request")
)
if "selection_kind" in request.data or "sources" not in request.data:
serializer = SourceExportSelectionSerializer(data=request.data)
if not serializer.is_valid():
return self._catalog_error(
export_catalog_service.ExportError("invalid_request")
)
try:
export_catalog_service.rate_limit(request.user.pk, "ticket")
result = export_catalog_service.issue_ticket(
request.user, serializer.validated_data
)
return Response(
result, status=201, headers={"Cache-Control": "private, no-store"}
)
except export_catalog_service.ExportError as error:
return self._catalog_error(error)
if not isinstance(request.data, Mapping) or set(request.data) != {
"sources",
"format",
}:
return self._catalog_error(
export_catalog_service.ExportError("invalid_request")
)
serializer = OrganizationSourceRecordExportRequestSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
@@ -1085,6 +1250,23 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
*args: Any,
**kwargs: Any,
) -> StreamingHttpResponse | Response:
if (
not isinstance(request.data, Mapping)
or set(request.data) != {"ticket"}
or (
hasattr(request.data, "getlist")
and len(request.data.getlist("ticket")) != 1
)
):
return self._catalog_error(
export_catalog_service.ExportError("ticket_malformed")
)
try:
response = export_catalog_service.consume_ticket(request.data.get("ticket"))
if response is not None:
return response
except export_catalog_service.ExportError as error:
return self._catalog_error(error)
serializer = OrganizationSourceRecordExportDownloadSerializer(data=request.data)
if not serializer.is_valid():
return Response(

View File

@@ -262,6 +262,16 @@ BACKUP_EXPORT_DIRECTORY = os.getenv(
"BACKUP_EXPORT_DIRECTORY",
str(PROJECT_ROOT / "media" / "backups"),
)
# Private persistent volume, mounted identically in web and workers; no media alias.
SOURCE_EXPORT_DIRECTORY = os.getenv(
"SOURCE_EXPORT_DIRECTORY", str(PROJECT_ROOT / "private-source-exports")
)
SOURCE_EXPORT_AUTOBUILD = (
os.getenv("SOURCE_EXPORT_AUTOBUILD", "false").lower() == "true"
)
SOURCE_EXPORT_DISABLED_GROUPS = tuple(
filter(None, os.getenv("SOURCE_EXPORT_DISABLED_GROUPS", "").split(","))
)
SOURCE_RECORD_EXPORT_DIRECTORY = os.getenv(
"SOURCE_RECORD_EXPORT_DIRECTORY",
str(PROJECT_ROOT / "media" / "source-record-exports"),