feat: add nightly source record exports

src/apps/external_data/export_serializers.py

@@ -0,0 +1,37 @@
+"""Request serializers for prepared external-data exports."""
+
+from apps.external_data.source_record_export import (
+    EXPORT_FORMATS,
+    SOURCE_GROUP_EXPORT_SPECS,
+)
+from rest_framework import serializers
+
+
+class SourceRecordExportRequestSerializer(serializers.Serializer):
+    """Validate selected source groups and the requested file format."""
+
+    sources = serializers.ListField(
+        child=serializers.ChoiceField(
+            choices=[
+                (source_group, source_group)
+                for source_group in SOURCE_GROUP_EXPORT_SPECS
+            ]
+        ),
+        allow_empty=False,
+    )
+    format = serializers.ChoiceField(
+        choices=[
+            (export_format, export_format.upper()) for export_format in EXPORT_FORMATS
+        ]
+    )
+
+    def validate_sources(self, value: list[str]) -> list[str]:
+        if len(value) != len(set(value)):
+            raise serializers.ValidationError("Источники не должны повторяться.")
+        return value
+
+
+class SourceRecordExportDownloadSerializer(serializers.Serializer):
+    """Validate a one-time ticket submitted by a native browser form."""
+
+    ticket = serializers.CharField(max_length=64, trim_whitespace=False)

src/apps/external_data/export_urls.py

@@ -0,0 +1,22 @@
+"""URL routes for prepared external-data exports."""
+
+from apps.external_data.export_views import (
+    SourceRecordExportDownloadView,
+    SourceRecordExportTicketView,
+    SourceRecordExportView,
+)
+from django.urls import path
+
+app_name = "source_record_exports"
+
+urlpatterns = [
+    path("export/", SourceRecordExportView.as_view(), name="export"),
+    path(
+        "export-ticket/", SourceRecordExportTicketView.as_view(), name="export-ticket"
+    ),
+    path(
+        "export-download/",
+        SourceRecordExportDownloadView.as_view(),
+        name="export-download",
+    ),
+]

src/apps/external_data/export_views.py

@@ -0,0 +1,174 @@
+"""HTTP endpoints for prepared external-data exports."""
+
+from apps.external_data.export_serializers import (
+    SourceRecordExportDownloadSerializer,
+    SourceRecordExportRequestSerializer,
+)
+from apps.external_data.source_record_export import (
+    SourceRecordExportArchive,
+    SourceRecordExportArtifactsUnavailable,
+    SourceRecordExportTicketInvalid,
+    build_source_records_export_archive,
+    consume_source_record_export_download_ticket,
+    create_source_record_export_download_ticket,
+)
+from django.http import StreamingHttpResponse
+from drf_yasg import openapi
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework import status
+from rest_framework.permissions import AllowAny, IsAdminUser
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+
+class SourceRecordExportResponseMixin:
+    """Build the shared streaming ZIP response for export endpoints."""
+
+    @staticmethod
+    def source_record_export_response(
+        package: SourceRecordExportArchive,
+    ) -> StreamingHttpResponse:
+        response = StreamingHttpResponse(
+            package.archive_chunks,
+            content_type="application/zip",
+        )
+        response[
+            "Content-Disposition"
+        ] = f'attachment; filename="{package.archive_name}"'
+        response["X-Source-Export-Files"] = str(package.files_count)
+        response["X-Source-Export-Generated-At"] = package.generated_at
+        return response
+
+    @staticmethod
+    def source_record_export_not_ready_response() -> Response:
+        return Response(
+            {
+                "detail": "Готовая ночная выгрузка ещё не сформирована.",
+                "code": "source_export_not_ready",
+            },
+            status=status.HTTP_503_SERVICE_UNAVAILABLE,
+            headers={"Retry-After": "3600"},
+        )
+
+
+class SourceRecordExportView(SourceRecordExportResponseMixin, APIView):
+    """Stream a request-specific ZIP from the current prepared generation."""
+
+    permission_classes = [IsAdminUser]
+
+    @swagger_auto_schema(
+        operation_summary="Выгрузить записи внешних источников",
+        operation_description=(
+            "Упаковывает в ZIP готовые файлы выбранных групп без повторного "
+            "чтения таблиц external_data. Финансовые показатели всегда JSON."
+        ),
+        request_body=SourceRecordExportRequestSerializer,
+        responses={
+            200: openapi.Response(
+                description="Потоковый ZIP-архив.",
+                schema=openapi.Schema(type=openapi.TYPE_FILE),
+            ),
+            400: "Некорректные параметры.",
+            403: "Доступ разрешён только администраторам.",
+            503: "Ночная выгрузка ещё не сформирована.",
+        },
+        tags=["Внешние данные"],
+    )
+    def post(self, request: Request) -> StreamingHttpResponse | Response:
+        serializer = SourceRecordExportRequestSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        try:
+            package = build_source_records_export_archive(
+                source_groups=serializer.validated_data["sources"],
+                export_format=serializer.validated_data["format"],
+            )
+        except SourceRecordExportArtifactsUnavailable:
+            return self.source_record_export_not_ready_response()
+        return self.source_record_export_response(package)
+
+
+class SourceRecordExportTicketView(SourceRecordExportResponseMixin, APIView):
+    """Issue a short-lived ticket for a native browser download."""
+
+    permission_classes = [IsAdminUser]
+
+    @swagger_auto_schema(
+        operation_summary="Подготовить нативное скачивание внешних данных",
+        request_body=SourceRecordExportRequestSerializer,
+        responses={
+            201: "Одноразовый ticket и имя ZIP-файла.",
+            400: "Некорректные параметры.",
+            403: "Доступ разрешён только администраторам.",
+            503: "Выгрузка или ticket временно недоступны.",
+        },
+        tags=["Внешние данные"],
+    )
+    def post(self, request: Request) -> Response:
+        serializer = SourceRecordExportRequestSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        try:
+            download_ticket = create_source_record_export_download_ticket(
+                source_groups=serializer.validated_data["sources"],
+                export_format=serializer.validated_data["format"],
+            )
+        except SourceRecordExportArtifactsUnavailable:
+            return self.source_record_export_not_ready_response()
+        except RuntimeError:
+            return Response(
+                {
+                    "detail": "Не удалось подготовить скачивание. Повторите запрос.",
+                    "code": "source_export_ticket_unavailable",
+                },
+                status=status.HTTP_503_SERVICE_UNAVAILABLE,
+                headers={"Retry-After": "5"},
+            )
+
+        return Response(
+            {
+                "ticket": download_ticket.ticket,
+                "file_name": download_ticket.archive_name,
+                "expires_in": download_ticket.expires_in,
+            },
+            status=status.HTTP_201_CREATED,
+        )
+
+
+class SourceRecordExportDownloadView(SourceRecordExportResponseMixin, APIView):
+    """Consume a one-time ticket and stream the prepared ZIP archive."""
+
+    authentication_classes: list = []
+    permission_classes = [AllowAny]
+
+    @swagger_auto_schema(
+        operation_summary="Скачать готовые внешние данные по ticket",
+        request_body=SourceRecordExportDownloadSerializer,
+        responses={
+            200: openapi.Response(
+                description="Потоковый ZIP-архив.",
+                schema=openapi.Schema(type=openapi.TYPE_FILE),
+            ),
+            400: "Ticket отсутствует или имеет неверный формат.",
+            410: "Ticket истёк или уже использован.",
+            503: "Опубликованная выгрузка больше недоступна.",
+        },
+        tags=["Внешние данные"],
+    )
+    def post(self, request: Request) -> StreamingHttpResponse | Response:
+        serializer = SourceRecordExportDownloadSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        try:
+            package = consume_source_record_export_download_ticket(
+                serializer.validated_data["ticket"]
+            )
+        except SourceRecordExportTicketInvalid:
+            return Response(
+                {
+                    "detail": "Ticket скачивания истёк или уже использован.",
+                    "code": "source_export_ticket_invalid",
+                },
+                status=status.HTTP_410_GONE,
+            )
+        except SourceRecordExportArtifactsUnavailable:
+            return self.source_record_export_not_ready_response()
+        return self.source_record_export_response(package)

src/apps/external_data/management/__init__.py

@@ -0,0 +1 @@
+"""Management package for external-data operations."""

src/apps/external_data/management/commands/__init__.py

@@ -0,0 +1 @@
+"""Management commands for external-data operations."""

src/apps/external_data/management/commands/build_source_record_exports.py

@@ -0,0 +1,32 @@
+"""Build the complete prepared external-data export matrix."""
+
+import json
+
+from apps.core.management.commands.base import BaseAppCommand
+from apps.external_data.source_record_export import (
+    build_source_record_export_artifacts,
+)
+
+
+class Command(BaseAppCommand):
+    """Build source-record files synchronously for bootstrap and recovery."""
+
+    help = "Формирует готовые CSV/XLSX/JSON выгрузки внешних источников"
+    use_transaction = False
+
+    def execute_command(self, *args, **options) -> str:
+        generation = build_source_record_export_artifacts()
+        rendered = json.dumps(
+            {
+                "generation_id": generation.generation_id,
+                "generated_at": generation.generated_at,
+                "artifacts_count": generation.artifacts_count,
+                "files_count": generation.files_count,
+                "records_count": generation.records_count,
+                "total_size": generation.total_size,
+            },
+            ensure_ascii=False,
+            sort_keys=True,
+        )
+        self.log_success(rendered)
+        return rendered

src/apps/external_data/migrations/0007_seed_nightly_source_record_exports.py

@@ -0,0 +1,61 @@
+import json
+
+from django.db import migrations
+
+NIGHTLY_SOURCE_EXPORT_TASK_NAME = "external-data:source-record-exports:nightly-msk"
+NIGHTLY_SOURCE_EXPORT_TASK_PATH = (
+    "apps.external_data.tasks.refresh_source_record_export_artifacts"
+)
+NIGHTLY_SOURCE_EXPORT_MSK_CRON = {
+    "minute": "30",
+    "hour": "5",
+    "day_of_week": "*",
+    "day_of_month": "*",
+    "month_of_year": "*",
+    "timezone": "Europe/Moscow",
+}
+
+
+def seed_nightly_source_record_export_schedule(apps, schema_editor):
+    CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule")
+    PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
+
+    crontab, _ = CrontabSchedule.objects.get_or_create(**NIGHTLY_SOURCE_EXPORT_MSK_CRON)
+    field_names = {field.name for field in PeriodicTask._meta.fields}
+    schedule_fields = {"crontab": crontab}
+    for field_name in ("interval", "solar", "clocked"):
+        if field_name in field_names:
+            schedule_fields[field_name] = None
+
+    PeriodicTask.objects.update_or_create(
+        name=NIGHTLY_SOURCE_EXPORT_TASK_NAME,
+        defaults={
+            "task": NIGHTLY_SOURCE_EXPORT_TASK_PATH,
+            "args": json.dumps([]),
+            "kwargs": json.dumps({}),
+            "enabled": True,
+            "description": (
+                "Nightly preparation of State Corp external-data export artifacts."
+            ),
+            **schedule_fields,
+        },
+    )
+
+
+def remove_nightly_source_record_export_schedule(apps, schema_editor):
+    PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
+    PeriodicTask.objects.filter(name=NIGHTLY_SOURCE_EXPORT_TASK_NAME).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_celery_beat", "0018_improve_crontab_helptext"),
+        ("external_data", "0006_bankruptcy_procedure_status_length"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            seed_nightly_source_record_export_schedule,
+            reverse_code=remove_nightly_source_record_export_schedule,
+        ),
+    ]

src/apps/external_data/tasks.py

@@ -0,0 +1,46 @@
+"""Celery tasks for prepared external-data exports."""
+
+import logging
+
+from apps.core.tasks import PeriodicTask as CorePeriodicTask
+from apps.external_data.source_record_export import (
+    build_source_record_export_artifacts,
+)
+from celery import shared_task
+from django.conf import settings
+from django.core.cache import cache
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(bind=True, base=CorePeriodicTask)
+def refresh_source_record_export_artifacts(self) -> dict:  # noqa: ARG001
+    """Build and atomically publish the nightly external-data export matrix."""
+
+    lock_key = getattr(
+        settings,
+        "SOURCE_RECORD_EXPORT_LOCK_KEY",
+        "external-data:source-record-exports:lock",
+    )
+    lock_ttl = int(
+        getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
+    )
+    if not cache.add(lock_key, "1", timeout=lock_ttl):
+        logger.info("Source-record export generation skipped: lock is already held")
+        return {"status": "skipped", "reason": "locked"}
+
+    try:
+        generation = build_source_record_export_artifacts()
+        result = {
+            "status": "success",
+            "generation_id": generation.generation_id,
+            "generated_at": generation.generated_at,
+            "artifacts_count": generation.artifacts_count,
+            "files_count": generation.files_count,
+            "records_count": generation.records_count,
+            "total_size": generation.total_size,
+        }
+        logger.info("Source-record export generation published: %s", result)
+        return result
+    finally:
+        cache.delete(lock_key)

src/core/api_v2_urls.py

@@ -0,0 +1,12 @@
+"""API v2 routes shared with the Mostovik administrative frontend contract."""
+
+from django.urls import include, path
+
+app_name = "api_v2"
+
+urlpatterns = [
+    path(
+        "organization-source-records/",
+        include("apps.external_data.export_urls"),
+    ),
+]

src/core/urls.py

@@ -40,6 +40,7 @@ urlpatterns = [
     path("admin/", admin.site.urls),
     path("health/", include("apps.core.urls")),
     path("api/v1/", include("core.api_v1_urls", namespace="api_v1")),
+    path("api/v2/", include("core.api_v2_urls", namespace="api_v2")),
     path("auth/", include("rest_framework.urls")),
 ]
 

src/settings/base.py

@@ -231,6 +231,26 @@ STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
 MEDIA_URL = "/media/"
 MEDIA_ROOT = PROJECT_ROOT / "media"
+SOURCE_RECORD_EXPORT_DIRECTORY = os.getenv(
+    "SOURCE_RECORD_EXPORT_DIRECTORY",
+    str(PROJECT_ROOT / "media" / "source-record-exports"),
+)
+SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP = int(
+    os.getenv("SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP", "2")
+)
+SOURCE_RECORD_EXPORT_LOCK_KEY = os.getenv(
+    "SOURCE_RECORD_EXPORT_LOCK_KEY",
+    "external-data:source-record-exports:lock",
+)
+SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS = int(
+    os.getenv("SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", str(6 * 60 * 60))
+)
+SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE = int(
+    os.getenv("SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE", "100000")
+)
+SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS = int(
+    os.getenv("SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS", "300")
+)
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 AUTH_USER_MODEL = "user.User"