feat: add nightly source record exports

tests/apps/external_data/test_export_tasks.py

@@ -0,0 +1,70 @@
+"""Tests for the external-data export task and schedule."""
+
+from importlib import import_module
+from tempfile import TemporaryDirectory
+
+from apps.external_data.tasks import refresh_source_record_export_artifacts
+from django.apps import apps as django_apps
+from django.conf import settings
+from django.core.cache import cache
+from django.test import TestCase, override_settings
+from django_celery_beat.models import PeriodicTask
+
+
+class SourceRecordExportArtifactsTaskTest(TestCase):
+    """Check nightly artifact generation and its distributed lock."""
+
+    def setUp(self):
+        cache.clear()
+        self.export_directory = TemporaryDirectory()
+        self.settings_override = override_settings(
+            SOURCE_RECORD_EXPORT_DIRECTORY=self.export_directory.name,
+            SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP=2,
+            SOURCE_RECORD_EXPORT_LOCK_KEY="test:state-corp-source-exports:lock",
+            SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS=300,
+        )
+        self.settings_override.enable()
+
+    def tearDown(self):
+        self.settings_override.disable()
+        self.export_directory.cleanup()
+        cache.clear()
+        super().tearDown()
+
+    def test_refresh_task_builds_matrix_and_releases_lock(self):
+        result = refresh_source_record_export_artifacts()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["artifacts_count"], 25)
+        self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))
+
+    def test_refresh_task_skips_when_generation_lock_is_held(self):
+        cache.set(settings.SOURCE_RECORD_EXPORT_LOCK_KEY, "busy", timeout=300)
+
+        result = refresh_source_record_export_artifacts()
+
+        self.assertEqual(result, {"status": "skipped", "reason": "locked"})
+
+
+class SourceRecordExportScheduleMigrationTest(TestCase):
+    """Check the nightly Celery Beat schedule for prepared exports."""
+
+    def test_migration_seeds_nightly_export_task_idempotently(self):
+        migration = import_module(
+            "apps.external_data.migrations.0007_seed_nightly_source_record_exports"
+        )
+
+        migration.seed_nightly_source_record_export_schedule(django_apps, None)
+        migration.seed_nightly_source_record_export_schedule(django_apps, None)
+
+        task = PeriodicTask.objects.get(name=migration.NIGHTLY_SOURCE_EXPORT_TASK_NAME)
+        self.assertEqual(
+            task.task,
+            "apps.external_data.tasks.refresh_source_record_export_artifacts",
+        )
+        self.assertTrue(task.enabled)
+        self.assertEqual(task.args, "[]")
+        self.assertEqual(task.kwargs, "{}")
+        self.assertEqual(task.crontab.minute, "30")
+        self.assertEqual(task.crontab.hour, "5")
+        self.assertEqual(str(task.crontab.timezone), "Europe/Moscow")

tests/apps/external_data/test_source_record_export.py

@@ -0,0 +1,225 @@
+"""Tests for prepared State Corp external-data exports."""
+
+import json
+import zipfile
+from io import BytesIO, StringIO
+from tempfile import TemporaryDirectory
+
+from apps.external_data.source_record_export import (
+    build_source_record_export_artifacts,
+    load_current_source_record_export_generation,
+)
+from django.core.management import call_command
+from django.test import override_settings
+from openpyxl import load_workbook
+from rest_framework import status
+from rest_framework.test import APITestCase
+
+from tests.apps.external_data.factories import (
+    FinancialReportFactory,
+    FinancialReportLineFactory,
+    IndustrialCertificateFactory,
+    IndustrialProductFactory,
+    ManufacturerRegistryEntryFactory,
+    ProsecutorCheckFactory,
+)
+from tests.apps.organization.factories import OrganizationFactory
+from tests.apps.user.factories import UserFactory
+
+
+class SourceRecordExportApiTest(APITestCase):
+    """Check admin access and zero-query delivery of prepared files."""
+
+    export_url = "/api/v2/organization-source-records/export/"
+    ticket_url = "/api/v2/organization-source-records/export-ticket/"
+    download_url = "/api/v2/organization-source-records/export-download/"
+
+    def setUp(self):
+        self.export_directory = TemporaryDirectory()
+        self.settings_override = override_settings(
+            SOURCE_RECORD_EXPORT_DIRECTORY=self.export_directory.name,
+            SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP=2,
+        )
+        self.settings_override.enable()
+
+    def tearDown(self):
+        self.settings_override.disable()
+        self.export_directory.cleanup()
+        super().tearDown()
+
+    @staticmethod
+    def _response_body(response) -> bytes:
+        return b"".join(response.streaming_content)
+
+    def test_export_is_unavailable_before_first_generation(self):
+        self.client.force_authenticate(UserFactory.create_superuser())
+
+        response = self.client.post(
+            self.export_url,
+            {"sources": ["planned_inspections"], "format": "json"},
+            format="json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE)
+        self.assertEqual(response.data["code"], "source_export_not_ready")
+        self.assertEqual(response["Retry-After"], "3600")
+
+    def test_generation_builds_full_matrix_from_normalized_tables(self):
+        organization = OrganizationFactory.create(
+            full_name='Акционерное общество "Экспорт"',
+            okpo="12345678",
+        )
+        IndustrialProductFactory.create(organization=organization)
+        IndustrialCertificateFactory.create(organization=organization)
+        ManufacturerRegistryEntryFactory.create(organization=organization)
+        ProsecutorCheckFactory.create(organization=organization)
+        report = FinancialReportFactory.create(organization=organization)
+        FinancialReportLineFactory.create(report=report, line_code="1600")
+
+        generation = build_source_record_export_artifacts()
+
+        self.assertEqual(generation.artifacts_count, 25)
+        self.assertEqual(generation.files_count, 25)
+        self.assertEqual(generation.records_count, 5)
+        industrial_path = next(
+            artifact.path
+            for artifact in generation.artifacts
+            if artifact.source_group == "industrial_production"
+            and artifact.file_format == "json"
+        )
+        industrial_rows = json.loads(industrial_path.read_text(encoding="utf-8"))
+        self.assertEqual(
+            {row["record_type"] for row in industrial_rows},
+            {
+                "industrial_certificate",
+                "industrial_product",
+                "manufacturer_registry_entry",
+            },
+        )
+        self.assertEqual({row["ОКПО"] for row in industrial_rows}, {"12345678"})
+
+        financial_path = next(
+            artifact.path
+            for artifact in generation.artifacts
+            if artifact.source_group == "financial_indicators"
+        )
+        financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
+        self.assertEqual(financial_rows[0]["financial_lines"][0]["line_code"], "1600")
+
+    def test_management_command_bootstraps_first_generation(self):
+        command_output = StringIO()
+
+        call_command("build_source_record_exports", stdout=command_output)
+
+        generation = load_current_source_record_export_generation()
+        self.assertEqual(generation.artifacts_count, 25)
+        self.assertIn('"artifacts_count": 25', command_output.getvalue())
+
+    def test_admin_streams_selected_prepared_files_without_database_queries(self):
+        self.client.force_authenticate(UserFactory.create_superuser())
+        organization = OrganizationFactory.create(okpo="87654321")
+        ProsecutorCheckFactory.create(organization=organization)
+        report = FinancialReportFactory.create(organization=organization)
+        FinancialReportLineFactory.create(report=report)
+        generation = build_source_record_export_artifacts()
+
+        with self.assertNumQueries(0):
+            response = self.client.post(
+                self.export_url,
+                {
+                    "sources": ["planned_inspections", "financial_indicators"],
+                    "format": "xlsx",
+                },
+                format="json",
+            )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertTrue(response.streaming)
+        self.assertEqual(response["Content-Type"], "application/zip")
+        self.assertEqual(
+            response["X-Source-Export-Generated-At"], generation.generated_at
+        )
+        self.assertNotIn("Content-Length", response)
+
+        with zipfile.ZipFile(BytesIO(self._response_body(response))) as archive:
+            self.assertEqual(
+                set(archive.namelist()),
+                {"planned-inspections.xlsx", "financial-indicators.json"},
+            )
+            workbook = load_workbook(
+                BytesIO(archive.read("planned-inspections.xlsx")),
+                read_only=True,
+            )
+            rows = list(workbook["data"].iter_rows(values_only=True))
+            self.assertEqual(
+                rows[0][:6],
+                ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО", "organization"),
+            )
+            self.assertEqual(rows[1][4], "87654321")
+
+    def test_ticket_is_admin_only_and_can_be_consumed_once_without_auth(self):
+        build_source_record_export_artifacts()
+        regular_user = UserFactory.create_user()
+        self.client.force_authenticate(regular_user)
+        forbidden_response = self.client.post(
+            self.ticket_url,
+            {"sources": ["bankruptcy"], "format": "json"},
+            format="json",
+        )
+        self.assertEqual(forbidden_response.status_code, status.HTTP_403_FORBIDDEN)
+
+        self.client.force_authenticate(UserFactory.create_superuser())
+        with self.assertNumQueries(0):
+            ticket_response = self.client.post(
+                self.ticket_url,
+                {"sources": ["bankruptcy"], "format": "json"},
+                format="json",
+            )
+        self.assertEqual(ticket_response.status_code, status.HTTP_201_CREATED)
+        self.assertRegex(ticket_response.data["ticket"], r"^[A-Za-z0-9_-]{43}$")
+        self.assertEqual(ticket_response.data["expires_in"], 300)
+
+        self.client.force_authenticate(user=None)
+        with self.assertNumQueries(0):
+            download_response = self.client.post(
+                self.download_url,
+                {"ticket": ticket_response.data["ticket"]},
+                format="multipart",
+            )
+        self.assertEqual(download_response.status_code, status.HTTP_200_OK)
+        with zipfile.ZipFile(
+            BytesIO(self._response_body(download_response))
+        ) as archive:
+            self.assertEqual(archive.namelist(), ["bankruptcy-procedures.json"])
+
+        consumed_response = self.client.post(
+            self.download_url,
+            {"ticket": ticket_response.data["ticket"]},
+            format="multipart",
+        )
+        self.assertEqual(consumed_response.status_code, status.HTTP_410_GONE)
+        self.assertEqual(consumed_response.data["code"], "source_export_ticket_invalid")
+
+    @override_settings(SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE=2)
+    def test_xlsx_is_split_into_bounded_workbook_parts(self):
+        organization = OrganizationFactory.create()
+        ProsecutorCheckFactory.create_batch(3, organization=organization)
+
+        generation = build_source_record_export_artifacts()
+
+        inspection_parts = [
+            artifact
+            for artifact in generation.artifacts
+            if artifact.source_group == "planned_inspections"
+            and artifact.file_format == "xlsx"
+        ]
+        self.assertEqual(
+            [artifact.part_number for artifact in inspection_parts], [1, 2]
+        )
+        self.assertEqual(
+            [artifact.file_name for artifact in inspection_parts],
+            [
+                "planned-inspections-part-001.xlsx",
+                "planned-inspections-part-002.xlsx",
+            ],
+        )