diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index 6ea14c7..01db42b 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -8,7 +8,9 @@ from organizations.models import ( OrganizationSourceExtension, OrganizationSourceFinancialLine, OrganizationSourceRecord, + SourceGroup, ) +from organizations.source_record_export import EXPORT_FORMATS class OrganizationSourceFinancialLineSerializer(serializers.ModelSerializer): @@ -127,6 +129,25 @@ class OrganizationSourceRecordListResponseSerializer(serializers.Serializer): meta = serializers.JSONField(read_only=True, allow_null=True) +class OrganizationSourceRecordExportRequestSerializer(serializers.Serializer): + """Request for source-record ZIP export.""" + + sources = serializers.ListField( + child=serializers.ChoiceField(choices=SourceGroup.choices), + allow_empty=False, + ) + format = serializers.ChoiceField( + choices=[ + (export_format, export_format.upper()) for export_format in EXPORT_FORMATS + ], + ) + + def validate_sources(self, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise serializers.ValidationError("Источники не должны повторяться.") + return value + + class OrganizationSourceExtensionSerializer(serializers.ModelSerializer): """Compact source extension representation.""" diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py new file mode 100644 index 0000000..7701361 --- /dev/null +++ b/src/organizations/source_record_export.py @@ -0,0 +1,316 @@ +"""ZIP export for organization source records.""" + +from __future__ import annotations + +import csv +import json +import zipfile +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from io import BytesIO, StringIO +from typing import Any + +from django.db.models import QuerySet +from django.utils import timezone +from openpyxl import Workbook + +from organizations.models import OrganizationSourceRecord, SourceGroup + +EXPORT_FORMAT_CSV = "csv" +EXPORT_FORMAT_XLSX = "xlsx" +EXPORT_FORMAT_JSON = "json" +EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON) +FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value + +SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = { + SourceGroup.FINANCIAL_INDICATORS.value: "financial-indicators", + SourceGroup.GOVERNMENT_PROCUREMENTS.value: "public-procurements", + SourceGroup.INDUSTRIAL_PRODUCTION.value: "manufacturers-and-products", + SourceGroup.PLANNED_INSPECTIONS.value: "planned-inspections", + SourceGroup.BANKRUPTCY.value: "bankruptcy-procedures", + SourceGroup.DEFENSE_SUPPLIERS.value: "defense-unreliable-suppliers", + SourceGroup.ARBITRATION.value: "arbitration-cases", + SourceGroup.SECURITY_REGISTRIES.value: "information-security-registries", + SourceGroup.VACANCIES.value: "labor-vacancies", +} + +ORGANIZATION_EXPORT_FIELDS = ["Наименование", "ИНН", "ОГРН", "КПП"] +SOURCE_RECORD_EXPORT_FIELDS = [ + "uid", + "source_group", + "source", + "record_type", + "external_id", + "title", + "record_date", + "amount", + "status", + "url", + "load_batch", + "created_at", + "updated_at", +] +FINANCIAL_LINE_FIELDS = [ + "id", + "form_code", + "line_code", + "line_name", + "year", + "period_start", + "period_end", +] + + +@dataclass(frozen=True) +class SourceRecordExportArchive: + """In-memory source records export archive.""" + + archive_name: str + archive_bytes: bytes + files_count: int + + +def build_source_records_export_archive( + *, + source_groups: Sequence[str], + export_format: str, + now: datetime | None = None, +) -> SourceRecordExportArchive: + """Build a ZIP archive with one source-record file per source group.""" + + timestamp = (now or timezone.now()).strftime("%Y%m%d_%H%M%S") + archive_buffer = BytesIO() + + with zipfile.ZipFile( + archive_buffer, + mode="w", + compression=zipfile.ZIP_DEFLATED, + ) as archive: + for source_group in source_groups: + file_format = _resolve_source_group_export_format( + source_group=source_group, + requested_format=export_format, + ) + file_name = _build_source_group_file_name( + source_group=source_group, + file_format=file_format, + ) + archive.writestr( + file_name, + _render_source_group_file( + source_group=source_group, + file_format=file_format, + ), + ) + + return SourceRecordExportArchive( + archive_name=f"organization_source_records_export_{timestamp}.zip", + archive_bytes=archive_buffer.getvalue(), + files_count=len(source_groups), + ) + + +def _resolve_source_group_export_format( + *, + source_group: str, + requested_format: str, +) -> str: + if source_group == FINANCIAL_SOURCE_GROUP: + return EXPORT_FORMAT_JSON + return requested_format + + +def _build_source_group_file_name(*, source_group: str, file_format: str) -> str: + return f"{SOURCE_GROUP_EXPORT_FILE_STEMS[source_group]}.{file_format}" + + +def _source_group_queryset(source_group: str) -> QuerySet[OrganizationSourceRecord]: + return ( + OrganizationSourceRecord.objects.filter(extension__source_group=source_group) + .select_related("extension", "extension__organization") + .prefetch_related("financial_lines") + .order_by( + "extension__organization__name", + "source", + "record_date", + "uid", + ) + ) + + +def _render_source_group_file(*, source_group: str, file_format: str) -> bytes: + queryset = _source_group_queryset(source_group) + payload_headers = _collect_payload_headers(queryset) + headers = [ + *ORGANIZATION_EXPORT_FIELDS, + *SOURCE_RECORD_EXPORT_FIELDS, + *payload_headers, + ] + include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP + + if file_format == EXPORT_FORMAT_CSV: + return _render_csv_file( + queryset=_source_group_queryset(source_group), + headers=headers, + payload_headers=payload_headers, + ) + if file_format == EXPORT_FORMAT_XLSX: + return _render_xlsx_file( + queryset=_source_group_queryset(source_group), + headers=headers, + payload_headers=payload_headers, + ) + return _render_json_file( + queryset=_source_group_queryset(source_group), + headers=headers, + payload_headers=payload_headers, + include_financial_lines=include_financial_lines, + ) + + +def _collect_payload_headers( + queryset: QuerySet[OrganizationSourceRecord], +) -> list[str]: + payload_headers: set[str] = set() + + for record in queryset.iterator(chunk_size=1000): + payload_headers.update(_flatten_payload(record.payload).keys()) + + return sorted(payload_headers) + + +def _render_csv_file( + *, + queryset: QuerySet[OrganizationSourceRecord], + headers: Sequence[str], + payload_headers: Sequence[str], +) -> bytes: + output = StringIO() + writer = csv.DictWriter(output, fieldnames=list(headers), lineterminator="\n") + writer.writeheader() + + for record in queryset.iterator(chunk_size=1000): + row = _build_record_row(record, payload_headers=payload_headers) + writer.writerow({key: _serialize_flat_value(row.get(key)) for key in headers}) + + return ("\ufeff" + output.getvalue()).encode("utf-8") + + +def _render_xlsx_file( + *, + queryset: QuerySet[OrganizationSourceRecord], + headers: Sequence[str], + payload_headers: Sequence[str], +) -> bytes: + workbook = Workbook(write_only=True) + worksheet = workbook.create_sheet(title="data") + worksheet.append(list(headers)) + + for record in queryset.iterator(chunk_size=1000): + row = _build_record_row(record, payload_headers=payload_headers) + worksheet.append([_serialize_flat_value(row.get(key)) for key in headers]) + + output = BytesIO() + workbook.save(output) + return output.getvalue() + + +def _render_json_file( + *, + queryset: QuerySet[OrganizationSourceRecord], + headers: Sequence[str], + payload_headers: Sequence[str], + include_financial_lines: bool, +) -> bytes: + rows = [] + + for record in queryset.iterator(chunk_size=1000): + row = _build_record_row(record, payload_headers=payload_headers) + json_row = {key: _serialize_json_value(row.get(key)) for key in headers} + if include_financial_lines: + json_row["financial_lines"] = [ + { + field_name: _serialize_json_value( + getattr(financial_line, field_name) + ) + for field_name in FINANCIAL_LINE_FIELDS + } + for financial_line in record.financial_lines.all() + ] + rows.append(json_row) + + return json.dumps(rows, ensure_ascii=False, indent=2).encode("utf-8") + + +def _build_record_row( + record: OrganizationSourceRecord, + *, + payload_headers: Sequence[str], +) -> dict[str, Any]: + organization = record.extension.organization + payload = _flatten_payload(record.payload) + + row: dict[str, Any] = { + "Наименование": organization.full_name + or organization.short_name + or organization.name, + "ИНН": organization.inn, + "ОГРН": organization.ogrn, + "КПП": organization.kpp, + "uid": str(record.uid), + "source_group": record.extension.source_group, + "source": record.source, + "record_type": record.record_type, + "external_id": record.external_id, + "title": record.title, + "record_date": record.record_date, + "amount": record.amount, + "status": record.status, + "url": record.url, + "load_batch": record.load_batch, + "created_at": record.created_at, + "updated_at": record.updated_at, + } + + for header in payload_headers: + row[header] = payload.get(header) + + return row + + +def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]: + if not isinstance(value, dict): + return {prefix: value} if value not in (None, "") else {} + + flattened: dict[str, Any] = {} + for key, item in value.items(): + item_key = f"{prefix}.{key}" + if isinstance(item, dict) and item: + flattened.update(_flatten_payload(item, prefix=item_key)) + else: + flattened[item_key] = item + return flattened + + +def _serialize_flat_value(value: Any) -> str | int | float | bool: + if value is None: + return "" + if isinstance(value, dict | list | tuple): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + if isinstance(value, Decimal): + return str(value) + if isinstance(value, datetime | date): + return value.isoformat() + return value + + +def _serialize_json_value(value: Any) -> Any: + if isinstance(value, dict | list | tuple): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + if isinstance(value, Decimal): + return str(value) + if isinstance(value, datetime | date): + return value.isoformat() + return value diff --git a/src/organizations/views.py b/src/organizations/views.py index 2bd2383..44c7498 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -11,12 +11,14 @@ from django.conf import settings from django.core.cache import cache from django.db.models import CharField, Q from django.db.models.functions import Cast +from django.http import HttpResponse from django_filters import rest_framework as filters from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema +from rest_framework import status from rest_framework.decorators import action from rest_framework.filters import OrderingFilter, SearchFilter -from rest_framework.permissions import AllowAny, IsAuthenticated +from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ReadOnlyModelViewSet @@ -35,9 +37,11 @@ from organizations.models import ( from organizations.serializers import ( OrganizationSerializer, OrganizationSourceExtensionSerializer, + OrganizationSourceRecordExportRequestSerializer, OrganizationSourceRecordListResponseSerializer, OrganizationSourceRecordSerializer, ) +from organizations.source_record_export import build_source_records_export_archive ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations") FALSE_QUERY_VALUES = {"0", "false", "no", "off"} @@ -449,6 +453,8 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): ordering = ["-created_at", "-uid"] def get_permissions(self): + if self.action == "export": + return [IsAdminUser()] if getattr(settings, "ORGANIZATIONS_V2_ALLOW_ANONYMOUS", False): return [AllowAny()] return super().get_permissions() @@ -553,3 +559,39 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): ) def list(self, request, *args: Any, **kwargs: Any) -> Response: return super().list(request, *args, **kwargs) + + @swagger_auto_schema( + tags=[ORGANIZATIONS_TAG], + operation_id="v2_organization_source_records_export", + operation_summary="Выгрузить записи источников", + operation_description=( + "Формирует ZIP-архив с одним файлом на выбранную группу источников. " + "Финансово-экономические показатели всегда выгружаются в JSON." + ), + request_body=OrganizationSourceRecordExportRequestSerializer, + responses={ + 200: openapi.Response( + description="ZIP-архив записей источников.", + schema=openapi.Schema(type=openapi.TYPE_FILE), + ), + 400: "Некорректные параметры выгрузки.", + 403: "Доступ разрешён только администраторам.", + }, + ) + @action(detail=False, methods=["post"], url_path="export") + def export(self, request, *args: Any, **kwargs: Any) -> HttpResponse: + serializer = OrganizationSourceRecordExportRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + package = build_source_records_export_archive( + source_groups=serializer.validated_data["sources"], + export_format=serializer.validated_data["format"], + ) + response = HttpResponse(package.archive_bytes, content_type="application/zip") + response.status_code = status.HTTP_200_OK + response[ + "Content-Disposition" + ] = f'attachment; filename="{package.archive_name}"' + response["Content-Length"] = str(len(package.archive_bytes)) + response["X-Source-Export-Files"] = str(package.files_count) + return response diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py new file mode 100644 index 0000000..2217489 --- /dev/null +++ b/tests/apps/organizations/test_source_record_export.py @@ -0,0 +1,233 @@ +"""Tests for organization source-record ZIP export.""" + +import csv +import json +import zipfile +from io import BytesIO, StringIO + +from django.urls import reverse +from openpyxl import load_workbook +from organizations.models import ( + FinancialIndicatorsExtension, + Organization, + OrganizationSourceFinancialLine, + OrganizationSourceRecord, + PlannedInspectionExtension, + SourceGroup, +) +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.user.factories import UserFactory + + +class OrganizationSourceRecordExportApiV2Test(APITestCase): + """Checks admin-only source-record export contract.""" + + def setUp(self): + self.url = reverse( + "api_v2:organizations:organization-source-records-export", + ) + + def test_admin_exports_selected_sources_to_zip(self): + self.client.force_authenticate(UserFactory.create_superuser()) + organization = Organization.objects.create( + name='ООО "Экспорт"', + full_name='Общество с ограниченной ответственностью "Экспорт"', + inn="7707083810", + ogrn="1027700132010", + kpp="770701001", + ) + inspection_extension = PlannedInspectionExtension.objects.create( + organization=organization, + title="Плановые проверки Генпрокуратуры России", + ) + OrganizationSourceRecord.objects.create( + extension=inspection_extension, + record_type="inspection", + source="inspections", + external_id="INSP-1", + title="Проверка экспорт", + payload={ + "risk": {"score": 7}, + "tags": ["плановая", "надзор"], + }, + ) + financial_extension = FinancialIndicatorsExtension.objects.create( + organization=organization, + title="Финансово-экономические показатели", + ) + financial_record = OrganizationSourceRecord.objects.create( + extension=financial_extension, + record_type="financial_report", + source="fns_reports", + external_id="FIN-1", + title="Финансовый отчет", + payload={"period": 2025}, + ) + OrganizationSourceFinancialLine.objects.create( + source_record=financial_record, + form_code="1", + line_code="1600", + line_name="Баланс", + year=2025, + period_start=100, + period_end=200, + ) + + response = self.client.post( + self.url, + { + "sources": [ + SourceGroup.PLANNED_INSPECTIONS.value, + SourceGroup.FINANCIAL_INDICATORS.value, + ], + "format": "xlsx", + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response["Content-Type"], "application/zip") + self.assertIn( + 'filename="organization_source_records_export_', + response["Content-Disposition"], + ) + + with zipfile.ZipFile(BytesIO(response.content)) as archive: + self.assertEqual( + set(archive.namelist()), + {"planned-inspections.xlsx", "financial-indicators.json"}, + ) + + workbook = load_workbook( + BytesIO(archive.read("planned-inspections.xlsx")), + read_only=True, + ) + worksheet = workbook["data"] + rows = list(worksheet.iter_rows(values_only=True)) + self.assertEqual( + rows[0][:4], + ("Наименование", "ИНН", "ОГРН", "КПП"), + ) + self.assertEqual( + rows[1][:4], + ( + 'Общество с ограниченной ответственностью "Экспорт"', + "7707083810", + "1027700132010", + "770701001", + ), + ) + self.assertIn("payload.risk.score", rows[0]) + self.assertIn("payload.tags", rows[0]) + + financial_rows = json.loads( + archive.read("financial-indicators.json").decode("utf-8"), + ) + self.assertEqual(financial_rows[0]["Наименование"], organization.full_name) + self.assertEqual( + financial_rows[0]["financial_lines"][0]["line_code"], "1600" + ) + self.assertEqual(financial_rows[0]["financial_lines"][0]["period_end"], 200) + + def test_csv_export_uses_bom_and_canonical_columns_before_payload(self): + self.client.force_authenticate(UserFactory.create_superuser()) + organization = Organization.objects.create( + name='ООО "CSV"', + short_name='ООО "CSV"', + inn="7707083811", + ogrn="1027700132011", + kpp="770701002", + ) + extension = PlannedInspectionExtension.objects.create( + organization=organization, + title="Плановые проверки Генпрокуратуры России", + ) + OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="inspections", + external_id="INSP-CSV", + title="CSV проверка", + payload={"nested": {"value": "данные"}}, + ) + + response = self.client.post( + self.url, + { + "sources": [SourceGroup.PLANNED_INSPECTIONS.value], + "format": "csv", + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + with zipfile.ZipFile(BytesIO(response.content)) as archive: + csv_bytes = archive.read("planned-inspections.csv") + self.assertTrue(csv_bytes.startswith(b"\xef\xbb\xbf")) + csv_text = csv_bytes.decode("utf-8-sig") + csv_rows = list(csv.reader(StringIO(csv_text))) + + self.assertEqual(csv_rows[0][:4], ["Наименование", "ИНН", "ОГРН", "КПП"]) + self.assertEqual( + csv_rows[1][:4], ['ООО "CSV"', "7707083811", "1027700132011", "770701002"] + ) + self.assertIn("payload.nested.value", csv_rows[0]) + + def test_export_rejects_non_admin_user(self): + self.client.force_authenticate(UserFactory.create_user()) + + response = self.client.post( + self.url, + { + "sources": [SourceGroup.PLANNED_INSPECTIONS.value], + "format": "json", + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_export_rejects_empty_duplicate_and_unknown_values(self): + self.client.force_authenticate(UserFactory.create_superuser()) + + empty_response = self.client.post( + self.url, + {"sources": [], "format": "json"}, + format="json", + ) + duplicate_response = self.client.post( + self.url, + { + "sources": [ + SourceGroup.PLANNED_INSPECTIONS.value, + SourceGroup.PLANNED_INSPECTIONS.value, + ], + "format": "json", + }, + format="json", + ) + unknown_source_response = self.client.post( + self.url, + {"sources": ["unknown"], "format": "json"}, + format="json", + ) + invalid_format_response = self.client.post( + self.url, + { + "sources": [SourceGroup.PLANNED_INSPECTIONS.value], + "format": "xml", + }, + format="json", + ) + + self.assertEqual(empty_response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(duplicate_response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + unknown_source_response.status_code, status.HTTP_400_BAD_REQUEST + ) + self.assertEqual( + invalid_format_response.status_code, status.HTTP_400_BAD_REQUEST + )