From 82ba7b78cddfba2e86b806cdc1a2ed100680a6aa Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Thu, 7 May 2026 14:39:20 +0200 Subject: [PATCH] Add v2 registry uploads and source CSV exports --- src/apps/parsers/api_v2_urls.py | 39 ++++++++ src/apps/parsers/serializers.py | 1 + src/apps/parsers/views.py | 71 ++++++++++++++ src/core/api_v2_urls.py | 4 + src/registers/serializers.py | 12 +++ src/registers/services.py | 43 +++++++- src/registers/urls.py | 19 ++++ src/registers/views.py | 96 +++++++++++++++++- src/templates/dashboard.html | 114 +++++++++++++++++++++- tests/apps/parsers/test_dashboard_page.py | 22 +++++ tests/apps/parsers/test_views.py | 40 ++++++++ tests/apps/registers/test_services.py | 94 ++++++++++++++++++ tests/apps/registers/test_views.py | 90 +++++++++++++++++ 13 files changed, 637 insertions(+), 8 deletions(-) create mode 100644 src/apps/parsers/api_v2_urls.py diff --git a/src/apps/parsers/api_v2_urls.py b/src/apps/parsers/api_v2_urls.py new file mode 100644 index 0000000..6b5d8db --- /dev/null +++ b/src/apps/parsers/api_v2_urls.py @@ -0,0 +1,39 @@ +"""API v2 routes for parser source exports.""" + +from apps.parsers.models import ParserLoadLog +from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor +from apps.parsers.views import SourceResultCsvDownloadView +from django.urls import path + +app_name = "parser_sources" + + +def _download_source_descriptors(): + """Return unique source result routes that support CSV download.""" + seen_routes = set() + for descriptor in PARSER_SOURCES.values(): + if not descriptor.api_route or descriptor.api_route in seen_routes: + continue + seen_routes.add(descriptor.api_route) + if descriptor.source == ParserLoadLog.Source.FNS_REPORTS: + continue + yield descriptor + + +def _download_view(descriptor: ParserSourceDescriptor): + class SourceDownloadView(SourceResultCsvDownloadView): + source_key = descriptor.key + + return SourceDownloadView.as_view() + + +urlpatterns = [] +for source_descriptor in _download_source_descriptors(): + route_name = source_descriptor.api_route.replace("/", "-") + urlpatterns.append( + path( + f"{source_descriptor.api_route}/download/", + _download_view(source_descriptor), + name=f"{route_name}-download", + ) + ) diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index ea72310..52b16a3 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -520,6 +520,7 @@ class ParserSourceSerializer(serializers.Serializer): parser_strategy = serializers.CharField() source_notes = serializers.CharField(allow_blank=True) supports_file_upload = serializers.BooleanField() + api_route = serializers.CharField(allow_blank=True) result_list_url = serializers.CharField() result_detail_url = serializers.CharField() upload_url = serializers.CharField(allow_blank=True) diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 2fcd554..0a79595 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -2295,6 +2295,77 @@ class SourceResultListView(APIView): ) +SOURCE_CSV_HEADERS = [ + "id", + "load_batch", + "source", + "external_id", + "inn", + "ogrn", + "organisation_name", + "title", + "record_date", + "amount", + "status", + "url", + "payload", + "created_at", + "updated_at", +] + + +def _csv_cell(value) -> str: + if value is None: + return "" + if isinstance(value, dict): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + if hasattr(value, "isoformat"): + return value.isoformat() + return str(value) + + +def _source_csv_filename(descriptor) -> str: + route_name = descriptor.api_route.replace("/", "-") or descriptor.key + return f"{get_valid_filename(route_name)}.csv" + + +class SourceResultCsvDownloadView(APIView): + """CSV download результата конкретного источника API v2.""" + + permission_classes = [IsAuthenticated] + source_key = "" + + def get(self, request: Request, source_key: str | None = None): + resolved_source_key = source_key or self.source_key + query_serializer = ParserResultQuerySerializer(data=request.query_params) + query_serializer.is_valid(raise_exception=True) + params = query_serializer.validated_data + params["include_payload"] = True + + descriptor, queryset = _filter_result_queryset(resolved_source_key, params) + if descriptor is None: + return _source_not_found_response(resolved_source_key) + if descriptor.source == ParserLoadLog.Source.FNS_REPORTS: + return _source_not_found_response(resolved_source_key) + + response = HttpResponse(content_type="text/csv; charset=utf-8") + response["Content-Disposition"] = ( + f'attachment; filename="{_source_csv_filename(descriptor)}"' + ) + + writer = csv.writer(response) + writer.writerow(SOURCE_CSV_HEADERS) + for record in queryset.iterator(chunk_size=1000): + row = _result_record_to_dict( + descriptor.source, + record, + include_payload=True, + ) + writer.writerow([_csv_cell(row.get(header)) for header in SOURCE_CSV_HEADERS]) + + return response + + class SourceResultDetailView(APIView): """GET одной записи результата источника.""" diff --git a/src/core/api_v2_urls.py b/src/core/api_v2_urls.py index b2bbeea..ff3de9a 100644 --- a/src/core/api_v2_urls.py +++ b/src/core/api_v2_urls.py @@ -1,10 +1,14 @@ """API v2 URL configuration.""" +from apps.parsers.api_v2_urls import urlpatterns as parser_source_urlpatterns from django.urls import include, path from organizations.urls import organizations_urlpatterns +from registers.urls import registers_v2_urlpatterns app_name = "api_v2" urlpatterns = [ path("", include((organizations_urlpatterns, "organizations"))), + path("registers/", include((registers_v2_urlpatterns, "registers"))), + path("sources/", include((parser_source_urlpatterns, "parser_sources"))), ] diff --git a/src/registers/serializers.py b/src/registers/serializers.py index 3945be9..49b4610 100644 --- a/src/registers/serializers.py +++ b/src/registers/serializers.py @@ -115,6 +115,18 @@ class RegisterFileUploadSerializer(serializers.Serializer): return value +class FixedRegisterFileUploadSerializer(serializers.Serializer): + """Сериализатор загрузки файла в предопределенный реестр.""" + + actual_date = serializers.DateField(required=False) + file = serializers.FileField() + + def validate_file(self, value): + if not value.name.lower().endswith(".xlsx"): + raise serializers.ValidationError("Поддерживаются только файлы .xlsx") + return value + + class OrganizationListQuerySerializer(serializers.Serializer): """Сериализатор query-параметров списка организаций.""" diff --git a/src/registers/services.py b/src/registers/services.py index aca67b7..1863293 100644 --- a/src/registers/services.py +++ b/src/registers/services.py @@ -39,6 +39,12 @@ class RegisterImportService: """Сервис импорта организаций из Excel в выбранный реестр.""" REQUIRED_HEADERS = {"pn_name", "mn_ogrn", "mn_inn", "mn_okpo"} + HEADER_ALIASES = { + "full_name": "pn_name", + "ogrn": "mn_ogrn", + "inn": "mn_inn", + "okpo": "mn_okpo", + } @classmethod def sync_registry_memberships( @@ -501,6 +507,11 @@ class RegisterImportService: for row_number, row_values in enumerate(row_iter, start=2): if cls._is_empty_row(row_values): continue + if cls._is_skippable_branch_row( + row_values=row_values, + header_index_map=header_index_map, + ): + continue organizations.append( ParsedOrganization( @@ -607,7 +618,8 @@ class RegisterImportService: @staticmethod def _normalize_header(value) -> str: - return str(value or "").strip().lower() + header = str(value or "").strip().lower() + return RegisterImportService.HEADER_ALIASES.get(header, header) @staticmethod def _is_empty_row(row_values: tuple) -> bool: @@ -616,6 +628,35 @@ class RegisterImportService: return False return True + @classmethod + def _is_skippable_branch_row( + cls, + *, + row_values: tuple, + header_index_map: dict[str, int], + ) -> bool: + filial_index = header_index_map.get("filial") + if filial_index is None: + return False + + filial_value = row_values[filial_index] + if not cls._is_truthy_cell(filial_value): + return False + + ogrn = row_values[header_index_map["mn_ogrn"]] + inn = row_values[header_index_map["mn_inn"]] + return cls._is_blank_cell(ogrn) and cls._is_blank_cell(inn) + + @staticmethod + def _is_blank_cell(value) -> bool: + return value is None or str(value).strip() == "" + + @staticmethod + def _is_truthy_cell(value) -> bool: + if isinstance(value, bool): + return value + return str(value or "").strip().lower() in {"1", "true", "yes", "да"} + @classmethod def _as_required_text(cls, value, *, field_name: str, row_number: int) -> str: text_value = str(value or "").strip() diff --git a/src/registers/urls.py b/src/registers/urls.py index ffdda4e..496d619 100644 --- a/src/registers/urls.py +++ b/src/registers/urls.py @@ -4,6 +4,8 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter from registers.views import ( + REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG, + FixedRegisterUploadView, OrganizationViewSet, RegisterUploadView, RegisterViewSet, @@ -26,4 +28,21 @@ registers_urlpatterns = [ path("", include(router.urls)), ] +registers_v2_urlpatterns = [ + *[ + path( + f"{slug}/upload/", + FixedRegisterUploadView.as_view(registry_name=registry_name), + name=f"register-upload-{slug}", + ) + for slug, registry_name in REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG.items() + ], + path( + "registries//organizations/", + RegistryOrganizationListView.as_view(), + name="registry-organizations-list", + ), + path("", include(router.urls)), +] + urlpatterns = [] diff --git a/src/registers/views.py b/src/registers/views.py index a205d9e..0bf7062 100644 --- a/src/registers/views.py +++ b/src/registers/views.py @@ -7,6 +7,7 @@ from django.db.models import Count, Q from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema +from organizations.tasks import refresh_all_organization_data_snapshots from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.generics import ListAPIView @@ -19,6 +20,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from registers.models import Organization, Register from registers.pagination import RegistersPagination from registers.serializers import ( + FixedRegisterFileUploadSerializer, OrganizationDetailSerializer, OrganizationListQuerySerializer, OrganizationSerializer, @@ -32,6 +34,20 @@ from registers.services import RegisterImportError, RegisterImportService REGISTERS_TAG = swagger_tag("Реестры организаций", "registers") +REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG = { + "opk": "Реестр предприятий ОПК", + "roscosmos": "Реестр госкорпорации Роскосмос", + "roscosmos-goz": "Реестр госкорпорации Роскосмос ГОЗ", + "roscosmos-opk": "Реестр госкорпорации Роскосмос ОПК", + "rosatom": "Реестр госкорпорации Росатом", + "rosatom-goz": "Реестр госкорпорации Росатом ГОЗ", + "rosatom-opk": "Реестр госкорпорации Росатом ОПК", +} + + +def _start_snapshot_refresh_task() -> None: + refresh_all_organization_data_snapshots.delay() + class RegisterViewSet(ReadOnlyModelViewSet): """API для просмотра списка реестров.""" @@ -346,7 +362,7 @@ class RegisterUploadView(APIView): actual_date = serializer.validated_data.get("actual_date") try: - RegisterImportService.sync_registry_memberships( + result = RegisterImportService.sync_registry_memberships( registry=registry, uploaded_file=uploaded_file, file_name=uploaded_file.name, @@ -356,7 +372,83 @@ class RegisterUploadView(APIView): except RegisterImportError as exc: raise ValidationError({"file": str(exc)}) from exc + _start_snapshot_refresh_task() + return Response( - {"success": True, "message": "Файл успешно загружен"}, + { + "success": True, + "message": "Файл успешно загружен", + **result, + }, + status=status.HTTP_201_CREATED, + ) + + +class FixedRegisterUploadView(APIView): + """API v2 загрузки Excel файла организаций в предопределенный реестр.""" + + parser_classes = [MultiPartParser] + permission_classes = [IsAdminUser] + registry_name = "" + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Загрузка списка организаций в фиксированный реестр", + operation_description=( + "Загружает Excel (.xlsx) с организациями в реестр, заданный URL.\n" + "Требуемые колонки: pn_name, mn_ogrn, mn_inn, mn_okpo.\n" + "Опциональная колонка: in_kpp." + ), + manual_parameters=[ + openapi.Parameter( + name="file", + in_=openapi.IN_FORM, + type=openapi.TYPE_FILE, + required=True, + description="Excel файл с организациями", + ), + openapi.Parameter( + name="actual_date", + in_=openapi.IN_FORM, + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, + required=False, + description="Дата актуальности среза", + ), + ], + consumes=["multipart/form-data"], + responses={ + 201: RegisterUploadSuccessSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN, + }, + ) + def post(self, request): + serializer = FixedRegisterFileUploadSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + registry = get_object_or_404(Register, name=self.registry_name) + uploaded_file = serializer.validated_data["file"] + actual_date = serializer.validated_data.get("actual_date") + + try: + result = RegisterImportService.sync_registry_memberships( + registry=registry, + uploaded_file=uploaded_file, + file_name=uploaded_file.name, + actual_date=actual_date, + uploaded_by=request.user, + ) + except RegisterImportError as exc: + raise ValidationError({"file": str(exc)}) from exc + + _start_snapshot_refresh_task() + + return Response( + { + "success": True, + "message": "Файл успешно загружен", + **result, + }, status=status.HTTP_201_CREATED, ) diff --git a/src/templates/dashboard.html b/src/templates/dashboard.html index 0d0f2b9..fb09966 100644 --- a/src/templates/dashboard.html +++ b/src/templates/dashboard.html @@ -1537,7 +1537,10 @@

Источник

- +
+ + +
@@ -1834,7 +1837,7 @@
- Запрос уйдет в /api/v1/registers/upload/ + Запрос уйдет в /api/v2/registers/{slug}/upload/
@@ -2188,6 +2191,24 @@ "trudvsem", "vacancies", ]; + const REGISTRY_UPLOAD_SLUGS_BY_NAME = { + "Реестр предприятий ОПК": "opk", + "Реестр госкорпорации Роскосмос": "roscosmos", + "Реестр госкорпорации Роскосмос ГОЗ": "roscosmos-goz", + "Реестр госкорпорации Роскосмос ОПК": "roscosmos-opk", + "Реестр госкорпорации Росатом": "rosatom", + "Реестр госкорпорации Росатом ГОЗ": "rosatom-goz", + "Реестр госкорпорации Росатом ОПК": "rosatom-opk", + }; + const REGISTRY_UPLOAD_URLS_BY_SLUG = { + opk: "/api/v2/registers/opk/upload/", + roscosmos: "/api/v2/registers/roscosmos/upload/", + "roscosmos-goz": "/api/v2/registers/roscosmos-goz/upload/", + "roscosmos-opk": "/api/v2/registers/roscosmos-opk/upload/", + rosatom: "/api/v2/registers/rosatom/upload/", + "rosatom-goz": "/api/v2/registers/rosatom-goz/upload/", + "rosatom-opk": "/api/v2/registers/rosatom-opk/upload/", + }; const ORGANIZATION_TABLE_COLUMN_LABELS = { "id": "ID", "external_id": "Внешний ID", @@ -2327,6 +2348,24 @@ return (dashboardData?.sources || []).find((source) => source.key === sourceKey); } + function sourceCsvDownloadUrl(source) { + if (!source?.api_route || source.source === "fns_reports") return ""; + return `/api/v2/sources/${source.api_route}/download/`; + } + + function registryUploadSlug(registry) { + return REGISTRY_UPLOAD_SLUGS_BY_NAME[registry?.name || ""] || ""; + } + + function registryUploadUrlForSelectedRegistry() { + const registryId = $("registrySelect").value; + const registry = latestRegistries.find((item) => String(item.id) === registryId); + const registrySlug = registryUploadSlug(registry); + if (!registrySlug) return ""; + return REGISTRY_UPLOAD_URLS_BY_SLUG[registrySlug] + || `/api/v2/registers/${registrySlug}/upload/`; + } + function withQuery(url, params) { const query = new URLSearchParams(params); return `${url}?${query.toString()}`; @@ -2586,6 +2625,35 @@ return data; } + async function apiDownload(url, filenameFallback = "download.csv") { + const started = performance.now(); + const response = await fetch(url, { + method: "GET", + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + }, + }); + const disposition = response.headers.get("content-disposition") || ""; + if (response.ok) { + const blob = await response.blob(); + const filenameMatch = disposition.match(/filename="([^"]+)"/); + const filename = filenameMatch ? filenameMatch[1] : filenameFallback; + logRequest("GET", url, {}, `${response.status} ${Math.round(performance.now() - started)}ms`, `blob: ${blob.size} bytes`); + const objectUrl = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = objectUrl; + link.download = filename; + link.click(); + URL.revokeObjectURL(objectUrl); + return { downloaded: true, filename }; + } + const text = await response.text(); + logRequest("GET", url, {}, `${response.status} ${Math.round(performance.now() - started)}ms`, text); + let data = {}; + try { data = text ? JSON.parse(text) : {}; } catch (_) { data = { raw: text }; } + throw attachStatus(data, response.status); + } + function formPayload(form) { const data = {}; new FormData(form).forEach((value, key) => { @@ -2845,7 +2913,7 @@ ["Organizations v2", "Organizations: search", "GET", `/api/v2/organizations/?page_size=1&search=770&has_registry=false&exclude_data=${encodeURIComponent(ORGANIZATION_EXCLUDED_DATA_SOURCES)}`], ["Registers", "Registries", "GET", "/api/v1/registers/registries/?page_size=1"], ["Registers", "Organizations", "GET", "/api/v1/registers/organizations/?page_size=1"], - ["Registers", "Upload registry", "POST", "/api/v1/registers/upload/"], + ["Registers", "Upload registry", "POST", "/api/v2/registers/opk/upload/"], ["Backups", "Export .bin", "POST", "/api/v1/backups/export/"], ["Exchange", "Connections", "GET", "/api/v1/exchange/connections/"], ["Exchange", "Create connection", "POST", "/api/v1/exchange/connections/"], @@ -2861,6 +2929,18 @@ ["System", "Parser logs export", "GET", "/api/v1/system/logs/export/"], ].forEach(([group, title, method, url]) => addEndpoint(catalog, { group, title, method, url })); + sourceList.forEach((source) => { + const csvUrl = sourceCsvDownloadUrl(source); + if (!csvUrl) return; + addEndpoint(catalog, { + group: "Sources v2", + title: `${source.title}: CSV`, + method: "GET", + url: csvUrl, + description: "CSV выгрузка результата источника", + }); + }); + [ { group: "Native data", @@ -4006,6 +4086,7 @@ $("sourceCards").innerHTML = items.map((source) => { const load = latestLoadForSource(source); const records = sourceRecordCount(source); + const csvUrl = sourceCsvDownloadUrl(source); return `
@@ -4023,6 +4104,7 @@
+ ${csvUrl ? `` : ""}
`; @@ -4080,6 +4162,7 @@
${escapeHtml(source.source_notes || "Дополнительное описание не задано в каталоге источников.")}
+ ${sourceCsvDownloadUrl(source) ? `` : ""} ${source.supports_file_upload ? `ручная загрузка разрешена` : `только официальный API`}
`; @@ -4159,7 +4242,7 @@ || registries.find((registry) => registry.name.includes("ОПК")) || registries[0]; $("registrySelect").innerHTML = registries.map((registry) => ( - `` + `` )).join(""); if (previousValue && registries.some((registry) => String(registry.id) === previousValue)) { $("registrySelect").value = previousValue; @@ -4171,6 +4254,7 @@ $("registrySummary").innerHTML = registries.map((registry) => `
${escapeHtml(registry.name)} + ${registryUploadSlug(registry) ? `upload: /api/v2/registers/${escapeHtml(registryUploadSlug(registry))}/upload/` : "upload: не настроен"} организаций: ${escapeHtml(registry.active_organizations ?? 0)} загрузок: ${escapeHtml(registry.uploads_count ?? 0)}
@@ -4528,6 +4612,9 @@ $("routeSubtitle").textContent = itemId ? `${source.agency} · запись #${itemId}` : `${source.agency} · ${source.source} · ${source.parser_strategy}`; + const csvUrl = sourceCsvDownloadUrl(source); + $("routeDownloadCsvButton").classList.toggle("hidden", !csvUrl || Boolean(itemId)); + $("routeDownloadCsvButton").dataset.sourceDownload = csvUrl && !itemId ? source.key : ""; }, showRouteError(title, subtitle, message) { $("routeTitle").textContent = title; @@ -4646,6 +4733,13 @@ } } + async function downloadSourceCsv(sourceKey) { + const source = sourceByKey(sourceKey); + const downloadUrl = sourceCsvDownloadUrl(source); + if (!downloadUrl) return; + await apiDownload(downloadUrl, `${sourceKey}.csv`); + } + async function refreshDashboard() { if (!accessToken) return; try { @@ -4776,9 +4870,15 @@ const formData = new FormData(event.target); const file = formData.get("file"); if (!file || !file.name) return; + const uploadUrl = registryUploadUrlForSelectedRegistry(); + if (!uploadUrl) { + $("registryUploadStatus").textContent = "Для выбранного реестра не настроен v2 upload endpoint"; + return; + } + formData.delete("registry"); $("registryUploadStatus").textContent = "Загрузка..."; try { - await apiUpload("/api/v1/registers/upload/", formData); + await apiUpload(uploadUrl, formData); event.target.querySelector('input[type="file"]').value = ""; $("registryUploadStatus").textContent = "Файл загружен"; await refreshDashboard(); @@ -4868,6 +4968,7 @@ const exchangeDelete = target.closest?.("[data-exchange-delete]"); const endpointRun = target.closest?.("[data-endpoint-run]"); const endpointDetail = target.closest?.("[data-endpoint-detail]"); + const sourceDownload = target.closest?.("[data-source-download]"); if (target.dataset.mainTab) showMainTab(target.dataset.mainTab); if (target.id === "requestDrawerButton") openLeftDrawer(); if (target.id === "operationsDrawerButton") openRightDrawer("jobs"); @@ -4968,6 +5069,9 @@ } if (target.id === "reloadRouteButton") await renderCurrentRoute(); if (target.id === "backToDashboardButton") navigateDashboard("/dashboard"); + if (sourceDownload?.dataset.sourceDownload) { + await downloadSourceCsv(sourceDownload.dataset.sourceDownload); + } if (target.dataset.sourceLink) { closeModals(); navigateDashboard(`/dashboard/${encodeURIComponent(target.dataset.sourceLink)}`); diff --git a/tests/apps/parsers/test_dashboard_page.py b/tests/apps/parsers/test_dashboard_page.py index 377968f..c360752 100644 --- a/tests/apps/parsers/test_dashboard_page.py +++ b/tests/apps/parsers/test_dashboard_page.py @@ -151,3 +151,25 @@ class ParserDashboardPageTest(TestCase): self.assertIn("На конец", content) self.assertIn("На начало", content) self.assertIn("Покрытие доп. данными", content) + + def test_dashboard_uses_v2_registry_upload_routes(self): + response = self.client.get("/dashboard") + + self.assertEqual(response.status_code, 200) + content = response.content.decode() + self.assertIn("REGISTRY_UPLOAD_SLUGS_BY_NAME", content) + self.assertIn("/api/v2/registers/${registrySlug}/upload/", content) + self.assertIn("/api/v2/registers/opk/upload/", content) + self.assertIn("registryUploadUrlForSelectedRegistry", content) + + def test_dashboard_exposes_v2_source_csv_downloads(self): + response = self.client.get("/dashboard") + + self.assertEqual(response.status_code, 200) + content = response.content.decode() + self.assertIn("function sourceCsvDownloadUrl", content) + self.assertIn("/api/v2/sources/${source.api_route}/download/", content) + self.assertIn("data-source-download", content) + self.assertIn("downloadSourceCsv", content) + self.assertIn("CSV", content) + self.assertNotIn("/api/v2/sources/fns/reports/download/", content) diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index e771300..5ebf730 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import io import os import tempfile @@ -99,6 +100,41 @@ class ParsersViewSetTest(APITestCase): ) self.assertEqual(detail.status_code, status.HTTP_200_OK) + def test_v2_source_csv_download_exports_source_rows(self): + record = IndustrialCertificateRecordFactory( + certificate_number="CERT-CSV-1", + organisation_name='АО "CSV"', + inn="7701000101", + ogrn="1027700000001", + ) + self.client.force_authenticate(self.user) + + response = self.client.get( + reverse("api_v2:parser_sources:minpromtorg-certificates-download") + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response["Content-Type"], "text/csv; charset=utf-8") + self.assertIn( + 'attachment; filename="minpromtorg-certificates.csv"', + response["Content-Disposition"], + ) + rows = list(csv.DictReader(io.StringIO(response.content.decode("utf-8")))) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["id"], str(record.id)) + self.assertEqual(rows[0]["source"], ParserLoadLog.Source.INDUSTRIAL) + self.assertEqual(rows[0]["external_id"], "CERT-CSV-1") + self.assertEqual(rows[0]["organisation_name"], 'АО "CSV"') + self.assertEqual(rows[0]["inn"], "7701000101") + self.assertIn("CERT-CSV-1", rows[0]["payload"]) + + def test_v2_source_csv_download_is_not_registered_for_financial_reports(self): + self.client.force_authenticate(self.user) + + response = self.client.get("/api/v2/sources/fns/reports/download/") + + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + def test_manufacturers_list_and_retrieve(self): record = ManufacturerRecordFactory() second_record = ManufacturerRecordFactory() @@ -404,6 +440,10 @@ class ParsersViewSetTest(APITestCase): sources["procurements_44fz"]["result_list_url"], "/api/v1/parsers/results/procurements_44fz/", ) + self.assertEqual( + sources["procurements_44fz"]["api_route"], + "eis/procurements-44fz", + ) self.assertEqual( sources["procurements_223fz"]["result_list_url"], "/api/v1/parsers/results/procurements_223fz/", diff --git a/tests/apps/registers/test_services.py b/tests/apps/registers/test_services.py index de2e590..84eb4c0 100644 --- a/tests/apps/registers/test_services.py +++ b/tests/apps/registers/test_services.py @@ -160,6 +160,100 @@ class RegisterImportServiceTest(TestCase): self.assertEqual(existing.in_kpp, 444) self.assertEqual(existing.mn_okpo, "87654321") + def test_parse_xlsx_accepts_opk_source_header_aliases(self): + upload = _upload( + "opk.xlsx", + [ + [ + "rn", + "okpo", + "ogrn", + "inn", + "filial", + "ropk_num", + "ropk_razdel_num", + "ropk_razdel_name", + "short_name", + "full_name", + ], + [ + 1, + "07506197", + "1027600980990", + "7601000086", + "", + "1", + "1", + "Раздел", + 'АО "ЯРЗ"', + 'АКЦИОНЕРНОЕ ОБЩЕСТВО "ЯРОСЛАВСКИЙ РАДИОЗАВОД"', + ], + ], + ) + + rows = RegisterImportService.parse_xlsx(upload) + + self.assertEqual(len(rows), 1) + self.assertEqual( + rows[0].pn_name, + 'АКЦИОНЕРНОЕ ОБЩЕСТВО "ЯРОСЛАВСКИЙ РАДИОЗАВОД"', + ) + self.assertEqual(rows[0].mn_ogrn, 1027600980990) + self.assertEqual(rows[0].mn_inn, 7601000086) + self.assertIsNone(rows[0].in_kpp) + self.assertEqual(rows[0].mn_okpo, "07506197") + + def test_parse_xlsx_skips_opk_branch_rows_without_identity(self): + upload = _upload( + "opk.xlsx", + [ + [ + "rn", + "okpo", + "ogrn", + "inn", + "filial", + "ropk_num", + "ropk_razdel_num", + "ropk_razdel_name", + "short_name", + "full_name", + ], + [ + 100, + "52511425", + None, + None, + True, + 48, + 1, + "Минпромторг России", + 'Филиал ПАО "Ил" - ВАСО', + ( + 'Филиал публичного акционерного общества "Авиационный ' + 'комплекс им. С.В. Ильюшина" - ВАСО' + ), + ], + [ + 1, + "07506197", + "1027600980990", + "7601000086", + False, + 1, + 1, + "Минпромторг России", + 'АО "ЯРЗ"', + 'АКЦИОНЕРНОЕ ОБЩЕСТВО "ЯРОСЛАВСКИЙ РАДИОЗАВОД"', + ], + ], + ) + + rows = RegisterImportService.parse_xlsx(upload) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0].mn_ogrn, 1027600980990) + def test_get_active_periods_by_org_returns_mapping(self): registry = RegisterFactory() active_period = RegistryMembershipPeriodFactory( diff --git a/tests/apps/registers/test_views.py b/tests/apps/registers/test_views.py index d909997..e108c82 100644 --- a/tests/apps/registers/test_views.py +++ b/tests/apps/registers/test_views.py @@ -4,6 +4,7 @@ from __future__ import annotations import io from datetime import date +from unittest.mock import patch from apps.registers.models import Organization, RegistryMembershipPeriod from django.core.files.uploadedfile import SimpleUploadedFile @@ -94,6 +95,35 @@ class RegistersViewsTest(APITestCase): self.client.force_authenticate(self.user) return response + def _post_v2_slug_upload( + self, + *, + slug: str, + rows: list[dict], + actual_date_value: date, + with_kpp: bool = True, + file_name: str = "registry.xlsx", + ): + self.client.force_authenticate(self.admin) + content = _build_register_excel_bytes(rows, with_kpp=with_kpp) + upload = SimpleUploadedFile( + file_name, + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + response = self.client.post( + reverse(f"api_v2:registers:register-upload-{slug}"), + { + "actual_date": actual_date_value.isoformat(), + "file": upload, + }, + format="multipart", + ) + self.client.force_authenticate(self.user) + return response + def test_registries_list_and_retrieve(self): registry = RegisterFactory(name="Росатом") RegisterUploadFactory(registry=registry) @@ -130,6 +160,66 @@ class RegistersViewsTest(APITestCase): self.assertIn("Реестр госкорпорации Росатом ГОЗ", names) self.assertIn("Реестр госкорпорации Росатом ОПК", names) + def test_v2_registry_slug_upload_uses_fixed_registry_and_refreshes_snapshots(self): + rows = [ + { + "pn_name": 'АО "Росатом ГОЗ"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07506197", + } + ] + + with patch("registers.views._start_snapshot_refresh_task") as refresh_task: + response = self._post_v2_slug_upload( + slug="rosatom-goz", + rows=rows, + actual_date_value=date(2026, 5, 7), + file_name="rosatom_goz.xlsx", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertTrue(response.data["success"]) + self.assertEqual( + response.data["registry_name"], + "Реестр госкорпорации Росатом ГОЗ", + ) + refresh_task.assert_called_once_with() + membership = RegistryMembershipPeriod.objects.get(ended_at__isnull=True) + self.assertEqual( + membership.registry.name, + "Реестр госкорпорации Росатом ГОЗ", + ) + + def test_v2_registry_slug_upload_url_does_not_duplicate_registers_segment(self): + self.assertEqual( + reverse("api_v2:registers:register-upload-rosatom-goz"), + "/api/v2/registers/rosatom-goz/upload/", + ) + + def test_v2_registry_slug_upload_does_not_refresh_snapshots_after_import_error(self): + rows = [ + { + "pn_name": 'АО "Невалидный ОКПО"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07A06197", + } + ] + + with patch("registers.views._start_snapshot_refresh_task") as refresh_task: + response = self._post_v2_slug_upload( + slug="rosatom-goz", + rows=rows, + actual_date_value=date(2026, 5, 7), + file_name="invalid_rosatom_goz.xlsx", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + refresh_task.assert_not_called() + def test_organizations_list_and_retrieve(self): organization = OrganizationFactory()