Add v2 registry uploads and source CSV exports
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 20s
CI/CD Pipeline / Build and Push Images (push) Successful in 10s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s

This commit is contained in:
2026-05-07 14:39:20 +02:00
parent 507ae2063a
commit 15360a3c8e
13 changed files with 639 additions and 8 deletions

View File

@@ -0,0 +1,39 @@
"""API v2 routes for parser source exports."""
from apps.parsers.models import ParserLoadLog
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
from apps.parsers.views import SourceResultCsvDownloadView
from django.urls import path
app_name = "parser_sources"
def _download_source_descriptors():
"""Return unique source result routes that support CSV download."""
seen_routes = set()
for descriptor in PARSER_SOURCES.values():
if not descriptor.api_route or descriptor.api_route in seen_routes:
continue
seen_routes.add(descriptor.api_route)
if descriptor.source == ParserLoadLog.Source.FNS_REPORTS:
continue
yield descriptor
def _download_view(descriptor: ParserSourceDescriptor):
class SourceDownloadView(SourceResultCsvDownloadView):
source_key = descriptor.key
return SourceDownloadView.as_view()
urlpatterns = []
for source_descriptor in _download_source_descriptors():
route_name = source_descriptor.api_route.replace("/", "-")
urlpatterns.append(
path(
f"{source_descriptor.api_route}/download/",
_download_view(source_descriptor),
name=f"{route_name}-download",
)
)

View File

@@ -520,6 +520,7 @@ class ParserSourceSerializer(serializers.Serializer):
parser_strategy = serializers.CharField()
source_notes = serializers.CharField(allow_blank=True)
supports_file_upload = serializers.BooleanField()
api_route = serializers.CharField(allow_blank=True)
result_list_url = serializers.CharField()
result_detail_url = serializers.CharField()
upload_url = serializers.CharField(allow_blank=True)

View File

@@ -2295,6 +2295,79 @@ class SourceResultListView(APIView):
)
SOURCE_CSV_HEADERS = [
"id",
"load_batch",
"source",
"external_id",
"inn",
"ogrn",
"organisation_name",
"title",
"record_date",
"amount",
"status",
"url",
"payload",
"created_at",
"updated_at",
]
def _csv_cell(value) -> str:
if value is None:
return ""
if isinstance(value, dict):
return json.dumps(value, ensure_ascii=False, sort_keys=True)
if hasattr(value, "isoformat"):
return value.isoformat()
return str(value)
def _source_csv_filename(descriptor) -> str:
route_name = descriptor.api_route.replace("/", "-") or descriptor.key
return f"{get_valid_filename(route_name)}.csv"
class SourceResultCsvDownloadView(APIView):
"""CSV download результата конкретного источника API v2."""
permission_classes = [IsAuthenticated]
source_key = ""
def get(self, request: Request, source_key: str | None = None):
resolved_source_key = source_key or self.source_key
query_serializer = ParserResultQuerySerializer(data=request.query_params)
query_serializer.is_valid(raise_exception=True)
params = query_serializer.validated_data
params["include_payload"] = True
descriptor, queryset = _filter_result_queryset(resolved_source_key, params)
if descriptor is None:
return _source_not_found_response(resolved_source_key)
if descriptor.source == ParserLoadLog.Source.FNS_REPORTS:
return _source_not_found_response(resolved_source_key)
response = HttpResponse(content_type="text/csv; charset=utf-8")
response[
"Content-Disposition"
] = f'attachment; filename="{_source_csv_filename(descriptor)}"'
writer = csv.writer(response)
writer.writerow(SOURCE_CSV_HEADERS)
for record in queryset.iterator(chunk_size=1000):
row = _result_record_to_dict(
descriptor.source,
record,
include_payload=True,
)
writer.writerow(
[_csv_cell(row.get(header)) for header in SOURCE_CSV_HEADERS]
)
return response
class SourceResultDetailView(APIView):
"""GET одной записи результата источника."""

View File

@@ -1,10 +1,14 @@
"""API v2 URL configuration."""
from apps.parsers.api_v2_urls import urlpatterns as parser_source_urlpatterns
from django.urls import include, path
from organizations.urls import organizations_urlpatterns
from registers.urls import registers_v2_urlpatterns
app_name = "api_v2"
urlpatterns = [
path("", include((organizations_urlpatterns, "organizations"))),
path("registers/", include((registers_v2_urlpatterns, "registers"))),
path("sources/", include((parser_source_urlpatterns, "parser_sources"))),
]

View File

@@ -115,6 +115,18 @@ class RegisterFileUploadSerializer(serializers.Serializer):
return value
class FixedRegisterFileUploadSerializer(serializers.Serializer):
"""Сериализатор загрузки файла в предопределенный реестр."""
actual_date = serializers.DateField(required=False)
file = serializers.FileField()
def validate_file(self, value):
if not value.name.lower().endswith(".xlsx"):
raise serializers.ValidationError("Поддерживаются только файлы .xlsx")
return value
class OrganizationListQuerySerializer(serializers.Serializer):
"""Сериализатор query-параметров списка организаций."""

View File

@@ -39,6 +39,12 @@ class RegisterImportService:
"""Сервис импорта организаций из Excel в выбранный реестр."""
REQUIRED_HEADERS = {"pn_name", "mn_ogrn", "mn_inn", "mn_okpo"}
HEADER_ALIASES = {
"full_name": "pn_name",
"ogrn": "mn_ogrn",
"inn": "mn_inn",
"okpo": "mn_okpo",
}
@classmethod
def sync_registry_memberships(
@@ -501,6 +507,11 @@ class RegisterImportService:
for row_number, row_values in enumerate(row_iter, start=2):
if cls._is_empty_row(row_values):
continue
if cls._is_skippable_branch_row(
row_values=row_values,
header_index_map=header_index_map,
):
continue
organizations.append(
ParsedOrganization(
@@ -607,7 +618,8 @@ class RegisterImportService:
@staticmethod
def _normalize_header(value) -> str:
return str(value or "").strip().lower()
header = str(value or "").strip().lower()
return RegisterImportService.HEADER_ALIASES.get(header, header)
@staticmethod
def _is_empty_row(row_values: tuple) -> bool:
@@ -616,6 +628,35 @@ class RegisterImportService:
return False
return True
@classmethod
def _is_skippable_branch_row(
cls,
*,
row_values: tuple,
header_index_map: dict[str, int],
) -> bool:
filial_index = header_index_map.get("filial")
if filial_index is None:
return False
filial_value = row_values[filial_index]
if not cls._is_truthy_cell(filial_value):
return False
ogrn = row_values[header_index_map["mn_ogrn"]]
inn = row_values[header_index_map["mn_inn"]]
return cls._is_blank_cell(ogrn) and cls._is_blank_cell(inn)
@staticmethod
def _is_blank_cell(value) -> bool:
return value is None or str(value).strip() == ""
@staticmethod
def _is_truthy_cell(value) -> bool:
if isinstance(value, bool):
return value
return str(value or "").strip().lower() in {"1", "true", "yes", "да"}
@classmethod
def _as_required_text(cls, value, *, field_name: str, row_number: int) -> str:
text_value = str(value or "").strip()

View File

@@ -4,6 +4,8 @@ from django.urls import include, path
from rest_framework.routers import DefaultRouter
from registers.views import (
REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG,
FixedRegisterUploadView,
OrganizationViewSet,
RegisterUploadView,
RegisterViewSet,
@@ -26,4 +28,21 @@ registers_urlpatterns = [
path("", include(router.urls)),
]
registers_v2_urlpatterns = [
*[
path(
f"{slug}/upload/",
FixedRegisterUploadView.as_view(registry_name=registry_name),
name=f"register-upload-{slug}",
)
for slug, registry_name in REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG.items()
],
path(
"registries/<uuid:registry_id>/organizations/",
RegistryOrganizationListView.as_view(),
name="registry-organizations-list",
),
path("", include(router.urls)),
]
urlpatterns = []

View File

@@ -7,6 +7,7 @@ from django.db.models import Count, Q
from django.shortcuts import get_object_or_404
from drf_yasg import openapi
from drf_yasg.utils import swagger_auto_schema
from organizations.tasks import refresh_all_organization_data_snapshots
from rest_framework import status
from rest_framework.exceptions import ValidationError
from rest_framework.generics import ListAPIView
@@ -19,6 +20,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet
from registers.models import Organization, Register
from registers.pagination import RegistersPagination
from registers.serializers import (
FixedRegisterFileUploadSerializer,
OrganizationDetailSerializer,
OrganizationListQuerySerializer,
OrganizationSerializer,
@@ -32,6 +34,20 @@ from registers.services import RegisterImportError, RegisterImportService
REGISTERS_TAG = swagger_tag("Реестры организаций", "registers")
REGISTER_UPLOAD_REGISTRY_NAMES_BY_SLUG = {
"opk": "Реестр предприятий ОПК",
"roscosmos": "Реестр госкорпорации Роскосмос",
"roscosmos-goz": "Реестр госкорпорации Роскосмос ГОЗ",
"roscosmos-opk": "Реестр госкорпорации Роскосмос ОПК",
"rosatom": "Реестр госкорпорации Росатом",
"rosatom-goz": "Реестр госкорпорации Росатом ГОЗ",
"rosatom-opk": "Реестр госкорпорации Росатом ОПК",
}
def _start_snapshot_refresh_task() -> None:
refresh_all_organization_data_snapshots.delay()
class RegisterViewSet(ReadOnlyModelViewSet):
"""API для просмотра списка реестров."""
@@ -346,7 +362,7 @@ class RegisterUploadView(APIView):
actual_date = serializer.validated_data.get("actual_date")
try:
RegisterImportService.sync_registry_memberships(
result = RegisterImportService.sync_registry_memberships(
registry=registry,
uploaded_file=uploaded_file,
file_name=uploaded_file.name,
@@ -356,7 +372,83 @@ class RegisterUploadView(APIView):
except RegisterImportError as exc:
raise ValidationError({"file": str(exc)}) from exc
_start_snapshot_refresh_task()
return Response(
{"success": True, "message": "Файл успешно загружен"},
{
"success": True,
"message": "Файл успешно загружен",
**result,
},
status=status.HTTP_201_CREATED,
)
class FixedRegisterUploadView(APIView):
"""API v2 загрузки Excel файла организаций в предопределенный реестр."""
parser_classes = [MultiPartParser]
permission_classes = [IsAdminUser]
registry_name = ""
@swagger_auto_schema(
tags=[REGISTERS_TAG],
operation_summary="Загрузка списка организаций в фиксированный реестр",
operation_description=(
"Загружает Excel (.xlsx) с организациями в реестр, заданный URL.\n"
"Требуемые колонки: pn_name, mn_ogrn, mn_inn, mn_okpo.\n"
"Опциональная колонка: in_kpp."
),
manual_parameters=[
openapi.Parameter(
name="file",
in_=openapi.IN_FORM,
type=openapi.TYPE_FILE,
required=True,
description="Excel файл с организациями",
),
openapi.Parameter(
name="actual_date",
in_=openapi.IN_FORM,
type=openapi.TYPE_STRING,
format=openapi.FORMAT_DATE,
required=False,
description="Дата актуальности среза",
),
],
consumes=["multipart/form-data"],
responses={
201: RegisterUploadSuccessSerializer,
400: CommonResponses.BAD_REQUEST,
**ErrorResponses.ADMIN,
},
)
def post(self, request):
serializer = FixedRegisterFileUploadSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
registry = get_object_or_404(Register, name=self.registry_name)
uploaded_file = serializer.validated_data["file"]
actual_date = serializer.validated_data.get("actual_date")
try:
result = RegisterImportService.sync_registry_memberships(
registry=registry,
uploaded_file=uploaded_file,
file_name=uploaded_file.name,
actual_date=actual_date,
uploaded_by=request.user,
)
except RegisterImportError as exc:
raise ValidationError({"file": str(exc)}) from exc
_start_snapshot_refresh_task()
return Response(
{
"success": True,
"message": "Файл успешно загружен",
**result,
},
status=status.HTTP_201_CREATED,
)

View File

@@ -1537,7 +1537,10 @@
<h2 id="routeTitle">Источник</h2>
<span id="routeSubtitle" class="muted"></span>
</div>
<button id="reloadRouteButton" class="secondary" type="button">Обновить данные</button>
<div class="toolbar">
<button id="routeDownloadCsvButton" class="secondary hidden" data-source-download="" type="button">CSV</button>
<button id="reloadRouteButton" class="secondary" type="button">Обновить данные</button>
</div>
</div>
<div id="sourceTableView">
<div id="sourceRecordsApp" class="source-records-app" v-cloak>
@@ -1834,7 +1837,7 @@
</div>
<div class="row">
<button type="submit">Загрузить организации ОПК</button>
<span class="muted">Запрос уйдет в <code>/api/v1/registers/upload/</code></span>
<span class="muted">Запрос уйдет в <code>/api/v2/registers/{slug}/upload/</code></span>
<span id="registryUploadStatus" class="muted"></span>
</div>
</form>
@@ -2188,6 +2191,24 @@
"trudvsem",
"vacancies",
];
const REGISTRY_UPLOAD_SLUGS_BY_NAME = {
"Реестр предприятий ОПК": "opk",
"Реестр госкорпорации Роскосмос": "roscosmos",
"Реестр госкорпорации Роскосмос ГОЗ": "roscosmos-goz",
"Реестр госкорпорации Роскосмос ОПК": "roscosmos-opk",
"Реестр госкорпорации Росатом": "rosatom",
"Реестр госкорпорации Росатом ГОЗ": "rosatom-goz",
"Реестр госкорпорации Росатом ОПК": "rosatom-opk",
};
const REGISTRY_UPLOAD_URLS_BY_SLUG = {
opk: "/api/v2/registers/opk/upload/",
roscosmos: "/api/v2/registers/roscosmos/upload/",
"roscosmos-goz": "/api/v2/registers/roscosmos-goz/upload/",
"roscosmos-opk": "/api/v2/registers/roscosmos-opk/upload/",
rosatom: "/api/v2/registers/rosatom/upload/",
"rosatom-goz": "/api/v2/registers/rosatom-goz/upload/",
"rosatom-opk": "/api/v2/registers/rosatom-opk/upload/",
};
const ORGANIZATION_TABLE_COLUMN_LABELS = {
"id": "ID",
"external_id": "Внешний ID",
@@ -2327,6 +2348,24 @@
return (dashboardData?.sources || []).find((source) => source.key === sourceKey);
}
function sourceCsvDownloadUrl(source) {
if (!source?.api_route || source.source === "fns_reports") return "";
return `/api/v2/sources/${source.api_route}/download/`;
}
function registryUploadSlug(registry) {
return REGISTRY_UPLOAD_SLUGS_BY_NAME[registry?.name || ""] || "";
}
function registryUploadUrlForSelectedRegistry() {
const registryId = $("registrySelect").value;
const registry = latestRegistries.find((item) => String(item.id) === registryId);
const registrySlug = registryUploadSlug(registry);
if (!registrySlug) return "";
return REGISTRY_UPLOAD_URLS_BY_SLUG[registrySlug]
|| `/api/v2/registers/${registrySlug}/upload/`;
}
function withQuery(url, params) {
const query = new URLSearchParams(params);
return `${url}?${query.toString()}`;
@@ -2586,6 +2625,35 @@
return data;
}
async function apiDownload(url, filenameFallback = "download.csv") {
const started = performance.now();
const response = await fetch(url, {
method: "GET",
headers: {
...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}),
},
});
const disposition = response.headers.get("content-disposition") || "";
if (response.ok) {
const blob = await response.blob();
const filenameMatch = disposition.match(/filename="([^"]+)"/);
const filename = filenameMatch ? filenameMatch[1] : filenameFallback;
logRequest("GET", url, {}, `${response.status} ${Math.round(performance.now() - started)}ms`, `blob: ${blob.size} bytes`);
const objectUrl = URL.createObjectURL(blob);
const link = document.createElement("a");
link.href = objectUrl;
link.download = filename;
link.click();
URL.revokeObjectURL(objectUrl);
return { downloaded: true, filename };
}
const text = await response.text();
logRequest("GET", url, {}, `${response.status} ${Math.round(performance.now() - started)}ms`, text);
let data = {};
try { data = text ? JSON.parse(text) : {}; } catch (_) { data = { raw: text }; }
throw attachStatus(data, response.status);
}
function formPayload(form) {
const data = {};
new FormData(form).forEach((value, key) => {
@@ -2845,7 +2913,7 @@
["Organizations v2", "Organizations: search", "GET", `/api/v2/organizations/?page_size=1&search=770&has_registry=false&exclude_data=${encodeURIComponent(ORGANIZATION_EXCLUDED_DATA_SOURCES)}`],
["Registers", "Registries", "GET", "/api/v1/registers/registries/?page_size=1"],
["Registers", "Organizations", "GET", "/api/v1/registers/organizations/?page_size=1"],
["Registers", "Upload registry", "POST", "/api/v1/registers/upload/"],
["Registers", "Upload registry", "POST", "/api/v2/registers/opk/upload/"],
["Backups", "Export .bin", "POST", "/api/v1/backups/export/"],
["Exchange", "Connections", "GET", "/api/v1/exchange/connections/"],
["Exchange", "Create connection", "POST", "/api/v1/exchange/connections/"],
@@ -2861,6 +2929,18 @@
["System", "Parser logs export", "GET", "/api/v1/system/logs/export/"],
].forEach(([group, title, method, url]) => addEndpoint(catalog, { group, title, method, url }));
sourceList.forEach((source) => {
const csvUrl = sourceCsvDownloadUrl(source);
if (!csvUrl) return;
addEndpoint(catalog, {
group: "Sources v2",
title: `${source.title}: CSV`,
method: "GET",
url: csvUrl,
description: "CSV выгрузка результата источника",
});
});
[
{
group: "Native data",
@@ -4006,6 +4086,7 @@
$("sourceCards").innerHTML = items.map((source) => {
const load = latestLoadForSource(source);
const records = sourceRecordCount(source);
const csvUrl = sourceCsvDownloadUrl(source);
return `
<article class="source-card">
<div class="source-title">
@@ -4023,6 +4104,7 @@
<div class="card-actions">
<button class="secondary" data-source-detail="${escapeHtml(source.key)}" type="button">Подробнее</button>
<button data-source-link="${escapeHtml(source.key)}" type="button">Данные</button>
${csvUrl ? `<button class="secondary" data-source-download="${escapeHtml(source.key)}" type="button">CSV</button>` : ""}
</div>
</article>
`;
@@ -4080,6 +4162,7 @@
<div class="empty-state">${escapeHtml(source.source_notes || "Дополнительное описание не задано в каталоге источников.")}</div>
<div class="row" style="margin-top: 12px;">
<button data-source-link="${escapeHtml(source.key)}" type="button">Открыть данные</button>
${sourceCsvDownloadUrl(source) ? `<button class="secondary" data-source-download="${escapeHtml(source.key)}" type="button">CSV</button>` : ""}
${source.supports_file_upload ? `<span class="badge warn">ручная загрузка разрешена</span>` : `<span class="badge info">только официальный API</span>`}
</div>
`;
@@ -4159,7 +4242,7 @@
|| registries.find((registry) => registry.name.includes("ОПК"))
|| registries[0];
$("registrySelect").innerHTML = registries.map((registry) => (
`<option value="${escapeHtml(registry.id)}">${escapeHtml(registry.name)}</option>`
`<option value="${escapeHtml(registry.id)}" ${registryUploadSlug(registry) ? "" : "disabled"}>${escapeHtml(registry.name)}</option>`
)).join("");
if (previousValue && registries.some((registry) => String(registry.id) === previousValue)) {
$("registrySelect").value = previousValue;
@@ -4171,6 +4254,7 @@
$("registrySummary").innerHTML = registries.map((registry) => `
<div class="registry-card">
<strong>${escapeHtml(registry.name)}</strong>
<span class="muted">${registryUploadSlug(registry) ? `upload: /api/v2/registers/${escapeHtml(registryUploadSlug(registry))}/upload/` : "upload: не настроен"}</span>
<span class="muted">организаций: ${escapeHtml(registry.active_organizations ?? 0)}</span>
<span class="muted">загрузок: ${escapeHtml(registry.uploads_count ?? 0)}</span>
</div>
@@ -4528,6 +4612,9 @@
$("routeSubtitle").textContent = itemId
? `${source.agency} · запись #${itemId}`
: `${source.agency} · ${source.source} · ${source.parser_strategy}`;
const csvUrl = sourceCsvDownloadUrl(source);
$("routeDownloadCsvButton").classList.toggle("hidden", !csvUrl || Boolean(itemId));
$("routeDownloadCsvButton").dataset.sourceDownload = csvUrl && !itemId ? source.key : "";
},
showRouteError(title, subtitle, message) {
$("routeTitle").textContent = title;
@@ -4646,6 +4733,13 @@
}
}
async function downloadSourceCsv(sourceKey) {
const source = sourceByKey(sourceKey);
const downloadUrl = sourceCsvDownloadUrl(source);
if (!downloadUrl) return;
await apiDownload(downloadUrl, `${sourceKey}.csv`);
}
async function refreshDashboard() {
if (!accessToken) return;
try {
@@ -4776,9 +4870,15 @@
const formData = new FormData(event.target);
const file = formData.get("file");
if (!file || !file.name) return;
const uploadUrl = registryUploadUrlForSelectedRegistry();
if (!uploadUrl) {
$("registryUploadStatus").textContent = "Для выбранного реестра не настроен v2 upload endpoint";
return;
}
formData.delete("registry");
$("registryUploadStatus").textContent = "Загрузка...";
try {
await apiUpload("/api/v1/registers/upload/", formData);
await apiUpload(uploadUrl, formData);
event.target.querySelector('input[type="file"]').value = "";
$("registryUploadStatus").textContent = "Файл загружен";
await refreshDashboard();
@@ -4868,6 +4968,7 @@
const exchangeDelete = target.closest?.("[data-exchange-delete]");
const endpointRun = target.closest?.("[data-endpoint-run]");
const endpointDetail = target.closest?.("[data-endpoint-detail]");
const sourceDownload = target.closest?.("[data-source-download]");
if (target.dataset.mainTab) showMainTab(target.dataset.mainTab);
if (target.id === "requestDrawerButton") openLeftDrawer();
if (target.id === "operationsDrawerButton") openRightDrawer("jobs");
@@ -4968,6 +5069,9 @@
}
if (target.id === "reloadRouteButton") await renderCurrentRoute();
if (target.id === "backToDashboardButton") navigateDashboard("/dashboard");
if (sourceDownload?.dataset.sourceDownload) {
await downloadSourceCsv(sourceDownload.dataset.sourceDownload);
}
if (target.dataset.sourceLink) {
closeModals();
navigateDashboard(`/dashboard/${encodeURIComponent(target.dataset.sourceLink)}`);