fix: align media upload and gosedo records
All checks were successful
All checks were successful
This commit is contained in:
@@ -6,6 +6,7 @@ from apps.parsers.serializers import (
|
|||||||
)
|
)
|
||||||
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
|
from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor
|
||||||
from apps.parsers.views import (
|
from apps.parsers.views import (
|
||||||
|
MEDIA_NEWS_UPLOAD_FILE_PARAM,
|
||||||
RESULT_DETAIL_PARAMS,
|
RESULT_DETAIL_PARAMS,
|
||||||
RESULT_LIST_PARAMS,
|
RESULT_LIST_PARAMS,
|
||||||
UPLOAD_FILE_PARAM,
|
UPLOAD_FILE_PARAM,
|
||||||
@@ -123,7 +124,11 @@ def _upload_view(descriptor: ParserSourceDescriptor):
|
|||||||
f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
|
f"Ручная загрузка файла для источника: {descriptor.data_scope}. "
|
||||||
"Файл обрабатывается через Celery."
|
"Файл обрабатывается через Celery."
|
||||||
),
|
),
|
||||||
manual_parameters=[UPLOAD_FILE_PARAM],
|
manual_parameters=[
|
||||||
|
MEDIA_NEWS_UPLOAD_FILE_PARAM
|
||||||
|
if descriptor.key == "media_news"
|
||||||
|
else UPLOAD_FILE_PARAM
|
||||||
|
],
|
||||||
consumes=["multipart/form-data"],
|
consumes=["multipart/form-data"],
|
||||||
tags=[tag],
|
tags=[tag],
|
||||||
responses={202: ParserRunResponseSerializer, 400: "Ошибка валидации"},
|
responses={202: ParserRunResponseSerializer, 400: "Ошибка валидации"},
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from apps.core.serializers import BackgroundJobListSerializer
|
|||||||
from apps.core.services import BackgroundJobService
|
from apps.core.services import BackgroundJobService
|
||||||
from apps.parsers import tasks
|
from apps.parsers import tasks
|
||||||
from apps.parsers.fns_upload import FNSUploadService
|
from apps.parsers.fns_upload import FNSUploadService
|
||||||
|
from apps.parsers.media_news import MEDIA_NEWS_MAX_BYTES
|
||||||
from apps.parsers.models import (
|
from apps.parsers.models import (
|
||||||
VACANCY_RECORD_SOURCES,
|
VACANCY_RECORD_SOURCES,
|
||||||
FinancialReport,
|
FinancialReport,
|
||||||
@@ -374,6 +375,13 @@ UPLOAD_FILE_PARAM = openapi.Parameter(
|
|||||||
type=openapi.TYPE_FILE,
|
type=openapi.TYPE_FILE,
|
||||||
required=True,
|
required=True,
|
||||||
)
|
)
|
||||||
|
MEDIA_NEWS_UPLOAD_FILE_PARAM = openapi.Parameter(
|
||||||
|
"file",
|
||||||
|
openapi.IN_FORM,
|
||||||
|
description="XLSX-файл новостей СМИ размером не более 25 МиБ",
|
||||||
|
type=openapi.TYPE_FILE,
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
RESULT_LIST_PARAMS = [
|
RESULT_LIST_PARAMS = [
|
||||||
PAGE_PARAM,
|
PAGE_PARAM,
|
||||||
PAGE_SIZE_PARAM,
|
PAGE_SIZE_PARAM,
|
||||||
@@ -2793,19 +2801,30 @@ class ParserUploadView(APIView):
|
|||||||
)
|
)
|
||||||
serializer = ParserUploadRequestSerializer(data=request.data)
|
serializer = ParserUploadRequestSerializer(data=request.data)
|
||||||
serializer.is_valid(raise_exception=True)
|
serializer.is_valid(raise_exception=True)
|
||||||
if source_key == "media_news" and not serializer.validated_data[
|
|
||||||
"file"
|
|
||||||
].name.lower().endswith(".xlsx"):
|
|
||||||
return api_error_response(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"code": "invalid_file_type",
|
|
||||||
"message": "Для новостей требуется XLSX",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
status_code=status.HTTP_400_BAD_REQUEST,
|
|
||||||
)
|
|
||||||
uploaded_file = serializer.validated_data["file"]
|
uploaded_file = serializer.validated_data["file"]
|
||||||
|
if source_key == "media_news":
|
||||||
|
if not uploaded_file.name.lower().endswith(".xlsx"):
|
||||||
|
return api_error_response(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"code": "invalid_file_type",
|
||||||
|
"field": "file",
|
||||||
|
"message": "Для новостей требуется XLSX",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
|
if uploaded_file.size > MEDIA_NEWS_MAX_BYTES:
|
||||||
|
return api_error_response(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"code": "file_too_large",
|
||||||
|
"field": "file",
|
||||||
|
"message": "Размер XLSX для новостей не должен превышать 25 МиБ",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
file_path = _save_uploaded_parser_file(uploaded_file)
|
file_path = _save_uploaded_parser_file(uploaded_file)
|
||||||
run_serializer = ParserRunRequestSerializer(data={"file_path": file_path})
|
run_serializer = ParserRunRequestSerializer(data={"file_path": file_path})
|
||||||
run_serializer.is_valid(raise_exception=True)
|
run_serializer.is_valid(raise_exception=True)
|
||||||
|
|||||||
@@ -94,6 +94,8 @@ SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup]
|
|||||||
|
|
||||||
SOURCE_RECORD_ORDERING_FIELDS = (
|
SOURCE_RECORD_ORDERING_FIELDS = (
|
||||||
"record_date",
|
"record_date",
|
||||||
|
"record_type",
|
||||||
|
"external_id",
|
||||||
"extension__organization__name",
|
"extension__organization__name",
|
||||||
"extension__organization__full_name",
|
"extension__organization__full_name",
|
||||||
"created_at",
|
"created_at",
|
||||||
@@ -105,6 +107,9 @@ SOURCE_RECORD_ORDERING_FIELDS = (
|
|||||||
"extension__organization__okpo",
|
"extension__organization__okpo",
|
||||||
"status",
|
"status",
|
||||||
"payload__attestation_status",
|
"payload__attestation_status",
|
||||||
|
"payload__full_name",
|
||||||
|
"payload__medo_address",
|
||||||
|
"payload__registration_number",
|
||||||
"payload__sentiment",
|
"payload__sentiment",
|
||||||
"payload__news_source",
|
"payload__news_source",
|
||||||
)
|
)
|
||||||
@@ -225,11 +230,14 @@ SOURCE_RECORD_LIST_PARAMS = [
|
|||||||
_query_parameter(
|
_query_parameter(
|
||||||
"ordering",
|
"ordering",
|
||||||
description=(
|
description=(
|
||||||
"Сортировка по полям: record_date, extension__organization__name, "
|
"Сортировка по полям: record_date, record_type, external_id, "
|
||||||
|
"extension__organization__name, "
|
||||||
"extension__organization__full_name, created_at, updated_at, title, "
|
"extension__organization__full_name, created_at, updated_at, title, "
|
||||||
"uid, extension__organization__inn, extension__organization__ogrn, "
|
"uid, extension__organization__inn, extension__organization__ogrn, "
|
||||||
"extension__organization__okpo, status, "
|
"extension__organization__okpo, status, "
|
||||||
"payload__attestation_status, payload__sentiment, "
|
"payload__attestation_status, payload__full_name, "
|
||||||
|
"payload__medo_address, payload__registration_number, "
|
||||||
|
"payload__sentiment, "
|
||||||
"payload__news_source. Для обратной сортировки используйте префикс -. "
|
"payload__news_source. Для обратной сортировки используйте префикс -. "
|
||||||
"Значения record_date с null сортируются последними."
|
"Значения record_date с null сортируются последними."
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -389,6 +389,113 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase):
|
|||||||
expected,
|
expected,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_flat_gosedo_records_include_all_linked_types_and_record_sorting(self):
|
||||||
|
organization = create_frontend_organization(
|
||||||
|
name="Связанная организация",
|
||||||
|
inn="7707083828",
|
||||||
|
ogrn="1027700132028",
|
||||||
|
)
|
||||||
|
extension = ElectronicDocumentExchangeExtension.objects.create(
|
||||||
|
organization=organization,
|
||||||
|
title="Электронный документооборот",
|
||||||
|
)
|
||||||
|
for (
|
||||||
|
record_type,
|
||||||
|
external_id,
|
||||||
|
title,
|
||||||
|
full_name,
|
||||||
|
medo_address,
|
||||||
|
registration_number,
|
||||||
|
) in (
|
||||||
|
(
|
||||||
|
"participant",
|
||||||
|
"GOSEDO-PARTICIPANT",
|
||||||
|
"Участник",
|
||||||
|
"Полное имя участника",
|
||||||
|
"MEDO-3",
|
||||||
|
"3",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"operator",
|
||||||
|
"GOSEDO-OPERATOR",
|
||||||
|
"Оператор",
|
||||||
|
"Полное имя оператора",
|
||||||
|
"MEDO-1",
|
||||||
|
"1",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"organizer",
|
||||||
|
"GOSEDO-ORGANIZER",
|
||||||
|
"Организатор",
|
||||||
|
"Полное имя организатора",
|
||||||
|
"MEDO-2",
|
||||||
|
"2",
|
||||||
|
),
|
||||||
|
):
|
||||||
|
OrganizationSourceRecord.objects.create(
|
||||||
|
extension=extension,
|
||||||
|
record_type=record_type,
|
||||||
|
source="gosedo_address_directory",
|
||||||
|
external_id=external_id,
|
||||||
|
title=title,
|
||||||
|
payload={
|
||||||
|
"full_name": full_name,
|
||||||
|
"medo_address": medo_address,
|
||||||
|
"registration_number": registration_number,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
endpoint = reverse("api_v2:organizations:organization-source-records-list")
|
||||||
|
params = {
|
||||||
|
"source_group": "electronic_document_exchange",
|
||||||
|
"source": "gosedo_address_directory",
|
||||||
|
"organization": str(organization.uid),
|
||||||
|
}
|
||||||
|
response = self.client.get(endpoint, params)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
self.assertEqual(
|
||||||
|
{item["record_type"] for item in response.data["data"]},
|
||||||
|
{"participant", "operator", "organizer"},
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_by_ordering = {
|
||||||
|
"record_type": [
|
||||||
|
"GOSEDO-OPERATOR",
|
||||||
|
"GOSEDO-ORGANIZER",
|
||||||
|
"GOSEDO-PARTICIPANT",
|
||||||
|
],
|
||||||
|
"external_id": [
|
||||||
|
"GOSEDO-OPERATOR",
|
||||||
|
"GOSEDO-ORGANIZER",
|
||||||
|
"GOSEDO-PARTICIPANT",
|
||||||
|
],
|
||||||
|
"payload__full_name": [
|
||||||
|
"GOSEDO-OPERATOR",
|
||||||
|
"GOSEDO-ORGANIZER",
|
||||||
|
"GOSEDO-PARTICIPANT",
|
||||||
|
],
|
||||||
|
"payload__medo_address": [
|
||||||
|
"GOSEDO-OPERATOR",
|
||||||
|
"GOSEDO-ORGANIZER",
|
||||||
|
"GOSEDO-PARTICIPANT",
|
||||||
|
],
|
||||||
|
"payload__registration_number": [
|
||||||
|
"GOSEDO-OPERATOR",
|
||||||
|
"GOSEDO-ORGANIZER",
|
||||||
|
"GOSEDO-PARTICIPANT",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
for ordering, expected in expected_by_ordering.items():
|
||||||
|
with self.subTest(ordering=ordering):
|
||||||
|
response = self.client.get(endpoint, {**params, "ordering": ordering})
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
self.assertEqual(
|
||||||
|
[item["external_id"] for item in response.data["data"]],
|
||||||
|
expected,
|
||||||
|
)
|
||||||
|
|
||||||
def test_flat_media_records_support_frontend_sorting_fields(self):
|
def test_flat_media_records_support_frontend_sorting_fields(self):
|
||||||
organization = create_frontend_organization(
|
organization = create_frontend_organization(
|
||||||
name="Media sorting organization",
|
name="Media sorting organization",
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from apps.parsers.gosedo import (
|
|||||||
stable_gosedo_uid,
|
stable_gosedo_uid,
|
||||||
)
|
)
|
||||||
from apps.parsers.media_news import (
|
from apps.parsers.media_news import (
|
||||||
|
MEDIA_NEWS_MAX_BYTES,
|
||||||
import_media_news,
|
import_media_news,
|
||||||
normalize_news_text,
|
normalize_news_text,
|
||||||
stable_media_external_id,
|
stable_media_external_id,
|
||||||
@@ -619,6 +620,58 @@ class MediaNewsPermissionsTest(APITestCase):
|
|||||||
task_id,
|
task_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_upload_rejects_non_xlsx_before_saving_or_queueing(self):
|
||||||
|
self.client.force_authenticate(self.admin)
|
||||||
|
with patch("apps.parsers.views._save_uploaded_parser_file") as save_file, patch(
|
||||||
|
"apps.parsers.tasks.parse_media_news.apply_async"
|
||||||
|
) as apply_async:
|
||||||
|
response = self.client.post(
|
||||||
|
self.url,
|
||||||
|
{"file": SimpleUploadedFile("media.csv", b"headline")},
|
||||||
|
format="multipart",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
|
self.assertEqual(response.data["errors"][0]["code"], "invalid_file_type")
|
||||||
|
save_file.assert_not_called()
|
||||||
|
apply_async.assert_not_called()
|
||||||
|
|
||||||
|
def test_upload_rejects_oversized_xlsx_before_saving_or_queueing(self):
|
||||||
|
self.client.force_authenticate(self.admin)
|
||||||
|
with patch("apps.parsers.views.MEDIA_NEWS_MAX_BYTES", 1), patch(
|
||||||
|
"apps.parsers.views._save_uploaded_parser_file"
|
||||||
|
) as save_file, patch(
|
||||||
|
"apps.parsers.tasks.parse_media_news.apply_async"
|
||||||
|
) as apply_async:
|
||||||
|
response = self.client.post(
|
||||||
|
self.url,
|
||||||
|
{"file": SimpleUploadedFile("media.xlsx", b"xx")},
|
||||||
|
format="multipart",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
|
self.assertEqual(response.data["errors"][0]["code"], "file_too_large")
|
||||||
|
save_file.assert_not_called()
|
||||||
|
apply_async.assert_not_called()
|
||||||
|
|
||||||
|
def test_upload_accepts_xlsx_at_size_limit(self):
|
||||||
|
self.client.force_authenticate(self.admin)
|
||||||
|
with patch("apps.parsers.views.MEDIA_NEWS_MAX_BYTES", 1), patch(
|
||||||
|
"apps.parsers.views._save_uploaded_parser_file",
|
||||||
|
return_value="parser_uploads/media.xlsx",
|
||||||
|
), patch(
|
||||||
|
"apps.parsers.tasks.parse_media_news.apply_async",
|
||||||
|
side_effect=lambda **kwargs: SimpleNamespace(id=kwargs["task_id"]),
|
||||||
|
):
|
||||||
|
response = self.client.post(
|
||||||
|
self.url,
|
||||||
|
{"file": SimpleUploadedFile("media.xlsx", b"x")},
|
||||||
|
format="multipart",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
|
||||||
|
self.assertEqual(MEDIA_NEWS_MAX_BYTES, 25 * 1024 * 1024)
|
||||||
|
|
||||||
def test_gosedo_manual_run_is_admin_only_and_returns_task_ids(self):
|
def test_gosedo_manual_run_is_admin_only_and_returns_task_ids(self):
|
||||||
url = reverse(
|
url = reverse(
|
||||||
"api_v1:parsers:run-parser",
|
"api_v1:parsers:run-parser",
|
||||||
|
|||||||
Reference in New Issue
Block a user