diff --git a/src/apps/parsers/api_result_urls.py b/src/apps/parsers/api_result_urls.py index 8102c62..0050455 100644 --- a/src/apps/parsers/api_result_urls.py +++ b/src/apps/parsers/api_result_urls.py @@ -6,6 +6,7 @@ from apps.parsers.serializers import ( ) from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor from apps.parsers.views import ( + MEDIA_NEWS_UPLOAD_FILE_PARAM, RESULT_DETAIL_PARAMS, RESULT_LIST_PARAMS, UPLOAD_FILE_PARAM, @@ -123,7 +124,11 @@ def _upload_view(descriptor: ParserSourceDescriptor): f"Ручная загрузка файла для источника: {descriptor.data_scope}. " "Файл обрабатывается через Celery." ), - manual_parameters=[UPLOAD_FILE_PARAM], + manual_parameters=[ + MEDIA_NEWS_UPLOAD_FILE_PARAM + if descriptor.key == "media_news" + else UPLOAD_FILE_PARAM + ], consumes=["multipart/form-data"], tags=[tag], responses={202: ParserRunResponseSerializer, 400: "Ошибка валидации"}, diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index fb806bc..993dce4 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -19,6 +19,7 @@ from apps.core.serializers import BackgroundJobListSerializer from apps.core.services import BackgroundJobService from apps.parsers import tasks from apps.parsers.fns_upload import FNSUploadService +from apps.parsers.media_news import MEDIA_NEWS_MAX_BYTES from apps.parsers.models import ( VACANCY_RECORD_SOURCES, FinancialReport, @@ -374,6 +375,13 @@ UPLOAD_FILE_PARAM = openapi.Parameter( type=openapi.TYPE_FILE, required=True, ) +MEDIA_NEWS_UPLOAD_FILE_PARAM = openapi.Parameter( + "file", + openapi.IN_FORM, + description="XLSX-файл новостей СМИ размером не более 25 МиБ", + type=openapi.TYPE_FILE, + required=True, +) RESULT_LIST_PARAMS = [ PAGE_PARAM, PAGE_SIZE_PARAM, @@ -2793,19 +2801,30 @@ class ParserUploadView(APIView): ) serializer = ParserUploadRequestSerializer(data=request.data) serializer.is_valid(raise_exception=True) - if source_key == "media_news" and not serializer.validated_data[ - "file" - ].name.lower().endswith(".xlsx"): - return api_error_response( - [ - { - "code": "invalid_file_type", - "message": "Для новостей требуется XLSX", - } - ], - status_code=status.HTTP_400_BAD_REQUEST, - ) uploaded_file = serializer.validated_data["file"] + if source_key == "media_news": + if not uploaded_file.name.lower().endswith(".xlsx"): + return api_error_response( + [ + { + "code": "invalid_file_type", + "field": "file", + "message": "Для новостей требуется XLSX", + } + ], + status_code=status.HTTP_400_BAD_REQUEST, + ) + if uploaded_file.size > MEDIA_NEWS_MAX_BYTES: + return api_error_response( + [ + { + "code": "file_too_large", + "field": "file", + "message": "Размер XLSX для новостей не должен превышать 25 МиБ", + } + ], + status_code=status.HTTP_400_BAD_REQUEST, + ) file_path = _save_uploaded_parser_file(uploaded_file) run_serializer = ParserRunRequestSerializer(data={"file_path": file_path}) run_serializer.is_valid(raise_exception=True) diff --git a/src/organizations/views.py b/src/organizations/views.py index 8d29e3e..48be882 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -94,6 +94,8 @@ SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup] SOURCE_RECORD_ORDERING_FIELDS = ( "record_date", + "record_type", + "external_id", "extension__organization__name", "extension__organization__full_name", "created_at", @@ -105,6 +107,9 @@ SOURCE_RECORD_ORDERING_FIELDS = ( "extension__organization__okpo", "status", "payload__attestation_status", + "payload__full_name", + "payload__medo_address", + "payload__registration_number", "payload__sentiment", "payload__news_source", ) @@ -225,11 +230,14 @@ SOURCE_RECORD_LIST_PARAMS = [ _query_parameter( "ordering", description=( - "Сортировка по полям: record_date, extension__organization__name, " + "Сортировка по полям: record_date, record_type, external_id, " + "extension__organization__name, " "extension__organization__full_name, created_at, updated_at, title, " "uid, extension__organization__inn, extension__organization__ogrn, " "extension__organization__okpo, status, " - "payload__attestation_status, payload__sentiment, " + "payload__attestation_status, payload__full_name, " + "payload__medo_address, payload__registration_number, " + "payload__sentiment, " "payload__news_source. Для обратной сортировки используйте префикс -. " "Значения record_date с null сортируются последними." ), diff --git a/tests/apps/organizations/test_api_v2_source_extensions.py b/tests/apps/organizations/test_api_v2_source_extensions.py index c083ec5..48e7a3e 100644 --- a/tests/apps/organizations/test_api_v2_source_extensions.py +++ b/tests/apps/organizations/test_api_v2_source_extensions.py @@ -389,6 +389,113 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): expected, ) + def test_flat_gosedo_records_include_all_linked_types_and_record_sorting(self): + organization = create_frontend_organization( + name="Связанная организация", + inn="7707083828", + ogrn="1027700132028", + ) + extension = ElectronicDocumentExchangeExtension.objects.create( + organization=organization, + title="Электронный документооборот", + ) + for ( + record_type, + external_id, + title, + full_name, + medo_address, + registration_number, + ) in ( + ( + "participant", + "GOSEDO-PARTICIPANT", + "Участник", + "Полное имя участника", + "MEDO-3", + "3", + ), + ( + "operator", + "GOSEDO-OPERATOR", + "Оператор", + "Полное имя оператора", + "MEDO-1", + "1", + ), + ( + "organizer", + "GOSEDO-ORGANIZER", + "Организатор", + "Полное имя организатора", + "MEDO-2", + "2", + ), + ): + OrganizationSourceRecord.objects.create( + extension=extension, + record_type=record_type, + source="gosedo_address_directory", + external_id=external_id, + title=title, + payload={ + "full_name": full_name, + "medo_address": medo_address, + "registration_number": registration_number, + }, + ) + + endpoint = reverse("api_v2:organizations:organization-source-records-list") + params = { + "source_group": "electronic_document_exchange", + "source": "gosedo_address_directory", + "organization": str(organization.uid), + } + response = self.client.get(endpoint, params) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + {item["record_type"] for item in response.data["data"]}, + {"participant", "operator", "organizer"}, + ) + + expected_by_ordering = { + "record_type": [ + "GOSEDO-OPERATOR", + "GOSEDO-ORGANIZER", + "GOSEDO-PARTICIPANT", + ], + "external_id": [ + "GOSEDO-OPERATOR", + "GOSEDO-ORGANIZER", + "GOSEDO-PARTICIPANT", + ], + "payload__full_name": [ + "GOSEDO-OPERATOR", + "GOSEDO-ORGANIZER", + "GOSEDO-PARTICIPANT", + ], + "payload__medo_address": [ + "GOSEDO-OPERATOR", + "GOSEDO-ORGANIZER", + "GOSEDO-PARTICIPANT", + ], + "payload__registration_number": [ + "GOSEDO-OPERATOR", + "GOSEDO-ORGANIZER", + "GOSEDO-PARTICIPANT", + ], + } + for ordering, expected in expected_by_ordering.items(): + with self.subTest(ordering=ordering): + response = self.client.get(endpoint, {**params, "ordering": ordering}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["external_id"] for item in response.data["data"]], + expected, + ) + def test_flat_media_records_support_frontend_sorting_fields(self): organization = create_frontend_organization( name="Media sorting organization", diff --git a/tests/apps/parsers/test_gosedo_media_news.py b/tests/apps/parsers/test_gosedo_media_news.py index dcadc50..d8ddf66 100644 --- a/tests/apps/parsers/test_gosedo_media_news.py +++ b/tests/apps/parsers/test_gosedo_media_news.py @@ -18,6 +18,7 @@ from apps.parsers.gosedo import ( stable_gosedo_uid, ) from apps.parsers.media_news import ( + MEDIA_NEWS_MAX_BYTES, import_media_news, normalize_news_text, stable_media_external_id, @@ -619,6 +620,58 @@ class MediaNewsPermissionsTest(APITestCase): task_id, ) + def test_upload_rejects_non_xlsx_before_saving_or_queueing(self): + self.client.force_authenticate(self.admin) + with patch("apps.parsers.views._save_uploaded_parser_file") as save_file, patch( + "apps.parsers.tasks.parse_media_news.apply_async" + ) as apply_async: + response = self.client.post( + self.url, + {"file": SimpleUploadedFile("media.csv", b"headline")}, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data["errors"][0]["code"], "invalid_file_type") + save_file.assert_not_called() + apply_async.assert_not_called() + + def test_upload_rejects_oversized_xlsx_before_saving_or_queueing(self): + self.client.force_authenticate(self.admin) + with patch("apps.parsers.views.MEDIA_NEWS_MAX_BYTES", 1), patch( + "apps.parsers.views._save_uploaded_parser_file" + ) as save_file, patch( + "apps.parsers.tasks.parse_media_news.apply_async" + ) as apply_async: + response = self.client.post( + self.url, + {"file": SimpleUploadedFile("media.xlsx", b"xx")}, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data["errors"][0]["code"], "file_too_large") + save_file.assert_not_called() + apply_async.assert_not_called() + + def test_upload_accepts_xlsx_at_size_limit(self): + self.client.force_authenticate(self.admin) + with patch("apps.parsers.views.MEDIA_NEWS_MAX_BYTES", 1), patch( + "apps.parsers.views._save_uploaded_parser_file", + return_value="parser_uploads/media.xlsx", + ), patch( + "apps.parsers.tasks.parse_media_news.apply_async", + side_effect=lambda **kwargs: SimpleNamespace(id=kwargs["task_id"]), + ): + response = self.client.post( + self.url, + {"file": SimpleUploadedFile("media.xlsx", b"x")}, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(MEDIA_NEWS_MAX_BYTES, 25 * 1024 * 1024) + def test_gosedo_manual_run_is_admin_only_and_returns_task_ids(self): url = reverse( "api_v1:parsers:run-parser",