From 2667646d0275ab459d58529f41f4b524dd1ee6a9 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 21 Jul 2026 18:18:56 +0200 Subject: [PATCH] feat(parsers): filter source results by date --- src/apps/parsers/serializers.py | 10 ++++ src/apps/parsers/views.py | 88 ++++++++++++++++++++++++++---- tests/apps/parsers/test_views.py | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 11 deletions(-) diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index f28e450..430a173 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -701,6 +701,8 @@ class ParserResultQuerySerializer(serializers.Serializer): batch_id = serializers.IntegerField(required=False, min_value=1) status = serializers.CharField(required=False, allow_blank=True) record_date = serializers.CharField(required=False, allow_blank=True) + date_from = serializers.DateField(required=False) + date_to = serializers.DateField(required=False) search = serializers.CharField(required=False, allow_blank=True) ordering = serializers.CharField(required=False, allow_blank=True) include_payload = serializers.BooleanField(required=False, default=True) @@ -712,6 +714,14 @@ class ParserResultQuerySerializer(serializers.Serializer): attrs["page_size"] = attrs["limit"] if attrs.get("batch_id") and not attrs.get("load_batch"): attrs["load_batch"] = attrs["batch_id"] + if ( + attrs.get("date_from") + and attrs.get("date_to") + and attrs["date_from"] > attrs["date_to"] + ): + raise serializers.ValidationError( + {"date_to": "date_to must be greater than or equal to date_from"} + ) return attrs diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 0574db9..1de9419 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -10,6 +10,7 @@ import json import uuid from collections import defaultdict +from apps.core.filters import BaseFilterSet from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from apps.core.response import api_error_response, api_response from apps.core.serializers import BackgroundJobListSerializer @@ -73,6 +74,7 @@ from django.db.models.functions import Cast, Lower from django.http import HttpResponse from django.utils.text import get_valid_filename from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask +from django_filters import rest_framework as filters from drf_yasg import openapi from drf_yasg.inspectors import SwaggerAutoSchema from drf_yasg.utils import no_body, swagger_auto_schema @@ -331,6 +333,20 @@ ORDERING_PARAM = openapi.Parameter( description="Сортировка", type=openapi.TYPE_STRING, ) +DATE_FROM_PARAM = openapi.Parameter( + "date_from", + openapi.IN_QUERY, + description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD)", + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, +) +DATE_TO_PARAM = openapi.Parameter( + "date_to", + openapi.IN_QUERY, + description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD)", + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, +) INCLUDE_PAYLOAD_PARAM = openapi.Parameter( "include_payload", openapi.IN_QUERY, @@ -355,6 +371,8 @@ RESULT_LIST_PARAMS = [ OGRN_PARAM, LOAD_BATCH_PARAM, STATUS_PARAM, + DATE_FROM_PARAM, + DATE_TO_PARAM, SEARCH_PARAM, ORDERING_PARAM, INCLUDE_PAYLOAD_PARAM, @@ -790,6 +808,32 @@ class IndustrialProductViewSet(ReadOnlyModelViewSet): # ============================================================================= +class InspectionFilter(BaseFilterSet): + """Filters inspections by their normalized start date.""" + + date_from = filters.DateFilter( + field_name="start_date_normalized", + lookup_expr="gte", + ) + date_to = filters.DateFilter( + field_name="start_date_normalized", + lookup_expr="lte", + ) + + class Meta: + model = InspectionRecord + fields = [ + "inn", + "ogrn", + "registration_number", + "is_federal_law_248", + "data_year", + "data_month", + "load_batch", + "registry_organization", + ] + + class InspectionViewSet(ReadOnlyModelViewSet): """ API для просмотра проверок из Единого реестра проверок. @@ -804,16 +848,15 @@ class InspectionViewSet(ReadOnlyModelViewSet): ).order_by("-created_at") serializer_class = InspectionSerializer permission_classes = [IsAuthenticated] - filterset_fields = [ - "inn", - "ogrn", - "registration_number", - "is_federal_law_248", - "data_year", - "data_month", - "load_batch", - "registry_organization", + filterset_class = InspectionFilter + ordering_fields = [ + "id", + "start_date_normalized", + "end_date_normalized", + "created_at", + "updated_at", ] + ordering = ["-start_date_normalized", "-created_at"] search_fields = [ "organisation_name", "registration_number", @@ -829,7 +872,10 @@ class InspectionViewSet(ReadOnlyModelViewSet): operation_description=( "Возвращает список проверок из Единого реестра.\n" "Поддерживает фильтрацию по: inn, ogrn, registration_number, " - "is_federal_law_248, data_year, data_month, load_batch.\n" + "is_federal_law_248, data_year, data_month, load_batch, " + "date_from, date_to.\n" + "Поддерживает сортировку по: start_date_normalized, " + "end_date_normalized, created_at, updated_at.\n" "Поддерживает поиск по: organisation_name, registration_number, " "inn, ogrn, control_authority.\n" f"{REGISTRY_ORGANIZATION_SEARCH_DESCRIPTION}" @@ -2202,7 +2248,7 @@ def _native_field_map(source: str) -> dict[str, str]: "external_id": "registration_number", "organisation_name": "organisation_name", "title": "control_authority", - "record_date": "start_date", + "record_date": "start_date_normalized", "status": "status", } @@ -2273,6 +2319,21 @@ def _apply_native_search(queryset, source: str, search: str): ) +def _apply_record_date_range(queryset, record_date_field: str, params: dict): + """Filter normalized or ISO-compatible record dates inclusively.""" + date_from = params.get("date_from") + if date_from: + queryset = queryset.filter( + **{f"{record_date_field}__gte": date_from.isoformat()} + ) + date_to = params.get("date_to") + if date_to: + queryset = queryset.filter( + **{f"{record_date_field}__lte": date_to.isoformat()} + ) + return queryset + + def _route_model_sources(descriptor) -> set[str]: if descriptor.source == ParserLoadLog.Source.TRUDVSEM: return set(VACANCY_RECORD_SOURCES) @@ -2317,6 +2378,10 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str]) queryset = queryset.filter(**{model_field: value}) if params.get("record_date") and field_map.get("record_date"): queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]}) + if field_map.get("record_date"): + queryset = _apply_record_date_range( + queryset, field_map["record_date"], params + ) if params.get("search"): queryset = _apply_native_search(queryset, source, params["search"]) ordering = _safe_ordering( @@ -2356,6 +2421,7 @@ def _filter_generic_result_queryset( queryset = queryset.filter(**{field: value}) if params.get("record_date"): queryset = queryset.filter(record_date=params["record_date"]) + queryset = _apply_record_date_range(queryset, "record_date", params) if params.get("search"): search = params["search"] queryset = queryset.filter( diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 6b8c53e..5d035f4 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -8,6 +8,7 @@ import io import os import tempfile import zipfile +from datetime import date from unittest.mock import Mock, patch from apps.parsers.models import ( @@ -480,6 +481,97 @@ class ParsersViewSetTest(APITestCase): self.assertEqual(response.data["data"][0]["id"], hh_record.id) self.assertEqual(response.data["data"][0]["source"], "hh") + def test_source_results_filter_and_sort_by_record_date(self): + arbitration_older = GenericParserRecord.objects.create( + load_batch=1, + source=ParserLoadLog.Source.ARBITRATION, + external_id="arbitration-older", + record_date="2024-01-15", + ) + arbitration_newer = GenericParserRecord.objects.create( + load_batch=1, + source=ParserLoadLog.Source.ARBITRATION, + external_id="arbitration-newer", + record_date="2024-03-15", + ) + vacancy_older = GenericParserRecord.objects.create( + load_batch=1, + source="hh", + external_id="hh-older", + record_date="2024-02-01", + payload={"vacancy_source": "hh"}, + ) + vacancy_newer = GenericParserRecord.objects.create( + load_batch=1, + source="hh", + external_id="hh-newer", + record_date="2024-04-01", + payload={"vacancy_source": "hh"}, + ) + self.client.force_authenticate(self.user) + + arbitration_response = self.client.get( + "/api/v1/arbitration/cases/", + {"date_from": "2024-02-01", "ordering": "-record_date"}, + ) + vacancy_response = self.client.get( + "/api/v1/trudvsem/vacancies/", + {"date_to": "2024-02-15", "ordering": "record_date"}, + ) + + self.assertEqual(arbitration_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in arbitration_response.data["data"]], + [arbitration_newer.id], + ) + self.assertNotIn( + arbitration_older.id, + [item["id"] for item in arbitration_response.data["data"]], + ) + self.assertEqual(vacancy_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in vacancy_response.data["data"]], + [vacancy_older.id], + ) + self.assertNotIn( + vacancy_newer.id, + [item["id"] for item in vacancy_response.data["data"]], + ) + + def test_inspection_source_results_filter_and_sort_by_record_date(self): + older = InspectionRecordFactory( + start_date="2024-01-10", + start_date_normalized=date(2024, 1, 10), + ) + newer = InspectionRecordFactory( + start_date="2024-03-10", + start_date_normalized=date(2024, 3, 10), + ) + self.client.force_authenticate(self.user) + + response = self.client.get( + "/api/v1/proverki/", + {"date_from": "2024-02-01", "ordering": "-start_date_normalized"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in response.data["data"]], + [newer.id], + ) + self.assertNotIn(older.id, [item["id"] for item in response.data["data"]]) + + source_response = self.client.get( + "/api/v1/parsers/results/inspections/", + {"date_from": "2024-02-01", "ordering": "-record_date"}, + ) + + self.assertEqual(source_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in source_response.data["data"]], + [newer.id], + ) + def test_parser_results_v1_enrich_missing_organization_fields_without_contract_change( self, ):