From 2667646d0275ab459d58529f41f4b524dd1ee6a9 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 21 Jul 2026 18:18:56 +0200 Subject: [PATCH 1/5] feat(parsers): filter source results by date --- src/apps/parsers/serializers.py | 10 ++++ src/apps/parsers/views.py | 88 ++++++++++++++++++++++++++---- tests/apps/parsers/test_views.py | 92 ++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 11 deletions(-) diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index f28e450..430a173 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -701,6 +701,8 @@ class ParserResultQuerySerializer(serializers.Serializer): batch_id = serializers.IntegerField(required=False, min_value=1) status = serializers.CharField(required=False, allow_blank=True) record_date = serializers.CharField(required=False, allow_blank=True) + date_from = serializers.DateField(required=False) + date_to = serializers.DateField(required=False) search = serializers.CharField(required=False, allow_blank=True) ordering = serializers.CharField(required=False, allow_blank=True) include_payload = serializers.BooleanField(required=False, default=True) @@ -712,6 +714,14 @@ class ParserResultQuerySerializer(serializers.Serializer): attrs["page_size"] = attrs["limit"] if attrs.get("batch_id") and not attrs.get("load_batch"): attrs["load_batch"] = attrs["batch_id"] + if ( + attrs.get("date_from") + and attrs.get("date_to") + and attrs["date_from"] > attrs["date_to"] + ): + raise serializers.ValidationError( + {"date_to": "date_to must be greater than or equal to date_from"} + ) return attrs diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 0574db9..1de9419 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -10,6 +10,7 @@ import json import uuid from collections import defaultdict +from apps.core.filters import BaseFilterSet from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from apps.core.response import api_error_response, api_response from apps.core.serializers import BackgroundJobListSerializer @@ -73,6 +74,7 @@ from django.db.models.functions import Cast, Lower from django.http import HttpResponse from django.utils.text import get_valid_filename from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask +from django_filters import rest_framework as filters from drf_yasg import openapi from drf_yasg.inspectors import SwaggerAutoSchema from drf_yasg.utils import no_body, swagger_auto_schema @@ -331,6 +333,20 @@ ORDERING_PARAM = openapi.Parameter( description="Сортировка", type=openapi.TYPE_STRING, ) +DATE_FROM_PARAM = openapi.Parameter( + "date_from", + openapi.IN_QUERY, + description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD)", + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, +) +DATE_TO_PARAM = openapi.Parameter( + "date_to", + openapi.IN_QUERY, + description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD)", + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, +) INCLUDE_PAYLOAD_PARAM = openapi.Parameter( "include_payload", openapi.IN_QUERY, @@ -355,6 +371,8 @@ RESULT_LIST_PARAMS = [ OGRN_PARAM, LOAD_BATCH_PARAM, STATUS_PARAM, + DATE_FROM_PARAM, + DATE_TO_PARAM, SEARCH_PARAM, ORDERING_PARAM, INCLUDE_PAYLOAD_PARAM, @@ -790,6 +808,32 @@ class IndustrialProductViewSet(ReadOnlyModelViewSet): # ============================================================================= +class InspectionFilter(BaseFilterSet): + """Filters inspections by their normalized start date.""" + + date_from = filters.DateFilter( + field_name="start_date_normalized", + lookup_expr="gte", + ) + date_to = filters.DateFilter( + field_name="start_date_normalized", + lookup_expr="lte", + ) + + class Meta: + model = InspectionRecord + fields = [ + "inn", + "ogrn", + "registration_number", + "is_federal_law_248", + "data_year", + "data_month", + "load_batch", + "registry_organization", + ] + + class InspectionViewSet(ReadOnlyModelViewSet): """ API для просмотра проверок из Единого реестра проверок. @@ -804,16 +848,15 @@ class InspectionViewSet(ReadOnlyModelViewSet): ).order_by("-created_at") serializer_class = InspectionSerializer permission_classes = [IsAuthenticated] - filterset_fields = [ - "inn", - "ogrn", - "registration_number", - "is_federal_law_248", - "data_year", - "data_month", - "load_batch", - "registry_organization", + filterset_class = InspectionFilter + ordering_fields = [ + "id", + "start_date_normalized", + "end_date_normalized", + "created_at", + "updated_at", ] + ordering = ["-start_date_normalized", "-created_at"] search_fields = [ "organisation_name", "registration_number", @@ -829,7 +872,10 @@ class InspectionViewSet(ReadOnlyModelViewSet): operation_description=( "Возвращает список проверок из Единого реестра.\n" "Поддерживает фильтрацию по: inn, ogrn, registration_number, " - "is_federal_law_248, data_year, data_month, load_batch.\n" + "is_federal_law_248, data_year, data_month, load_batch, " + "date_from, date_to.\n" + "Поддерживает сортировку по: start_date_normalized, " + "end_date_normalized, created_at, updated_at.\n" "Поддерживает поиск по: organisation_name, registration_number, " "inn, ogrn, control_authority.\n" f"{REGISTRY_ORGANIZATION_SEARCH_DESCRIPTION}" @@ -2202,7 +2248,7 @@ def _native_field_map(source: str) -> dict[str, str]: "external_id": "registration_number", "organisation_name": "organisation_name", "title": "control_authority", - "record_date": "start_date", + "record_date": "start_date_normalized", "status": "status", } @@ -2273,6 +2319,21 @@ def _apply_native_search(queryset, source: str, search: str): ) +def _apply_record_date_range(queryset, record_date_field: str, params: dict): + """Filter normalized or ISO-compatible record dates inclusively.""" + date_from = params.get("date_from") + if date_from: + queryset = queryset.filter( + **{f"{record_date_field}__gte": date_from.isoformat()} + ) + date_to = params.get("date_to") + if date_to: + queryset = queryset.filter( + **{f"{record_date_field}__lte": date_to.isoformat()} + ) + return queryset + + def _route_model_sources(descriptor) -> set[str]: if descriptor.source == ParserLoadLog.Source.TRUDVSEM: return set(VACANCY_RECORD_SOURCES) @@ -2317,6 +2378,10 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str]) queryset = queryset.filter(**{model_field: value}) if params.get("record_date") and field_map.get("record_date"): queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]}) + if field_map.get("record_date"): + queryset = _apply_record_date_range( + queryset, field_map["record_date"], params + ) if params.get("search"): queryset = _apply_native_search(queryset, source, params["search"]) ordering = _safe_ordering( @@ -2356,6 +2421,7 @@ def _filter_generic_result_queryset( queryset = queryset.filter(**{field: value}) if params.get("record_date"): queryset = queryset.filter(record_date=params["record_date"]) + queryset = _apply_record_date_range(queryset, "record_date", params) if params.get("search"): search = params["search"] queryset = queryset.filter( diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 6b8c53e..5d035f4 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -8,6 +8,7 @@ import io import os import tempfile import zipfile +from datetime import date from unittest.mock import Mock, patch from apps.parsers.models import ( @@ -480,6 +481,97 @@ class ParsersViewSetTest(APITestCase): self.assertEqual(response.data["data"][0]["id"], hh_record.id) self.assertEqual(response.data["data"][0]["source"], "hh") + def test_source_results_filter_and_sort_by_record_date(self): + arbitration_older = GenericParserRecord.objects.create( + load_batch=1, + source=ParserLoadLog.Source.ARBITRATION, + external_id="arbitration-older", + record_date="2024-01-15", + ) + arbitration_newer = GenericParserRecord.objects.create( + load_batch=1, + source=ParserLoadLog.Source.ARBITRATION, + external_id="arbitration-newer", + record_date="2024-03-15", + ) + vacancy_older = GenericParserRecord.objects.create( + load_batch=1, + source="hh", + external_id="hh-older", + record_date="2024-02-01", + payload={"vacancy_source": "hh"}, + ) + vacancy_newer = GenericParserRecord.objects.create( + load_batch=1, + source="hh", + external_id="hh-newer", + record_date="2024-04-01", + payload={"vacancy_source": "hh"}, + ) + self.client.force_authenticate(self.user) + + arbitration_response = self.client.get( + "/api/v1/arbitration/cases/", + {"date_from": "2024-02-01", "ordering": "-record_date"}, + ) + vacancy_response = self.client.get( + "/api/v1/trudvsem/vacancies/", + {"date_to": "2024-02-15", "ordering": "record_date"}, + ) + + self.assertEqual(arbitration_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in arbitration_response.data["data"]], + [arbitration_newer.id], + ) + self.assertNotIn( + arbitration_older.id, + [item["id"] for item in arbitration_response.data["data"]], + ) + self.assertEqual(vacancy_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in vacancy_response.data["data"]], + [vacancy_older.id], + ) + self.assertNotIn( + vacancy_newer.id, + [item["id"] for item in vacancy_response.data["data"]], + ) + + def test_inspection_source_results_filter_and_sort_by_record_date(self): + older = InspectionRecordFactory( + start_date="2024-01-10", + start_date_normalized=date(2024, 1, 10), + ) + newer = InspectionRecordFactory( + start_date="2024-03-10", + start_date_normalized=date(2024, 3, 10), + ) + self.client.force_authenticate(self.user) + + response = self.client.get( + "/api/v1/proverki/", + {"date_from": "2024-02-01", "ordering": "-start_date_normalized"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in response.data["data"]], + [newer.id], + ) + self.assertNotIn(older.id, [item["id"] for item in response.data["data"]]) + + source_response = self.client.get( + "/api/v1/parsers/results/inspections/", + {"date_from": "2024-02-01", "ordering": "-record_date"}, + ) + + self.assertEqual(source_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in source_response.data["data"]], + [newer.id], + ) + def test_parser_results_v1_enrich_missing_organization_fields_without_contract_change( self, ): -- 2.39.5 From b4c67a6188aff507da3d7239f5bf2299ae7db3b4 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 21 Jul 2026 18:23:07 +0200 Subject: [PATCH 2/5] style(parsers): format date result filters --- src/apps/parsers/views.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 1de9419..e5ab40f 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -2328,9 +2328,7 @@ def _apply_record_date_range(queryset, record_date_field: str, params: dict): ) date_to = params.get("date_to") if date_to: - queryset = queryset.filter( - **{f"{record_date_field}__lte": date_to.isoformat()} - ) + queryset = queryset.filter(**{f"{record_date_field}__lte": date_to.isoformat()}) return queryset @@ -2379,9 +2377,7 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str]) if params.get("record_date") and field_map.get("record_date"): queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]}) if field_map.get("record_date"): - queryset = _apply_record_date_range( - queryset, field_map["record_date"], params - ) + queryset = _apply_record_date_range(queryset, field_map["record_date"], params) if params.get("search"): queryset = _apply_native_search(queryset, source, params["search"]) ordering = _safe_ordering( -- 2.39.5 From 991b08263f490ba9ced0258cb216600a4cf780fc Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 14:23:04 +0200 Subject: [PATCH 3/5] fix: exclude demo companies from state corp exchange --- src/apps/exchange/state_corp_services.py | 5 +++- .../apps/exchange/test_state_corp_services.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index a6ecf75..5d5125d 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -35,6 +35,7 @@ from django.conf import settings from django.db.models import Q from django.utils import timezone from organizations.models import Organization, OrganizationSourceRecord +from organizations.test_companies import TestCompanyDatasetService class StateCorpExchangeError(ValueError): @@ -289,7 +290,9 @@ class StateCorpExchangeService: @classmethod def _rosatom_roscosmos_queryset(cls): - queryset = Organization.objects.exclude(inn="") + queryset = Organization.objects.exclude(inn="").exclude( + uid__in=TestCompanyDatasetService.company_uids() + ) name_query = Q() for keyword in cls.ROSATOM_ROSCOSMOS_GK_NAME_KEYWORDS: name_query |= Q(gk_name__icontains=keyword) diff --git a/tests/apps/exchange/test_state_corp_services.py b/tests/apps/exchange/test_state_corp_services.py index 3720266..36a3ad5 100644 --- a/tests/apps/exchange/test_state_corp_services.py +++ b/tests/apps/exchange/test_state_corp_services.py @@ -23,6 +23,7 @@ from apps.parsers.models import ( from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.test import TestCase, override_settings from organizations.models import Organization +from organizations.test_companies import TestCompanyDatasetService from tests.apps.parsers.factories import ( IndustrialCertificateRecordFactory, @@ -67,6 +68,30 @@ TEST_STATE_CORP_TOKEN = "state-corp-test-exchange-token" # noqa: S105 class StateCorpExchangeServiceTest(TestCase): """Verify package compatibility with state-corp receiver contract.""" + def test_default_package_excludes_deterministic_test_companies(self): + real_organization = Organization.objects.create( + rn=1001, + name="Реальная организация", + inn="7707083893", + gk_name="Росатом", + ) + Organization.objects.create( + uid=TestCompanyDatasetService.company_uids()[0], + rn=9001, + name="Тестовая компания 1", + inn="7700000001", + gk_name="Росатом", + ) + + package = StateCorpExchangeService.build_package() + payload = _decode_package_payload(package) + + self.assertEqual(package.payload_counts["organizations"], 1) + self.assertEqual( + [row["mostovik_uid"] for row in payload["data"]["organizations"]], + [str(real_organization.uid)], + ) + def test_build_package_contains_expected_payload(self): organization = Organization.objects.create( rn=1001, -- 2.39.5 From 08c3d8465262a0462181343771baa75390aa7af4 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 14:33:08 +0200 Subject: [PATCH 4/5] test: align demo company exchange expectations --- .../test_test_companies_commands.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py index 6848af0..c4741d2 100644 --- a/tests/apps/organizations/test_test_companies_commands.py +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -133,7 +133,7 @@ class TestCompaniesCommandsTest(TestCase): self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15) @override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN) - def test_created_source_records_are_exported_to_state_corp_package(self): + def test_created_source_records_are_excluded_from_state_corp_package(self): call_command("create_test_companies", stdout=StringIO()) company_inns = list( @@ -146,18 +146,18 @@ class TestCompaniesCommandsTest(TestCase): self.assertEqual( package.payload_counts, { - "organizations": 20, - "industrial_certificates": 20, - "manufacturers": 20, - "industrial_products": 20, - "prosecutor_checks": 20, - "public_procurements": 60, - "financial_reports": 20, - "arbitration_cases": 20, - "bankruptcy_procedures": 20, - "defense_unreliable_suppliers": 40, - "information_security_registries": 20, - "labor_vacancies": 20, + "organizations": 0, + "industrial_certificates": 0, + "manufacturers": 0, + "industrial_products": 0, + "prosecutor_checks": 0, + "public_procurements": 0, + "financial_reports": 0, + "arbitration_cases": 0, + "bankruptcy_procedures": 0, + "defense_unreliable_suppliers": 0, + "information_security_registries": 0, + "labor_vacancies": 0, }, ) -- 2.39.5 From 2644822de540be0c0e74fb6ac08cbe824fbcffde Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Fri, 24 Jul 2026 10:04:54 +0200 Subject: [PATCH 5/5] fix: collect sources for state corp corporations --- src/apps/parsers/tasks.py | 16 +++++++++++- tests/apps/parsers/test_checko_collection.py | 27 +++++++++++++++++++- tests/apps/parsers/test_tasks.py | 20 +++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index de9ee60..2ad171a 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -178,6 +178,17 @@ def _resolve_lookup_limit( return max(resolved, 0) +def _rosatom_roscosmos_scope_query() -> Q: + """Вернуть условие принадлежности к контуру Росатома или Роскосмоса.""" + return ( + Q(gk_name__icontains="Росатом") + | Q(gk_name__icontains="Роскосмос") + | Q(gk_code__iexact="rosatom") + | Q(gk_code__iexact="roscosmos") + | Q(gk_code__iexact="roskosmos") + ) + + def _active_registry_lookup_targets( *, limit: int | None = None, @@ -186,7 +197,9 @@ def _active_registry_lookup_targets( organization_ids: list[str] | None = None, ) -> list[RegistryLookupTarget]: """Вернуть организации, которые сейчас состоят хотя бы в одном реестре.""" - queryset = SourceOrganization.objects.filter(opk_registry_membership=True) + queryset = SourceOrganization.objects.filter( + Q(opk_registry_membership=True) | _rosatom_roscosmos_scope_query() + ) if organization_ids is not None: queryset = queryset.filter(uid__in=organization_ids) if require_inn: @@ -3977,6 +3990,7 @@ def _active_fns_registry_inns(*, limit: int) -> list[str]: Q(opk_registry_membership=True) | Q(goz_participation=True) | ~Q(ropk_num="") + | _rosatom_roscosmos_scope_query() ) .exclude(inn="") .order_by("inn") diff --git a/tests/apps/parsers/test_checko_collection.py b/tests/apps/parsers/test_checko_collection.py index b6b55e7..5f4931d 100644 --- a/tests/apps/parsers/test_checko_collection.py +++ b/tests/apps/parsers/test_checko_collection.py @@ -89,4 +89,29 @@ class CheckoCollectionClaimTest(TestCase): checko_source=CheckoCollectionAttempt.Source.INSPECTIONS, ) - self.assertEqual([target.organization_id for target in targets], [str(second.id)]) + self.assertEqual( + [target.organization_id for target in targets], [str(second.id)] + ) + + def test_lookup_targets_include_corporation_members_outside_opk(self): + self.organization.gk_name = 'Госкорпорация "Росатом"' + self.organization.save(update_fields=["gk_name"]) + outside_scope = create_directory_organization( + pn_name='ООО "Вне контура обмена"', + mn_ogrn=1027700000399, + mn_inn=7701000399, + in_kpp=770101001, + mn_okpo="11223344", + ) + + targets = parser_tasks._active_registry_lookup_targets( + organization_ids=[ + str(self.organization.id), + str(outside_scope.id), + ], + ) + + self.assertEqual( + [target.organization_id for target in targets], + [str(self.organization.id)], + ) diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index dcccb57..1a4e52f 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -1307,6 +1307,26 @@ class FNSApiSyncTaskTestCase(TestCase): self.assertEqual(targets, ["1000000001", "1000000002"]) + def test_active_registry_targets_include_corporation_members_outside_opk(self): + OrganizationFactory( + inn="1000000010", + gk_name='Госкорпорация "Роскосмос"', + opk_registry_membership=False, + goz_participation=False, + ropk_num="", + ) + OrganizationFactory( + inn="1000000011", + gk_name="", + opk_registry_membership=False, + goz_participation=False, + ropk_num="", + ) + + targets = parser_tasks._active_fns_registry_inns(limit=10) + + self.assertEqual(targets, ["1000000010"]) + @override_settings( CELERY_TASK_ALWAYS_EAGER=True, -- 2.39.5