Merge pull request 'Collect sources for State Corp corporations' (#38)

src/apps/parsers/serializers.py

@@ -701,6 +701,8 @@ class ParserResultQuerySerializer(serializers.Serializer):
     batch_id = serializers.IntegerField(required=False, min_value=1)
     status = serializers.CharField(required=False, allow_blank=True)
     record_date = serializers.CharField(required=False, allow_blank=True)
+    date_from = serializers.DateField(required=False)
+    date_to = serializers.DateField(required=False)
     search = serializers.CharField(required=False, allow_blank=True)
     ordering = serializers.CharField(required=False, allow_blank=True)
     include_payload = serializers.BooleanField(required=False, default=True)
@@ -712,6 +714,14 @@ class ParserResultQuerySerializer(serializers.Serializer):
             attrs["page_size"] = attrs["limit"]
         if attrs.get("batch_id") and not attrs.get("load_batch"):
             attrs["load_batch"] = attrs["batch_id"]
+        if (
+            attrs.get("date_from")
+            and attrs.get("date_to")
+            and attrs["date_from"] > attrs["date_to"]
+        ):
+            raise serializers.ValidationError(
+                {"date_to": "date_to must be greater than or equal to date_from"}
+            )
         return attrs
 
 

src/apps/parsers/tasks.py

@@ -178,6 +178,17 @@ def _resolve_lookup_limit(
     return max(resolved, 0)
 
 
+def _rosatom_roscosmos_scope_query() -> Q:
+    """Вернуть условие принадлежности к контуру Росатома или Роскосмоса."""
+    return (
+        Q(gk_name__icontains="Росатом")
+        | Q(gk_name__icontains="Роскосмос")
+        | Q(gk_code__iexact="rosatom")
+        | Q(gk_code__iexact="roscosmos")
+        | Q(gk_code__iexact="roskosmos")
+    )
+
+
 def _active_registry_lookup_targets(
     *,
     limit: int | None = None,
@@ -186,7 +197,9 @@ def _active_registry_lookup_targets(
     organization_ids: list[str] | None = None,
 ) -> list[RegistryLookupTarget]:
     """Вернуть организации, которые сейчас состоят хотя бы в одном реестре."""
-    queryset = SourceOrganization.objects.filter(opk_registry_membership=True)
+    queryset = SourceOrganization.objects.filter(
+        Q(opk_registry_membership=True) | _rosatom_roscosmos_scope_query()
+    )
     if organization_ids is not None:
         queryset = queryset.filter(uid__in=organization_ids)
     if require_inn:
@@ -3977,6 +3990,7 @@ def _active_fns_registry_inns(*, limit: int) -> list[str]:
             Q(opk_registry_membership=True)
             | Q(goz_participation=True)
             | ~Q(ropk_num="")
+            | _rosatom_roscosmos_scope_query()
         )
         .exclude(inn="")
         .order_by("inn")

src/apps/parsers/views.py

@@ -10,6 +10,7 @@ import json
 import uuid
 from collections import defaultdict
 
+from apps.core.filters import BaseFilterSet
 from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag
 from apps.core.response import api_error_response, api_response
 from apps.core.serializers import BackgroundJobListSerializer
@@ -73,6 +74,7 @@ from django.db.models.functions import Cast, Lower
 from django.http import HttpResponse
 from django.utils.text import get_valid_filename
 from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask
+from django_filters import rest_framework as filters
 from drf_yasg import openapi
 from drf_yasg.inspectors import SwaggerAutoSchema
 from drf_yasg.utils import no_body, swagger_auto_schema
@@ -331,6 +333,20 @@ ORDERING_PARAM = openapi.Parameter(
     description="Сортировка",
     type=openapi.TYPE_STRING,
 )
+DATE_FROM_PARAM = openapi.Parameter(
+    "date_from",
+    openapi.IN_QUERY,
+    description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD)",
+    type=openapi.TYPE_STRING,
+    format=openapi.FORMAT_DATE,
+)
+DATE_TO_PARAM = openapi.Parameter(
+    "date_to",
+    openapi.IN_QUERY,
+    description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD)",
+    type=openapi.TYPE_STRING,
+    format=openapi.FORMAT_DATE,
+)
 INCLUDE_PAYLOAD_PARAM = openapi.Parameter(
     "include_payload",
     openapi.IN_QUERY,
@@ -355,6 +371,8 @@ RESULT_LIST_PARAMS = [
     OGRN_PARAM,
     LOAD_BATCH_PARAM,
     STATUS_PARAM,
+    DATE_FROM_PARAM,
+    DATE_TO_PARAM,
     SEARCH_PARAM,
     ORDERING_PARAM,
     INCLUDE_PAYLOAD_PARAM,
@@ -790,6 +808,32 @@ class IndustrialProductViewSet(ReadOnlyModelViewSet):
 # =============================================================================
 
 
+class InspectionFilter(BaseFilterSet):
+    """Filters inspections by their normalized start date."""
+
+    date_from = filters.DateFilter(
+        field_name="start_date_normalized",
+        lookup_expr="gte",
+    )
+    date_to = filters.DateFilter(
+        field_name="start_date_normalized",
+        lookup_expr="lte",
+    )
+
+    class Meta:
+        model = InspectionRecord
+        fields = [
+            "inn",
+            "ogrn",
+            "registration_number",
+            "is_federal_law_248",
+            "data_year",
+            "data_month",
+            "load_batch",
+            "registry_organization",
+        ]
+
+
 class InspectionViewSet(ReadOnlyModelViewSet):
     """
     API для просмотра проверок из Единого реестра проверок.
@@ -804,16 +848,15 @@ class InspectionViewSet(ReadOnlyModelViewSet):
     ).order_by("-created_at")
     serializer_class = InspectionSerializer
     permission_classes = [IsAuthenticated]
-    filterset_fields = [
+    filterset_class = InspectionFilter
-        "inn",
+    ordering_fields = [
-        "ogrn",
+        "id",
-        "registration_number",
+        "start_date_normalized",
-        "is_federal_law_248",
+        "end_date_normalized",
-        "data_year",
+        "created_at",
-        "data_month",
+        "updated_at",
-        "load_batch",
-        "registry_organization",
     ]
+    ordering = ["-start_date_normalized", "-created_at"]
     search_fields = [
         "organisation_name",
         "registration_number",
@@ -829,7 +872,10 @@ class InspectionViewSet(ReadOnlyModelViewSet):
         operation_description=(
             "Возвращает список проверок из Единого реестра.\n"
             "Поддерживает фильтрацию по: inn, ogrn, registration_number, "
-            "is_federal_law_248, data_year, data_month, load_batch.\n"
+            "is_federal_law_248, data_year, data_month, load_batch, "
+            "date_from, date_to.\n"
+            "Поддерживает сортировку по: start_date_normalized, "
+            "end_date_normalized, created_at, updated_at.\n"
             "Поддерживает поиск по: organisation_name, registration_number, "
             "inn, ogrn, control_authority.\n"
             f"{REGISTRY_ORGANIZATION_SEARCH_DESCRIPTION}"
@@ -2202,7 +2248,7 @@ def _native_field_map(source: str) -> dict[str, str]:
         "external_id": "registration_number",
         "organisation_name": "organisation_name",
         "title": "control_authority",
-        "record_date": "start_date",
+        "record_date": "start_date_normalized",
         "status": "status",
     }
 
@@ -2273,6 +2319,19 @@ def _apply_native_search(queryset, source: str, search: str):
     )
 
 
+def _apply_record_date_range(queryset, record_date_field: str, params: dict):
+    """Filter normalized or ISO-compatible record dates inclusively."""
+    date_from = params.get("date_from")
+    if date_from:
+        queryset = queryset.filter(
+            **{f"{record_date_field}__gte": date_from.isoformat()}
+        )
+    date_to = params.get("date_to")
+    if date_to:
+        queryset = queryset.filter(**{f"{record_date_field}__lte": date_to.isoformat()})
+    return queryset
+
+
 def _route_model_sources(descriptor) -> set[str]:
     if descriptor.source == ParserLoadLog.Source.TRUDVSEM:
         return set(VACANCY_RECORD_SOURCES)
@@ -2317,6 +2376,8 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str])
             queryset = queryset.filter(**{model_field: value})
     if params.get("record_date") and field_map.get("record_date"):
         queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]})
+    if field_map.get("record_date"):
+        queryset = _apply_record_date_range(queryset, field_map["record_date"], params)
     if params.get("search"):
         queryset = _apply_native_search(queryset, source, params["search"])
     ordering = _safe_ordering(
@@ -2356,6 +2417,7 @@ def _filter_generic_result_queryset(
             queryset = queryset.filter(**{field: value})
     if params.get("record_date"):
         queryset = queryset.filter(record_date=params["record_date"])
+    queryset = _apply_record_date_range(queryset, "record_date", params)
     if params.get("search"):
         search = params["search"]
         queryset = queryset.filter(

tests/apps/organizations/test_test_companies_commands.py

@@ -133,7 +133,7 @@ class TestCompaniesCommandsTest(TestCase):
         self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15)
 
     @override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN)
-    def test_created_source_records_are_exported_to_state_corp_package(self):
+    def test_created_source_records_are_included_in_state_corp_package(self):
         call_command("create_test_companies", stdout=StringIO())
 
         company_inns = list(

tests/apps/parsers/test_checko_collection.py

@@ -89,4 +89,29 @@ class CheckoCollectionClaimTest(TestCase):
             checko_source=CheckoCollectionAttempt.Source.INSPECTIONS,
         )
 
-        self.assertEqual([target.organization_id for target in targets], [str(second.id)])
+        self.assertEqual(
+            [target.organization_id for target in targets], [str(second.id)]
+        )
+
+    def test_lookup_targets_include_corporation_members_outside_opk(self):
+        self.organization.gk_name = 'Госкорпорация "Росатом"'
+        self.organization.save(update_fields=["gk_name"])
+        outside_scope = create_directory_organization(
+            pn_name='ООО "Вне контура обмена"',
+            mn_ogrn=1027700000399,
+            mn_inn=7701000399,
+            in_kpp=770101001,
+            mn_okpo="11223344",
+        )
+
+        targets = parser_tasks._active_registry_lookup_targets(
+            organization_ids=[
+                str(self.organization.id),
+                str(outside_scope.id),
+            ],
+        )
+
+        self.assertEqual(
+            [target.organization_id for target in targets],
+            [str(self.organization.id)],
+        )

tests/apps/parsers/test_tasks.py

@@ -1307,6 +1307,26 @@ class FNSApiSyncTaskTestCase(TestCase):
 
         self.assertEqual(targets, ["1000000001", "1000000002"])
 
+    def test_active_registry_targets_include_corporation_members_outside_opk(self):
+        OrganizationFactory(
+            inn="1000000010",
+            gk_name='Госкорпорация "Роскосмос"',
+            opk_registry_membership=False,
+            goz_participation=False,
+            ropk_num="",
+        )
+        OrganizationFactory(
+            inn="1000000011",
+            gk_name="",
+            opk_registry_membership=False,
+            goz_participation=False,
+            ropk_num="",
+        )
+
+        targets = parser_tasks._active_fns_registry_inns(limit=10)
+
+        self.assertEqual(targets, ["1000000010"])
+
 
 @override_settings(
     CELERY_TASK_ALWAYS_EAGER=True,

tests/apps/parsers/test_views.py

@@ -8,6 +8,7 @@ import io
 import os
 import tempfile
 import zipfile
+from datetime import date
 from unittest.mock import Mock, patch
 
 from apps.parsers.models import (
@@ -480,6 +481,97 @@ class ParsersViewSetTest(APITestCase):
         self.assertEqual(response.data["data"][0]["id"], hh_record.id)
         self.assertEqual(response.data["data"][0]["source"], "hh")
 
+    def test_source_results_filter_and_sort_by_record_date(self):
+        arbitration_older = GenericParserRecord.objects.create(
+            load_batch=1,
+            source=ParserLoadLog.Source.ARBITRATION,
+            external_id="arbitration-older",
+            record_date="2024-01-15",
+        )
+        arbitration_newer = GenericParserRecord.objects.create(
+            load_batch=1,
+            source=ParserLoadLog.Source.ARBITRATION,
+            external_id="arbitration-newer",
+            record_date="2024-03-15",
+        )
+        vacancy_older = GenericParserRecord.objects.create(
+            load_batch=1,
+            source="hh",
+            external_id="hh-older",
+            record_date="2024-02-01",
+            payload={"vacancy_source": "hh"},
+        )
+        vacancy_newer = GenericParserRecord.objects.create(
+            load_batch=1,
+            source="hh",
+            external_id="hh-newer",
+            record_date="2024-04-01",
+            payload={"vacancy_source": "hh"},
+        )
+        self.client.force_authenticate(self.user)
+
+        arbitration_response = self.client.get(
+            "/api/v1/arbitration/cases/",
+            {"date_from": "2024-02-01", "ordering": "-record_date"},
+        )
+        vacancy_response = self.client.get(
+            "/api/v1/trudvsem/vacancies/",
+            {"date_to": "2024-02-15", "ordering": "record_date"},
+        )
+
+        self.assertEqual(arbitration_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [item["id"] for item in arbitration_response.data["data"]],
+            [arbitration_newer.id],
+        )
+        self.assertNotIn(
+            arbitration_older.id,
+            [item["id"] for item in arbitration_response.data["data"]],
+        )
+        self.assertEqual(vacancy_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [item["id"] for item in vacancy_response.data["data"]],
+            [vacancy_older.id],
+        )
+        self.assertNotIn(
+            vacancy_newer.id,
+            [item["id"] for item in vacancy_response.data["data"]],
+        )
+
+    def test_inspection_source_results_filter_and_sort_by_record_date(self):
+        older = InspectionRecordFactory(
+            start_date="2024-01-10",
+            start_date_normalized=date(2024, 1, 10),
+        )
+        newer = InspectionRecordFactory(
+            start_date="2024-03-10",
+            start_date_normalized=date(2024, 3, 10),
+        )
+        self.client.force_authenticate(self.user)
+
+        response = self.client.get(
+            "/api/v1/proverki/",
+            {"date_from": "2024-02-01", "ordering": "-start_date_normalized"},
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [item["id"] for item in response.data["data"]],
+            [newer.id],
+        )
+        self.assertNotIn(older.id, [item["id"] for item in response.data["data"]])
+
+        source_response = self.client.get(
+            "/api/v1/parsers/results/inspections/",
+            {"date_from": "2024-02-01", "ordering": "-record_date"},
+        )
+
+        self.assertEqual(source_response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [item["id"] for item in source_response.data["data"]],
+            [newer.id],
+        )
+
     def test_parser_results_v1_enrich_missing_organization_fields_without_contract_change(
         self,
     ):