Merge remote-tracking branch 'origin/main' into dev

# Conflicts:
#	tests/apps/exchange/test_state_corp_services.py

src/apps/exchange/state_corp_services.py

@@ -35,7 +35,6 @@ from django.conf import settings
 from django.db.models import Q
 from django.utils import timezone
 from organizations.models import Organization, OrganizationSourceRecord
-from organizations.test_companies import TestCompanyDatasetService
 
 
 class StateCorpExchangeError(ValueError):
@@ -290,9 +289,7 @@ class StateCorpExchangeService:
 
     @classmethod
     def _rosatom_roscosmos_queryset(cls):
-        queryset = Organization.objects.exclude(inn="").exclude(
-            uid__in=TestCompanyDatasetService.company_uids()
-        )
+        queryset = Organization.objects.exclude(inn="")
         name_query = Q()
         for keyword in cls.ROSATOM_ROSCOSMOS_GK_NAME_KEYWORDS:
             name_query |= Q(gk_name__icontains=keyword)

src/apps/parsers/services.py

@@ -893,6 +893,11 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR
 
         records_by_source: dict[str, list[SourceRecordInput]] = defaultdict(list)
         for (record_source, _external_id), record in unique_records.items():
+            record_date = record.record_date
+            if record_source in cls.vacancy_record_sources | {
+                ParserLoadLog.Source.ARBITRATION
+            }:
+                record_date = _date_to_iso(normalize_to_date(record.record_date)) or ""
             payload = dict(record.payload) if isinstance(record.payload, dict) else {}
             payload.update(
                 {
@@ -903,7 +908,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR
                     "ogrn": record.ogrn,
                     "organisation_name": record.organisation_name,
                     "title": record.title,
-                    "record_date": record.record_date,
+                    "record_date": record_date,
                     "amount": str(record.amount) if record.amount is not None else None,
                     "status": record.status,
                     "url": record.url,
@@ -916,7 +921,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR
                     organization_name=record.organisation_name,
                     inn=record.inn,
                     ogrn=record.ogrn,
-                    record_date=record.record_date,
+                    record_date=record_date,
                     amount=record.amount,
                     status=record.status,
                     url=record.url,
@@ -1503,7 +1508,7 @@ class InspectionService(BulkOperationsMixin, BaseService[InspectionRecord]):
                 organization_name=insp.organisation_name,
                 inn=insp.inn,
                 ogrn=insp.ogrn,
-                record_date=insp.start_date,
+                record_date=_date_to_iso(normalize_to_date(insp.start_date)) or "",
                 status=insp.status,
                 payload={
                     "load_batch": batch_id,

src/organizations/serializers.py

@@ -65,6 +65,7 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
     source_group = serializers.CharField(
         source="extension.source_group", read_only=True
     )
+    record_date = serializers.SerializerMethodField()
 
     class Meta:
         model = OrganizationSourceRecord
@@ -91,6 +92,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
         ]
         read_only_fields = fields
 
+    @swagger_serializer_method(
+        serializer_or_field=serializers.DateField(allow_null=True, read_only=True),
+    )
+    def get_record_date(self, obj) -> str | None:
+        return getattr(obj, "canonical_record_date", obj.record_date) or None
+
     @swagger_serializer_method(
         serializer_or_field=OrganizationSourceRecordOrganizationSerializer,
     )

src/organizations/views.py

@@ -6,14 +6,16 @@ import hashlib
 import json
 import os
 from contextlib import suppress
+from datetime import datetime
 from tempfile import NamedTemporaryFile
 from typing import Any
 
 from apps.core.openapi import swagger_tag
 from django.conf import settings
 from django.core.cache import cache
-from django.db.models import CharField, Q
-from django.db.models.functions import Cast
+from django.db.models import Case, CharField, F, Q, Value, When
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Cast, Coalesce, NullIf
 from django.http import HttpResponse
 from django_filters import rest_framework as filters
 from drf_yasg import openapi
@@ -81,6 +83,22 @@ def _query_parameter(
 
 SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup]
 
+SOURCE_RECORD_ORDERING_FIELDS = (
+    "record_date",
+    "extension__organization__name",
+    "created_at",
+    "updated_at",
+    "title",
+    "uid",
+    "extension__organization__inn",
+    "extension__organization__ogrn",
+)
+SOURCE_RECORD_ORDERING_VALUES = [
+    value
+    for field_name in SOURCE_RECORD_ORDERING_FIELDS
+    for value in (field_name, f"-{field_name}")
+]
+
 ORGANIZATION_LIST_PARAMS = [
     _query_parameter(
         "page",
@@ -178,6 +196,26 @@ SOURCE_RECORD_LIST_PARAMS = [
             "статусу, датам, URL и исходным данным записи."
         ),
     ),
+    _query_parameter(
+        "date_from",
+        description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD).",
+        format_=openapi.FORMAT_DATE,
+    ),
+    _query_parameter(
+        "date_to",
+        description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD).",
+        format_=openapi.FORMAT_DATE,
+    ),
+    _query_parameter(
+        "ordering",
+        description=(
+            "Сортировка по полям: record_date, extension__organization__name, "
+            "created_at, updated_at, title, uid, extension__organization__inn, "
+            "extension__organization__ogrn. Для обратной сортировки используйте "
+            "префикс -. Значения record_date с null сортируются последними."
+        ),
+        enum=SOURCE_RECORD_ORDERING_VALUES,
+    ),
     _query_parameter(
         "page", description="Номер страницы.", param_type=openapi.TYPE_INTEGER
     ),
@@ -472,7 +510,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
     serializer_class = OrganizationSourceRecordSerializer
     permission_classes = [IsAuthenticated]
     lookup_field = "uid"
-    filter_backends = [OrderingFilter]
+    filter_backends = []
     search_fields = [
         "title",
         "external_id",
@@ -494,16 +532,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         "extension__organization__ogrn",
         "extension__organization__ogrip",
     ]
-    ordering_fields = [
-        "created_at",
-        "updated_at",
-        "record_date",
-        "title",
-        "uid",
-        "extension__organization__name",
-        "extension__organization__inn",
-        "extension__organization__ogrn",
-    ]
+    ordering_fields = SOURCE_RECORD_ORDERING_FIELDS
     ordering = ["-created_at", "-uid"]
 
     def get_permissions(self):
@@ -514,7 +543,28 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         return super().get_permissions()
 
     def get_queryset(self):
-        queryset = super().get_queryset()
+        raw_record_date = NullIf(F("record_date"), Value(""))
+        inspection_record_date = Coalesce(
+            NullIf(
+                KeyTextTransform("start_date_normalized", "payload"),
+                Value(""),
+            ),
+            raw_record_date,
+        )
+        queryset = (
+            super()
+            .get_queryset()
+            .annotate(
+                canonical_record_date=Case(
+                    When(
+                        extension__source_group=SourceGroup.PLANNED_INSPECTIONS,
+                        then=inspection_record_date,
+                    ),
+                    default=raw_record_date,
+                    output_field=CharField(),
+                )
+            )
+        )
         params = self.request.query_params
         source_group = params.get("source_group")
         source = params.get("source")
@@ -533,8 +583,80 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         if search_terms:
             queryset = self._filter_search_queryset(queryset, search_terms)
 
+        date_from = getattr(self, "_source_record_date_from", None)
+        date_to = getattr(self, "_source_record_date_to", None)
+        if date_from:
+            queryset = queryset.filter(canonical_record_date__gte=date_from)
+        if date_to:
+            queryset = queryset.filter(canonical_record_date__lte=date_to)
+
+        ordering = getattr(self, "_source_record_ordering", None)
+        if ordering:
+            return self._order_source_records(queryset, ordering)
+
         return queryset
 
+    @staticmethod
+    def _order_source_records(queryset, ordering: str):
+        if ordering.lstrip("-") == "record_date":
+            expression = F("canonical_record_date")
+            if ordering.startswith("-"):
+                expression = expression.desc(nulls_last=True)
+            else:
+                expression = expression.asc(nulls_last=True)
+            return queryset.order_by(expression, "uid")
+        return queryset.order_by(ordering, "uid")
+
+    def _validate_source_record_query(self) -> list[dict[str, str]]:
+        errors: list[dict[str, str]] = []
+        parsed_dates: dict[str, str] = {}
+        for field_name in ("date_from", "date_to"):
+            raw_value = self.request.query_params.get(field_name)
+            if not raw_value:
+                continue
+            try:
+                parsed_value = datetime.strptime(raw_value, "%Y-%m-%d").date()
+                if parsed_value.isoformat() != raw_value:
+                    raise ValueError
+            except (TypeError, ValueError):
+                errors.append(
+                    {
+                        "code": "invalid_date",
+                        "field": field_name,
+                        "message": f"{field_name} должен иметь формат YYYY-MM-DD",
+                    }
+                )
+                continue
+            parsed_dates[field_name] = parsed_value.isoformat()
+
+        date_from = parsed_dates.get("date_from")
+        date_to = parsed_dates.get("date_to")
+        if date_from and date_to and date_from > date_to:
+            errors.append(
+                {
+                    "code": "invalid_date_range",
+                    "field": "date_from",
+                    "message": "date_from не может быть позже date_to",
+                }
+            )
+
+        ordering = self.request.query_params.get("ordering")
+        if ordering and ordering not in SOURCE_RECORD_ORDERING_VALUES:
+            errors.append(
+                {
+                    "code": "invalid_ordering",
+                    "field": "ordering",
+                    "message": (
+                        "Поддерживаются: " + ", ".join(SOURCE_RECORD_ORDERING_VALUES)
+                    ),
+                }
+            )
+
+        self._source_record_date_from = date_from
+        self._source_record_date_to = date_to
+        self._source_record_ordering = ordering
+        return errors
+
     @classmethod
     def _filter_search_queryset(cls, queryset, search_terms: list[str]):
         queryset = queryset.annotate(
@@ -592,6 +714,17 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         responses={200: OrganizationSourceRecordListResponseSerializer},
     )
     def list(self, request, *args: Any, **kwargs: Any) -> Response:
+        errors = self._validate_source_record_query()
+        if errors:
+            return Response(
+                {
+                    "success": False,
+                    "data": [],
+                    "errors": errors,
+                    "meta": None,
+                },
+                status=status.HTTP_400_BAD_REQUEST,
+            )
         return super().list(request, *args, **kwargs)
 
     @swagger_auto_schema(