Merge pull request 'Align source API and exchange demo data' (#37) from feature/frontend-exchange-corrections-20260722 into main

src/apps/exchange/state_corp_services.py

@@ -35,7 +35,6 @@ from django.conf import settings
 from django.db.models import Q
 from django.utils import timezone
 from organizations.models import Organization, OrganizationSourceRecord
-from organizations.test_companies import TestCompanyDatasetService
 
 
 class StateCorpExchangeError(ValueError):
@@ -290,9 +289,7 @@ class StateCorpExchangeService:
 
     @classmethod
     def _rosatom_roscosmos_queryset(cls):
-        queryset = Organization.objects.exclude(inn="").exclude(
-            uid__in=TestCompanyDatasetService.company_uids()
-        )
+        queryset = Organization.objects.exclude(inn="")
         name_query = Q()
         for keyword in cls.ROSATOM_ROSCOSMOS_GK_NAME_KEYWORDS:
             name_query |= Q(gk_name__icontains=keyword)

src/apps/parsers/services.py

@@ -893,6 +893,11 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR
 
         records_by_source: dict[str, list[SourceRecordInput]] = defaultdict(list)
         for (record_source, _external_id), record in unique_records.items():
+            record_date = record.record_date
+            if record_source in cls.vacancy_record_sources | {
+                ParserLoadLog.Source.ARBITRATION
+            }:
+                record_date = _date_to_iso(normalize_to_date(record.record_date)) or ""
             payload = dict(record.payload) if isinstance(record.payload, dict) else {}
             payload.update(
                 {
@@ -903,7 +908,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR
                     "ogrn": record.ogrn,
                     "organisation_name": record.organisation_name,
                     "title": record.title,
-                    "record_date": record.record_date,
+                    "record_date": record_date,
                     "amount": str(record.amount) if record.amount is not None else None,
                     "status": record.status,
                     "url": record.url,
@@ -916,7 +921,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR
                     organization_name=record.organisation_name,
                     inn=record.inn,
                     ogrn=record.ogrn,
-                    record_date=record.record_date,
+                    record_date=record_date,
                     amount=record.amount,
                     status=record.status,
                     url=record.url,
@@ -1503,7 +1508,7 @@ class InspectionService(BulkOperationsMixin, BaseService[InspectionRecord]):
                 organization_name=insp.organisation_name,
                 inn=insp.inn,
                 ogrn=insp.ogrn,
-                record_date=insp.start_date,
+                record_date=_date_to_iso(normalize_to_date(insp.start_date)) or "",
                 status=insp.status,
                 payload={
                     "load_batch": batch_id,

src/organizations/serializers.py

@@ -65,6 +65,7 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
     source_group = serializers.CharField(
         source="extension.source_group", read_only=True
     )
+    record_date = serializers.SerializerMethodField()
 
     class Meta:
         model = OrganizationSourceRecord
@@ -91,6 +92,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
         ]
         read_only_fields = fields
 
+    @swagger_serializer_method(
+        serializer_or_field=serializers.DateField(allow_null=True, read_only=True),
+    )
+    def get_record_date(self, obj) -> str | None:
+        return getattr(obj, "canonical_record_date", obj.record_date) or None
+
     @swagger_serializer_method(
         serializer_or_field=OrganizationSourceRecordOrganizationSerializer,
     )

src/organizations/views.py

@@ -6,14 +6,16 @@ import hashlib
 import json
 import os
 from contextlib import suppress
+from datetime import datetime
 from tempfile import NamedTemporaryFile
 from typing import Any
 
 from apps.core.openapi import swagger_tag
 from django.conf import settings
 from django.core.cache import cache
-from django.db.models import CharField, Q
-from django.db.models.functions import Cast
+from django.db.models import Case, CharField, F, Q, Value, When
+from django.db.models.fields.json import KeyTextTransform
+from django.db.models.functions import Cast, Coalesce, NullIf
 from django.http import HttpResponse
 from django_filters import rest_framework as filters
 from drf_yasg import openapi
@@ -81,6 +83,22 @@ def _query_parameter(
 
 SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup]
 
+SOURCE_RECORD_ORDERING_FIELDS = (
+    "record_date",
+    "extension__organization__name",
+    "created_at",
+    "updated_at",
+    "title",
+    "uid",
+    "extension__organization__inn",
+    "extension__organization__ogrn",
+)
+SOURCE_RECORD_ORDERING_VALUES = [
+    value
+    for field_name in SOURCE_RECORD_ORDERING_FIELDS
+    for value in (field_name, f"-{field_name}")
+]
+
 ORGANIZATION_LIST_PARAMS = [
     _query_parameter(
         "page",
@@ -178,6 +196,26 @@ SOURCE_RECORD_LIST_PARAMS = [
             "статусу, датам, URL и исходным данным записи."
         ),
     ),
+    _query_parameter(
+        "date_from",
+        description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD).",
+        format_=openapi.FORMAT_DATE,
+    ),
+    _query_parameter(
+        "date_to",
+        description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD).",
+        format_=openapi.FORMAT_DATE,
+    ),
+    _query_parameter(
+        "ordering",
+        description=(
+            "Сортировка по полям: record_date, extension__organization__name, "
+            "created_at, updated_at, title, uid, extension__organization__inn, "
+            "extension__organization__ogrn. Для обратной сортировки используйте "
+            "префикс -. Значения record_date с null сортируются последними."
+        ),
+        enum=SOURCE_RECORD_ORDERING_VALUES,
+    ),
     _query_parameter(
         "page", description="Номер страницы.", param_type=openapi.TYPE_INTEGER
     ),
@@ -472,7 +510,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
     serializer_class = OrganizationSourceRecordSerializer
     permission_classes = [IsAuthenticated]
     lookup_field = "uid"
-    filter_backends = [OrderingFilter]
+    filter_backends = []
     search_fields = [
         "title",
         "external_id",
@@ -494,16 +532,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         "extension__organization__ogrn",
         "extension__organization__ogrip",
     ]
-    ordering_fields = [
-        "created_at",
-        "updated_at",
-        "record_date",
-        "title",
-        "uid",
-        "extension__organization__name",
-        "extension__organization__inn",
-        "extension__organization__ogrn",
-    ]
+    ordering_fields = SOURCE_RECORD_ORDERING_FIELDS
     ordering = ["-created_at", "-uid"]
 
     def get_permissions(self):
@@ -514,7 +543,28 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         return super().get_permissions()
 
     def get_queryset(self):
-        queryset = super().get_queryset()
+        raw_record_date = NullIf(F("record_date"), Value(""))
+        inspection_record_date = Coalesce(
+            NullIf(
+                KeyTextTransform("start_date_normalized", "payload"),
+                Value(""),
+            ),
+            raw_record_date,
+        )
+        queryset = (
+            super()
+            .get_queryset()
+            .annotate(
+                canonical_record_date=Case(
+                    When(
+                        extension__source_group=SourceGroup.PLANNED_INSPECTIONS,
+                        then=inspection_record_date,
+                    ),
+                    default=raw_record_date,
+                    output_field=CharField(),
+                )
+            )
+        )
         params = self.request.query_params
         source_group = params.get("source_group")
         source = params.get("source")
@@ -533,8 +583,80 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         if search_terms:
             queryset = self._filter_search_queryset(queryset, search_terms)
 
+        date_from = getattr(self, "_source_record_date_from", None)
+        date_to = getattr(self, "_source_record_date_to", None)
+        if date_from:
+            queryset = queryset.filter(canonical_record_date__gte=date_from)
+        if date_to:
+            queryset = queryset.filter(canonical_record_date__lte=date_to)
+
+        ordering = getattr(self, "_source_record_ordering", None)
+        if ordering:
+            return self._order_source_records(queryset, ordering)
+
         return queryset
 
+    @staticmethod
+    def _order_source_records(queryset, ordering: str):
+        if ordering.lstrip("-") == "record_date":
+            expression = F("canonical_record_date")
+            if ordering.startswith("-"):
+                expression = expression.desc(nulls_last=True)
+            else:
+                expression = expression.asc(nulls_last=True)
+            return queryset.order_by(expression, "uid")
+        return queryset.order_by(ordering, "uid")
+
+    def _validate_source_record_query(self) -> list[dict[str, str]]:
+        errors: list[dict[str, str]] = []
+        parsed_dates: dict[str, str] = {}
+        for field_name in ("date_from", "date_to"):
+            raw_value = self.request.query_params.get(field_name)
+            if not raw_value:
+                continue
+            try:
+                parsed_value = datetime.strptime(raw_value, "%Y-%m-%d").date()
+                if parsed_value.isoformat() != raw_value:
+                    raise ValueError
+            except (TypeError, ValueError):
+                errors.append(
+                    {
+                        "code": "invalid_date",
+                        "field": field_name,
+                        "message": f"{field_name} должен иметь формат YYYY-MM-DD",
+                    }
+                )
+                continue
+            parsed_dates[field_name] = parsed_value.isoformat()
+
+        date_from = parsed_dates.get("date_from")
+        date_to = parsed_dates.get("date_to")
+        if date_from and date_to and date_from > date_to:
+            errors.append(
+                {
+                    "code": "invalid_date_range",
+                    "field": "date_from",
+                    "message": "date_from не может быть позже date_to",
+                }
+            )
+
+        ordering = self.request.query_params.get("ordering")
+        if ordering and ordering not in SOURCE_RECORD_ORDERING_VALUES:
+            errors.append(
+                {
+                    "code": "invalid_ordering",
+                    "field": "ordering",
+                    "message": (
+                        "Поддерживаются: " + ", ".join(SOURCE_RECORD_ORDERING_VALUES)
+                    ),
+                }
+            )
+
+        self._source_record_date_from = date_from
+        self._source_record_date_to = date_to
+        self._source_record_ordering = ordering
+        return errors
+
     @classmethod
     def _filter_search_queryset(cls, queryset, search_terms: list[str]):
         queryset = queryset.annotate(
@@ -592,6 +714,17 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet):
         responses={200: OrganizationSourceRecordListResponseSerializer},
     )
     def list(self, request, *args: Any, **kwargs: Any) -> Response:
+        errors = self._validate_source_record_query()
+        if errors:
+            return Response(
+                {
+                    "success": False,
+                    "data": [],
+                    "errors": errors,
+                    "meta": None,
+                },
+                status=status.HTTP_400_BAD_REQUEST,
+            )
         return super().list(request, *args, **kwargs)
 
     @swagger_auto_schema(

tests/apps/exchange/test_state_corp_services.py

@@ -68,7 +68,7 @@ TEST_STATE_CORP_TOKEN = "state-corp-test-exchange-token"  # noqa: S105
 class StateCorpExchangeServiceTest(TestCase):
     """Verify package compatibility with state-corp receiver contract."""
 
-    def test_default_package_excludes_deterministic_test_companies(self):
+    def test_default_package_includes_deterministic_test_companies(self):
         real_organization = Organization.objects.create(
             rn=1001,
             name="Реальная организация",
@@ -86,10 +86,13 @@ class StateCorpExchangeServiceTest(TestCase):
         package = StateCorpExchangeService.build_package()
         payload = _decode_package_payload(package)
 
-        self.assertEqual(package.payload_counts["organizations"], 1)
+        self.assertEqual(package.payload_counts["organizations"], 2)
         self.assertEqual(
             [row["mostovik_uid"] for row in payload["data"]["organizations"]],
-            [str(real_organization.uid)],
+            [
+                str(real_organization.uid),
+                str(TestCompanyDatasetService.company_uids()[0]),
+            ],
         )
 
     def test_build_package_contains_expected_payload(self):

tests/apps/organizations/test_api_v2_source_extensions.py

@@ -1,11 +1,15 @@
 """Tests for organization API v2 backed by source extensions."""
 
+from uuid import UUID
+
 from django.core.cache import cache
 from django.urls import reverse
 from organizations.models import (
+    ArbitrationExtension,
     Organization,
     OrganizationSourceRecord,
     PlannedInspectionExtension,
+    VacancyExtension,
 )
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -166,6 +170,191 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase):
         self.assertEqual(record["source_group"], "planned_inspections")
         self.assertEqual(record["organization"]["uid"], str(target.uid))
 
+    def test_flat_source_records_filters_supported_groups_by_canonical_date(self):
+        organization = create_frontend_organization(
+            name='ООО "Canonical dates"',
+            inn="7707083890",
+            ogrn="1027700132090",
+        )
+        extensions = {
+            "planned_inspections": PlannedInspectionExtension.objects.create(
+                organization=organization,
+                title="Проверки",
+            ),
+            "vacancies": VacancyExtension.objects.create(
+                organization=organization,
+                title="Вакансии",
+            ),
+            "arbitration": ArbitrationExtension.objects.create(
+                organization=organization,
+                title="Арбитраж",
+            ),
+        }
+        OrganizationSourceRecord.objects.create(
+            extension=extensions["planned_inspections"],
+            record_type="inspection",
+            source="inspections",
+            external_id="DATE-INSPECTION",
+            record_date="12.03.2026",
+            payload={"start_date_normalized": "2026-03-12"},
+        )
+        OrganizationSourceRecord.objects.create(
+            extension=extensions["vacancies"],
+            record_type="vacancy",
+            source="trudvsem",
+            external_id="DATE-VACANCY",
+            record_date="2026-03-31",
+        )
+        OrganizationSourceRecord.objects.create(
+            extension=extensions["arbitration"],
+            record_type="arbitration_case",
+            source="arbitration",
+            external_id="DATE-ARBITRATION",
+            record_date="2026-04-01",
+        )
+        OrganizationSourceRecord.objects.create(
+            extension=extensions["vacancies"],
+            record_type="vacancy",
+            source="trudvsem",
+            external_id="DATE-NULL",
+            record_date="",
+        )
+
+        response = self.client.get(
+            reverse("api_v2:organizations:organization-source-records-list"),
+            {
+                "date_from": "2026-03-12",
+                "date_to": "2026-03-31",
+                "ordering": "record_date",
+            },
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["meta"]["pagination"]["total_count"], 2)
+        self.assertEqual(
+            [item["external_id"] for item in response.data["data"]],
+            ["DATE-INSPECTION", "DATE-VACANCY"],
+        )
+        self.assertEqual(
+            [item["record_date"] for item in response.data["data"]],
+            ["2026-03-12", "2026-03-31"],
+        )
+
+    def test_flat_source_records_orders_dates_stably_with_nulls_last(self):
+        organization = create_frontend_organization(
+            name='ООО "Stable dates"',
+            inn="7707083891",
+            ogrn="1027700132091",
+        )
+        extension = VacancyExtension.objects.create(
+            organization=organization,
+            title="Вакансии",
+        )
+        records = [
+            ("00000000-0000-0000-0000-000000000004", "DATE-NULL", ""),
+            ("00000000-0000-0000-0000-000000000003", "DATE-LATE", "2026-05-01"),
+            ("00000000-0000-0000-0000-000000000002", "DATE-EQUAL-2", "2026-04-01"),
+            ("00000000-0000-0000-0000-000000000001", "DATE-EQUAL-1", "2026-04-01"),
+        ]
+        for uid, external_id, record_date in records:
+            OrganizationSourceRecord.objects.create(
+                uid=UUID(uid),
+                extension=extension,
+                record_type="vacancy",
+                source="trudvsem",
+                external_id=external_id,
+                record_date=record_date,
+            )
+
+        endpoint = reverse("api_v2:organizations:organization-source-records-list")
+        ascending = self.client.get(endpoint, {"ordering": "record_date"})
+        descending = self.client.get(endpoint, {"ordering": "-record_date"})
+
+        self.assertEqual(ascending.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [item["external_id"] for item in ascending.data["data"]],
+            ["DATE-EQUAL-1", "DATE-EQUAL-2", "DATE-LATE", "DATE-NULL"],
+        )
+        self.assertEqual(
+            [item["external_id"] for item in descending.data["data"]],
+            ["DATE-LATE", "DATE-EQUAL-1", "DATE-EQUAL-2", "DATE-NULL"],
+        )
+        self.assertIsNone(ascending.data["data"][-1]["record_date"])
+        self.assertIsNone(descending.data["data"][-1]["record_date"])
+
+    def test_flat_source_records_rejects_invalid_date_range_and_ordering(self):
+        response = self.client.get(
+            reverse("api_v2:organizations:organization-source-records-list"),
+            {
+                "date_from": "2026-04-01",
+                "date_to": "2026-03-01",
+                "ordering": "unknown_date",
+            },
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertFalse(response.data["success"])
+        self.assertEqual(response.data["data"], [])
+        self.assertEqual(
+            {error["code"] for error in response.data["errors"]},
+            {"invalid_date_range", "invalid_ordering"},
+        )
+
+    def test_flat_source_records_rejects_invalid_date_format(self):
+        for invalid_date in ("01.04.2026", "2026-4-1"):
+            with self.subTest(invalid_date=invalid_date):
+                response = self.client.get(
+                    reverse("api_v2:organizations:organization-source-records-list"),
+                    {"date_from": invalid_date},
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertEqual(response.data["errors"][0]["code"], "invalid_date")
+                self.assertEqual(response.data["errors"][0]["field"], "date_from")
+                self.assertIn("YYYY-MM-DD", response.data["errors"][0]["message"])
+
+    def test_flat_source_record_date_filter_precedes_pagination(self):
+        organization = create_frontend_organization(
+            name='ООО "Filtered pagination"',
+            inn="7707083892",
+            ogrn="1027700132092",
+        )
+        extension = ArbitrationExtension.objects.create(
+            organization=organization,
+            title="Арбитраж",
+        )
+        for index, record_date in enumerate(
+            ("2026-01-01", "2026-02-01", "2026-03-01"),
+            start=1,
+        ):
+            OrganizationSourceRecord.objects.create(
+                extension=extension,
+                record_type="arbitration_case",
+                source="arbitration",
+                external_id=f"PAGE-{index}",
+                title="Искомое дело",
+                record_date=record_date,
+            )
+
+        response = self.client.get(
+            reverse("api_v2:organizations:organization-source-records-list"),
+            {
+                "source_group": "arbitration",
+                "source": "arbitration",
+                "organization": str(organization.uid),
+                "search": "Искомое",
+                "date_from": "2026-02-01",
+                "ordering": "record_date",
+                "page_size": 1,
+                "page": 2,
+            },
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["meta"]["pagination"]["total_count"], 2)
+        self.assertEqual(response.data["meta"]["pagination"]["total_pages"], 2)
+        self.assertEqual(response.data["data"][0]["external_id"], "PAGE-3")
+
     def test_flat_source_records_scope_cannot_be_lifted_by_has_registry(self):
         with_registry = create_frontend_organization(
             name='ООО "With Registry Source"',

tests/apps/organizations/test_test_companies_commands.py

@@ -133,7 +133,7 @@ class TestCompaniesCommandsTest(TestCase):
         self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15)
 
     @override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN)
-    def test_created_source_records_are_excluded_from_state_corp_package(self):
+    def test_created_source_records_are_exported_to_state_corp_package(self):
         call_command("create_test_companies", stdout=StringIO())
 
         company_inns = list(
@@ -146,18 +146,18 @@ class TestCompaniesCommandsTest(TestCase):
         self.assertEqual(
             package.payload_counts,
             {
-                "organizations": 0,
-                "industrial_certificates": 0,
-                "manufacturers": 0,
-                "industrial_products": 0,
-                "prosecutor_checks": 0,
-                "public_procurements": 0,
-                "financial_reports": 0,
-                "arbitration_cases": 0,
-                "bankruptcy_procedures": 0,
-                "defense_unreliable_suppliers": 0,
-                "information_security_registries": 0,
-                "labor_vacancies": 0,
+                "organizations": 20,
+                "industrial_certificates": 20,
+                "manufacturers": 20,
+                "industrial_products": 20,
+                "prosecutor_checks": 20,
+                "public_procurements": 60,
+                "financial_reports": 20,
+                "arbitration_cases": 20,
+                "bankruptcy_procedures": 20,
+                "defense_unreliable_suppliers": 40,
+                "information_security_registries": 20,
+                "labor_vacancies": 20,
             },
         )
 

tests/apps/parsers/test_direct_ingestion_services.py

@@ -32,7 +32,6 @@ from apps.parsers.services import (
 )
 from django.test import TestCase
 from organizations.models import (
-    Organization,
     OrganizationSourceFinancialLine,
     OrganizationSourceRecord,
 )
@@ -62,7 +61,9 @@ class DirectIngestionParserServicesTest(TestCase):
                 okpo="",
             )
 
-    def test_industrial_certificate_save_records_writes_organization_source_records(self):
+    def test_industrial_certificate_save_records_writes_organization_source_records(
+        self,
+    ):
         saved = IndustrialCertificateService.save_certificates(
             [
                 IndustrialCertificate(
@@ -89,7 +90,9 @@ class DirectIngestionParserServicesTest(TestCase):
         self.assertEqual(record.payload["expiry_date_normalized"], "2029-02-01")
         self.assertEqual(record.url, "https://example.test/cert.pdf")
 
-    def test_industrial_certificate_save_records_skips_records_without_certificate_number(self):
+    def test_industrial_certificate_save_records_skips_records_without_certificate_number(
+        self,
+    ):
         saved = IndustrialCertificateService.save_certificates(
             [
                 IndustrialCertificate(
@@ -255,6 +258,39 @@ class DirectIngestionParserServicesTest(TestCase):
         self.assertEqual(record.payload["registry"], "fas")
         self.assertEqual(record.load_batch, 51)
 
+    def test_vacancy_and_arbitration_dates_are_normalized_for_source_records(self):
+        for index, (source, raw_date, expected_date) in enumerate(
+            (
+                (ParserLoadLog.Source.TRUDVSEM, "04.12.2025", "2025-12-04"),
+                (ParserLoadLog.Source.ARBITRATION, "14.07.2026", "2026-07-14"),
+            ),
+            start=1,
+        ):
+            external_id = f"canonical-date-{index}"
+            saved = GenericParserRecordService.save_records(
+                [
+                    GenericParserItem(
+                        source=source,
+                        external_id=external_id,
+                        inn="7707083803",
+                        ogrn="",
+                        organisation_name='ООО "Каноническая дата"',
+                        title="Запись с датой",
+                        record_date=raw_date,
+                    )
+                ],
+                batch_id=60 + index,
+                source=source,
+            )
+
+            self.assertEqual(saved, 1)
+            record = OrganizationSourceRecord.objects.get(
+                source=source,
+                external_id=external_id,
+            )
+            self.assertEqual(record.record_date, expected_date)
+            self.assertEqual(record.payload["record_date"], expected_date)
+
     def test_inspection_save_records_writes_organization_source_records(self):
         saved = InspectionService.save_inspections(
             [
@@ -285,6 +321,7 @@ class DirectIngestionParserServicesTest(TestCase):
             external_id="INSP-DIRECT-1",
         )
         self.assertEqual(record.record_type, "inspection")
+        self.assertEqual(record.record_date, "2026-03-01")
         self.assertEqual(record.payload["control_authority"], "Контроль")
         self.assertEqual(record.payload["start_date_normalized"], "2026-03-01")
         self.assertEqual(record.payload["data_year"], 2026)