diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index 5d5125d..a6ecf75 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -35,7 +35,6 @@ from django.conf import settings from django.db.models import Q from django.utils import timezone from organizations.models import Organization, OrganizationSourceRecord -from organizations.test_companies import TestCompanyDatasetService class StateCorpExchangeError(ValueError): @@ -290,9 +289,7 @@ class StateCorpExchangeService: @classmethod def _rosatom_roscosmos_queryset(cls): - queryset = Organization.objects.exclude(inn="").exclude( - uid__in=TestCompanyDatasetService.company_uids() - ) + queryset = Organization.objects.exclude(inn="") name_query = Q() for keyword in cls.ROSATOM_ROSCOSMOS_GK_NAME_KEYWORDS: name_query |= Q(gk_name__icontains=keyword) diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index 6eb7379..88f5609 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -893,6 +893,11 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR records_by_source: dict[str, list[SourceRecordInput]] = defaultdict(list) for (record_source, _external_id), record in unique_records.items(): + record_date = record.record_date + if record_source in cls.vacancy_record_sources | { + ParserLoadLog.Source.ARBITRATION + }: + record_date = _date_to_iso(normalize_to_date(record.record_date)) or "" payload = dict(record.payload) if isinstance(record.payload, dict) else {} payload.update( { @@ -903,7 +908,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR "ogrn": record.ogrn, "organisation_name": record.organisation_name, "title": record.title, - "record_date": record.record_date, + "record_date": record_date, "amount": str(record.amount) if record.amount is not None else None, "status": record.status, "url": record.url, @@ -916,7 +921,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR organization_name=record.organisation_name, inn=record.inn, ogrn=record.ogrn, - record_date=record.record_date, + record_date=record_date, amount=record.amount, status=record.status, url=record.url, @@ -1503,7 +1508,7 @@ class InspectionService(BulkOperationsMixin, BaseService[InspectionRecord]): organization_name=insp.organisation_name, inn=insp.inn, ogrn=insp.ogrn, - record_date=insp.start_date, + record_date=_date_to_iso(normalize_to_date(insp.start_date)) or "", status=insp.status, payload={ "load_batch": batch_id, diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index 77cfdcc..1ca9309 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -65,6 +65,7 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): source_group = serializers.CharField( source="extension.source_group", read_only=True ) + record_date = serializers.SerializerMethodField() class Meta: model = OrganizationSourceRecord @@ -91,6 +92,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): ] read_only_fields = fields + @swagger_serializer_method( + serializer_or_field=serializers.DateField(allow_null=True, read_only=True), + ) + def get_record_date(self, obj) -> str | None: + return getattr(obj, "canonical_record_date", obj.record_date) or None + @swagger_serializer_method( serializer_or_field=OrganizationSourceRecordOrganizationSerializer, ) diff --git a/src/organizations/views.py b/src/organizations/views.py index 3dcfbf1..acb1583 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -6,14 +6,16 @@ import hashlib import json import os from contextlib import suppress +from datetime import datetime from tempfile import NamedTemporaryFile from typing import Any from apps.core.openapi import swagger_tag from django.conf import settings from django.core.cache import cache -from django.db.models import CharField, Q -from django.db.models.functions import Cast +from django.db.models import Case, CharField, F, Q, Value, When +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Cast, Coalesce, NullIf from django.http import HttpResponse from django_filters import rest_framework as filters from drf_yasg import openapi @@ -81,6 +83,22 @@ def _query_parameter( SOURCE_GROUP_VALUES = [choice.value for choice in SourceGroup] +SOURCE_RECORD_ORDERING_FIELDS = ( + "record_date", + "extension__organization__name", + "created_at", + "updated_at", + "title", + "uid", + "extension__organization__inn", + "extension__organization__ogrn", +) +SOURCE_RECORD_ORDERING_VALUES = [ + value + for field_name in SOURCE_RECORD_ORDERING_FIELDS + for value in (field_name, f"-{field_name}") +] + ORGANIZATION_LIST_PARAMS = [ _query_parameter( "page", @@ -178,6 +196,26 @@ SOURCE_RECORD_LIST_PARAMS = [ "статусу, датам, URL и исходным данным записи." ), ), + _query_parameter( + "date_from", + description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD).", + format_=openapi.FORMAT_DATE, + ), + _query_parameter( + "date_to", + description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD).", + format_=openapi.FORMAT_DATE, + ), + _query_parameter( + "ordering", + description=( + "Сортировка по полям: record_date, extension__organization__name, " + "created_at, updated_at, title, uid, extension__organization__inn, " + "extension__organization__ogrn. Для обратной сортировки используйте " + "префикс -. Значения record_date с null сортируются последними." + ), + enum=SOURCE_RECORD_ORDERING_VALUES, + ), _query_parameter( "page", description="Номер страницы.", param_type=openapi.TYPE_INTEGER ), @@ -472,7 +510,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): serializer_class = OrganizationSourceRecordSerializer permission_classes = [IsAuthenticated] lookup_field = "uid" - filter_backends = [OrderingFilter] + filter_backends = [] search_fields = [ "title", "external_id", @@ -494,16 +532,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): "extension__organization__ogrn", "extension__organization__ogrip", ] - ordering_fields = [ - "created_at", - "updated_at", - "record_date", - "title", - "uid", - "extension__organization__name", - "extension__organization__inn", - "extension__organization__ogrn", - ] + ordering_fields = SOURCE_RECORD_ORDERING_FIELDS ordering = ["-created_at", "-uid"] def get_permissions(self): @@ -514,7 +543,28 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): return super().get_permissions() def get_queryset(self): - queryset = super().get_queryset() + raw_record_date = NullIf(F("record_date"), Value("")) + inspection_record_date = Coalesce( + NullIf( + KeyTextTransform("start_date_normalized", "payload"), + Value(""), + ), + raw_record_date, + ) + queryset = ( + super() + .get_queryset() + .annotate( + canonical_record_date=Case( + When( + extension__source_group=SourceGroup.PLANNED_INSPECTIONS, + then=inspection_record_date, + ), + default=raw_record_date, + output_field=CharField(), + ) + ) + ) params = self.request.query_params source_group = params.get("source_group") source = params.get("source") @@ -533,8 +583,80 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): if search_terms: queryset = self._filter_search_queryset(queryset, search_terms) + date_from = getattr(self, "_source_record_date_from", None) + date_to = getattr(self, "_source_record_date_to", None) + if date_from: + queryset = queryset.filter(canonical_record_date__gte=date_from) + if date_to: + queryset = queryset.filter(canonical_record_date__lte=date_to) + + ordering = getattr(self, "_source_record_ordering", None) + if ordering: + return self._order_source_records(queryset, ordering) + return queryset + @staticmethod + def _order_source_records(queryset, ordering: str): + if ordering.lstrip("-") == "record_date": + expression = F("canonical_record_date") + if ordering.startswith("-"): + expression = expression.desc(nulls_last=True) + else: + expression = expression.asc(nulls_last=True) + return queryset.order_by(expression, "uid") + return queryset.order_by(ordering, "uid") + + def _validate_source_record_query(self) -> list[dict[str, str]]: + errors: list[dict[str, str]] = [] + parsed_dates: dict[str, str] = {} + for field_name in ("date_from", "date_to"): + raw_value = self.request.query_params.get(field_name) + if not raw_value: + continue + try: + parsed_value = datetime.strptime(raw_value, "%Y-%m-%d").date() + if parsed_value.isoformat() != raw_value: + raise ValueError + except (TypeError, ValueError): + errors.append( + { + "code": "invalid_date", + "field": field_name, + "message": f"{field_name} должен иметь формат YYYY-MM-DD", + } + ) + continue + parsed_dates[field_name] = parsed_value.isoformat() + + date_from = parsed_dates.get("date_from") + date_to = parsed_dates.get("date_to") + if date_from and date_to and date_from > date_to: + errors.append( + { + "code": "invalid_date_range", + "field": "date_from", + "message": "date_from не может быть позже date_to", + } + ) + + ordering = self.request.query_params.get("ordering") + if ordering and ordering not in SOURCE_RECORD_ORDERING_VALUES: + errors.append( + { + "code": "invalid_ordering", + "field": "ordering", + "message": ( + "Поддерживаются: " + ", ".join(SOURCE_RECORD_ORDERING_VALUES) + ), + } + ) + + self._source_record_date_from = date_from + self._source_record_date_to = date_to + self._source_record_ordering = ordering + return errors + @classmethod def _filter_search_queryset(cls, queryset, search_terms: list[str]): queryset = queryset.annotate( @@ -592,6 +714,17 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): responses={200: OrganizationSourceRecordListResponseSerializer}, ) def list(self, request, *args: Any, **kwargs: Any) -> Response: + errors = self._validate_source_record_query() + if errors: + return Response( + { + "success": False, + "data": [], + "errors": errors, + "meta": None, + }, + status=status.HTTP_400_BAD_REQUEST, + ) return super().list(request, *args, **kwargs) @swagger_auto_schema( diff --git a/tests/apps/exchange/test_state_corp_services.py b/tests/apps/exchange/test_state_corp_services.py index 36a3ad5..7b60287 100644 --- a/tests/apps/exchange/test_state_corp_services.py +++ b/tests/apps/exchange/test_state_corp_services.py @@ -68,7 +68,7 @@ TEST_STATE_CORP_TOKEN = "state-corp-test-exchange-token" # noqa: S105 class StateCorpExchangeServiceTest(TestCase): """Verify package compatibility with state-corp receiver contract.""" - def test_default_package_excludes_deterministic_test_companies(self): + def test_default_package_includes_deterministic_test_companies(self): real_organization = Organization.objects.create( rn=1001, name="Реальная организация", @@ -86,10 +86,13 @@ class StateCorpExchangeServiceTest(TestCase): package = StateCorpExchangeService.build_package() payload = _decode_package_payload(package) - self.assertEqual(package.payload_counts["organizations"], 1) + self.assertEqual(package.payload_counts["organizations"], 2) self.assertEqual( [row["mostovik_uid"] for row in payload["data"]["organizations"]], - [str(real_organization.uid)], + [ + str(real_organization.uid), + str(TestCompanyDatasetService.company_uids()[0]), + ], ) def test_build_package_contains_expected_payload(self): diff --git a/tests/apps/organizations/test_api_v2_source_extensions.py b/tests/apps/organizations/test_api_v2_source_extensions.py index 6adb849..3a6740b 100644 --- a/tests/apps/organizations/test_api_v2_source_extensions.py +++ b/tests/apps/organizations/test_api_v2_source_extensions.py @@ -1,11 +1,15 @@ """Tests for organization API v2 backed by source extensions.""" +from uuid import UUID + from django.core.cache import cache from django.urls import reverse from organizations.models import ( + ArbitrationExtension, Organization, OrganizationSourceRecord, PlannedInspectionExtension, + VacancyExtension, ) from rest_framework import status from rest_framework.test import APITestCase @@ -166,6 +170,191 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(record["source_group"], "planned_inspections") self.assertEqual(record["organization"]["uid"], str(target.uid)) + def test_flat_source_records_filters_supported_groups_by_canonical_date(self): + organization = create_frontend_organization( + name='ООО "Canonical dates"', + inn="7707083890", + ogrn="1027700132090", + ) + extensions = { + "planned_inspections": PlannedInspectionExtension.objects.create( + organization=organization, + title="Проверки", + ), + "vacancies": VacancyExtension.objects.create( + organization=organization, + title="Вакансии", + ), + "arbitration": ArbitrationExtension.objects.create( + organization=organization, + title="Арбитраж", + ), + } + OrganizationSourceRecord.objects.create( + extension=extensions["planned_inspections"], + record_type="inspection", + source="inspections", + external_id="DATE-INSPECTION", + record_date="12.03.2026", + payload={"start_date_normalized": "2026-03-12"}, + ) + OrganizationSourceRecord.objects.create( + extension=extensions["vacancies"], + record_type="vacancy", + source="trudvsem", + external_id="DATE-VACANCY", + record_date="2026-03-31", + ) + OrganizationSourceRecord.objects.create( + extension=extensions["arbitration"], + record_type="arbitration_case", + source="arbitration", + external_id="DATE-ARBITRATION", + record_date="2026-04-01", + ) + OrganizationSourceRecord.objects.create( + extension=extensions["vacancies"], + record_type="vacancy", + source="trudvsem", + external_id="DATE-NULL", + record_date="", + ) + + response = self.client.get( + reverse("api_v2:organizations:organization-source-records-list"), + { + "date_from": "2026-03-12", + "date_to": "2026-03-31", + "ordering": "record_date", + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["meta"]["pagination"]["total_count"], 2) + self.assertEqual( + [item["external_id"] for item in response.data["data"]], + ["DATE-INSPECTION", "DATE-VACANCY"], + ) + self.assertEqual( + [item["record_date"] for item in response.data["data"]], + ["2026-03-12", "2026-03-31"], + ) + + def test_flat_source_records_orders_dates_stably_with_nulls_last(self): + organization = create_frontend_organization( + name='ООО "Stable dates"', + inn="7707083891", + ogrn="1027700132091", + ) + extension = VacancyExtension.objects.create( + organization=organization, + title="Вакансии", + ) + records = [ + ("00000000-0000-0000-0000-000000000004", "DATE-NULL", ""), + ("00000000-0000-0000-0000-000000000003", "DATE-LATE", "2026-05-01"), + ("00000000-0000-0000-0000-000000000002", "DATE-EQUAL-2", "2026-04-01"), + ("00000000-0000-0000-0000-000000000001", "DATE-EQUAL-1", "2026-04-01"), + ] + for uid, external_id, record_date in records: + OrganizationSourceRecord.objects.create( + uid=UUID(uid), + extension=extension, + record_type="vacancy", + source="trudvsem", + external_id=external_id, + record_date=record_date, + ) + + endpoint = reverse("api_v2:organizations:organization-source-records-list") + ascending = self.client.get(endpoint, {"ordering": "record_date"}) + descending = self.client.get(endpoint, {"ordering": "-record_date"}) + + self.assertEqual(ascending.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["external_id"] for item in ascending.data["data"]], + ["DATE-EQUAL-1", "DATE-EQUAL-2", "DATE-LATE", "DATE-NULL"], + ) + self.assertEqual( + [item["external_id"] for item in descending.data["data"]], + ["DATE-LATE", "DATE-EQUAL-1", "DATE-EQUAL-2", "DATE-NULL"], + ) + self.assertIsNone(ascending.data["data"][-1]["record_date"]) + self.assertIsNone(descending.data["data"][-1]["record_date"]) + + def test_flat_source_records_rejects_invalid_date_range_and_ordering(self): + response = self.client.get( + reverse("api_v2:organizations:organization-source-records-list"), + { + "date_from": "2026-04-01", + "date_to": "2026-03-01", + "ordering": "unknown_date", + }, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(response.data["success"]) + self.assertEqual(response.data["data"], []) + self.assertEqual( + {error["code"] for error in response.data["errors"]}, + {"invalid_date_range", "invalid_ordering"}, + ) + + def test_flat_source_records_rejects_invalid_date_format(self): + for invalid_date in ("01.04.2026", "2026-4-1"): + with self.subTest(invalid_date=invalid_date): + response = self.client.get( + reverse("api_v2:organizations:organization-source-records-list"), + {"date_from": invalid_date}, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data["errors"][0]["code"], "invalid_date") + self.assertEqual(response.data["errors"][0]["field"], "date_from") + self.assertIn("YYYY-MM-DD", response.data["errors"][0]["message"]) + + def test_flat_source_record_date_filter_precedes_pagination(self): + organization = create_frontend_organization( + name='ООО "Filtered pagination"', + inn="7707083892", + ogrn="1027700132092", + ) + extension = ArbitrationExtension.objects.create( + organization=organization, + title="Арбитраж", + ) + for index, record_date in enumerate( + ("2026-01-01", "2026-02-01", "2026-03-01"), + start=1, + ): + OrganizationSourceRecord.objects.create( + extension=extension, + record_type="arbitration_case", + source="arbitration", + external_id=f"PAGE-{index}", + title="Искомое дело", + record_date=record_date, + ) + + response = self.client.get( + reverse("api_v2:organizations:organization-source-records-list"), + { + "source_group": "arbitration", + "source": "arbitration", + "organization": str(organization.uid), + "search": "Искомое", + "date_from": "2026-02-01", + "ordering": "record_date", + "page_size": 1, + "page": 2, + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["meta"]["pagination"]["total_count"], 2) + self.assertEqual(response.data["meta"]["pagination"]["total_pages"], 2) + self.assertEqual(response.data["data"][0]["external_id"], "PAGE-3") + def test_flat_source_records_scope_cannot_be_lifted_by_has_registry(self): with_registry = create_frontend_organization( name='ООО "With Registry Source"', diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py index c4741d2..6848af0 100644 --- a/tests/apps/organizations/test_test_companies_commands.py +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -133,7 +133,7 @@ class TestCompaniesCommandsTest(TestCase): self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15) @override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN) - def test_created_source_records_are_excluded_from_state_corp_package(self): + def test_created_source_records_are_exported_to_state_corp_package(self): call_command("create_test_companies", stdout=StringIO()) company_inns = list( @@ -146,18 +146,18 @@ class TestCompaniesCommandsTest(TestCase): self.assertEqual( package.payload_counts, { - "organizations": 0, - "industrial_certificates": 0, - "manufacturers": 0, - "industrial_products": 0, - "prosecutor_checks": 0, - "public_procurements": 0, - "financial_reports": 0, - "arbitration_cases": 0, - "bankruptcy_procedures": 0, - "defense_unreliable_suppliers": 0, - "information_security_registries": 0, - "labor_vacancies": 0, + "organizations": 20, + "industrial_certificates": 20, + "manufacturers": 20, + "industrial_products": 20, + "prosecutor_checks": 20, + "public_procurements": 60, + "financial_reports": 20, + "arbitration_cases": 20, + "bankruptcy_procedures": 20, + "defense_unreliable_suppliers": 40, + "information_security_registries": 20, + "labor_vacancies": 20, }, ) diff --git a/tests/apps/parsers/test_direct_ingestion_services.py b/tests/apps/parsers/test_direct_ingestion_services.py index 3202653..e2ddaa4 100644 --- a/tests/apps/parsers/test_direct_ingestion_services.py +++ b/tests/apps/parsers/test_direct_ingestion_services.py @@ -32,7 +32,6 @@ from apps.parsers.services import ( ) from django.test import TestCase from organizations.models import ( - Organization, OrganizationSourceFinancialLine, OrganizationSourceRecord, ) @@ -62,7 +61,9 @@ class DirectIngestionParserServicesTest(TestCase): okpo="", ) - def test_industrial_certificate_save_records_writes_organization_source_records(self): + def test_industrial_certificate_save_records_writes_organization_source_records( + self, + ): saved = IndustrialCertificateService.save_certificates( [ IndustrialCertificate( @@ -89,7 +90,9 @@ class DirectIngestionParserServicesTest(TestCase): self.assertEqual(record.payload["expiry_date_normalized"], "2029-02-01") self.assertEqual(record.url, "https://example.test/cert.pdf") - def test_industrial_certificate_save_records_skips_records_without_certificate_number(self): + def test_industrial_certificate_save_records_skips_records_without_certificate_number( + self, + ): saved = IndustrialCertificateService.save_certificates( [ IndustrialCertificate( @@ -255,6 +258,39 @@ class DirectIngestionParserServicesTest(TestCase): self.assertEqual(record.payload["registry"], "fas") self.assertEqual(record.load_batch, 51) + def test_vacancy_and_arbitration_dates_are_normalized_for_source_records(self): + for index, (source, raw_date, expected_date) in enumerate( + ( + (ParserLoadLog.Source.TRUDVSEM, "04.12.2025", "2025-12-04"), + (ParserLoadLog.Source.ARBITRATION, "14.07.2026", "2026-07-14"), + ), + start=1, + ): + external_id = f"canonical-date-{index}" + saved = GenericParserRecordService.save_records( + [ + GenericParserItem( + source=source, + external_id=external_id, + inn="7707083803", + ogrn="", + organisation_name='ООО "Каноническая дата"', + title="Запись с датой", + record_date=raw_date, + ) + ], + batch_id=60 + index, + source=source, + ) + + self.assertEqual(saved, 1) + record = OrganizationSourceRecord.objects.get( + source=source, + external_id=external_id, + ) + self.assertEqual(record.record_date, expected_date) + self.assertEqual(record.payload["record_date"], expected_date) + def test_inspection_save_records_writes_organization_source_records(self): saved = InspectionService.save_inspections( [ @@ -285,6 +321,7 @@ class DirectIngestionParserServicesTest(TestCase): external_id="INSP-DIRECT-1", ) self.assertEqual(record.record_type, "inspection") + self.assertEqual(record.record_date, "2026-03-01") self.assertEqual(record.payload["control_authority"], "Контроль") self.assertEqual(record.payload["start_date_normalized"], "2026-03-01") self.assertEqual(record.payload["data_year"], 2026)