Add organizations v2 API and registry enrichment

src/organizations/views.py

@@ -0,0 +1,456 @@
+"""Views for organizations API v2."""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+from apps.core.openapi import swagger_tag
+from django.conf import settings
+from django.core.cache import cache
+from django_filters import rest_framework as filters
+from drf_yasg import openapi
+from drf_yasg.utils import swagger_auto_schema
+from rest_framework.exceptions import ValidationError
+from rest_framework.filters import OrderingFilter, SearchFilter
+from rest_framework.permissions import AllowAny, IsAuthenticated
+from rest_framework.response import Response
+from rest_framework.viewsets import ReadOnlyModelViewSet
+
+from organizations.api_enrichment import (
+    API_DATA_SOURCE_KEY_SET,
+    OrganizationApiEnrichmentService,
+    to_api_data_source,
+    to_internal_data_source,
+)
+from organizations.filters import OrganizationFilter
+from organizations.models import Organization
+from organizations.serializers import OrganizationSerializer
+
+ORGANIZATIONS_API_CACHE_TIMEOUT_SECONDS = 300
+ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations")
+ORGANIZATION_DATA_SOURCE_KEYS = ", ".join(sorted(API_DATA_SOURCE_KEY_SET))
+
+
+def _query_parameter(
+    name: str,
+    *,
+    description: str,
+    param_type: str = openapi.TYPE_STRING,
+    default: str | int | bool | None = None,
+    enum: list[str] | None = None,
+    format_: str | None = None,
+) -> openapi.Parameter:
+    return openapi.Parameter(
+        name=name,
+        in_=openapi.IN_QUERY,
+        type=param_type,
+        required=False,
+        description=description,
+        default=default,
+        enum=enum,
+        format=format_,
+    )
+
+
+ORGANIZATION_DATA_PARAMS = [
+    _query_parameter(
+        "data",
+        description=(
+            "Ограничить блок data одним или несколькими источниками. "
+            f"Допустимые значения: {ORGANIZATION_DATA_SOURCE_KEYS}. "
+            "Можно передать несколько параметров или CSV-строку."
+        ),
+    ),
+    _query_parameter(
+        "data_sources",
+        description=(
+            "Alias параметра data. Оставлен для явного указания набора источников."
+        ),
+    ),
+    _query_parameter(
+        "exclude_data",
+        description=(
+            "Исключить один или несколько источников из блока data. "
+            f"Допустимые значения: {ORGANIZATION_DATA_SOURCE_KEYS}."
+        ),
+    ),
+    _query_parameter(
+        "exclude_data_sources",
+        description=(
+            "Alias параметра exclude_data. Можно передать несколько значений "
+            "или CSV-строку."
+        ),
+    ),
+]
+ORGANIZATION_LIST_PARAMS = [
+    _query_parameter(
+        "page",
+        description="Номер страницы пагинации.",
+        param_type=openapi.TYPE_INTEGER,
+        default=1,
+    ),
+    _query_parameter(
+        "page_size",
+        description="Размер страницы. Максимум 100.",
+        param_type=openapi.TYPE_INTEGER,
+        default=20,
+    ),
+    _query_parameter(
+        "search",
+        description="Полнотекстовый поиск по наименованию, ИНН, КПП, ОГРН и ОГРИП.",
+    ),
+    _query_parameter(
+        "ordering",
+        description=(
+            "Сортировка по uid, name, inn, kpp, ogrn или ogrip. "
+            "Префикс '-' включает обратный порядок."
+        ),
+    ),
+    _query_parameter("name", description="Фильтр по части полного наименования."),
+    _query_parameter("inn", description="Точный фильтр по ИНН."),
+    _query_parameter("kpp", description="Точный фильтр по КПП."),
+    _query_parameter("ogrn", description="Точный фильтр по ОГРН."),
+    _query_parameter("ogrip", description="Точный фильтр по ОГРИП."),
+    _query_parameter(
+        "registry",
+        description="UUID реестра. Возвращает организации из активного участия.",
+        format_=openapi.FORMAT_UUID,
+    ),
+    _query_parameter(
+        "registry_name",
+        description="Фильтр по части наименования реестра.",
+    ),
+    _query_parameter(
+        "has_registry",
+        description=(
+            "Фильтр наличия активного участия в любом реестре; по умолчанию true "
+            "для list endpoint, если параметр не передан."
+        ),
+        param_type=openapi.TYPE_BOOLEAN,
+        default=True,
+    ),
+    *[
+        _query_parameter(
+            f"has_{source}",
+            description=f"Фильтр наличия данных источника {source}.",
+            param_type=openapi.TYPE_BOOLEAN,
+        )
+        for source in sorted(API_DATA_SOURCE_KEY_SET)
+    ],
+    *ORGANIZATION_DATA_PARAMS,
+]
+ORGANIZATION_DETAIL_PARAMS = [
+    openapi.Parameter(
+        name="uid",
+        in_=openapi.IN_PATH,
+        type=openapi.TYPE_STRING,
+        format=openapi.FORMAT_UUID,
+        required=True,
+        description="UID организации.",
+    ),
+    *ORGANIZATION_DATA_PARAMS,
+]
+ORGANIZATION_SCHEMA = openapi.Schema(
+    type=openapi.TYPE_OBJECT,
+    required=["uid", "name", "inn", "data", "data_sources", "registries"],
+    properties={
+        "uid": openapi.Schema(type=openapi.TYPE_STRING, format=openapi.FORMAT_UUID),
+        "name": openapi.Schema(type=openapi.TYPE_STRING),
+        "normalized_name": openapi.Schema(type=openapi.TYPE_STRING),
+        "inn": openapi.Schema(type=openapi.TYPE_STRING),
+        "kpp": openapi.Schema(type=openapi.TYPE_STRING),
+        "ogrn": openapi.Schema(type=openapi.TYPE_STRING),
+        "ogrip": openapi.Schema(type=openapi.TYPE_STRING),
+        "data": openapi.Schema(
+            type=openapi.TYPE_OBJECT,
+            description=(
+                "Данные по источникам. Ключи управляются параметрами data/"
+                "exclude_data."
+            ),
+            additional_properties=openapi.Schema(
+                type=openapi.TYPE_ARRAY,
+                items=openapi.Schema(type=openapi.TYPE_OBJECT),
+            ),
+        ),
+        "data_sources": openapi.Schema(
+            type=openapi.TYPE_ARRAY,
+            items=openapi.Schema(
+                type=openapi.TYPE_OBJECT,
+                properties={
+                    "source": openapi.Schema(type=openapi.TYPE_STRING),
+                    "count": openapi.Schema(type=openapi.TYPE_INTEGER),
+                },
+            ),
+        ),
+        "registries": openapi.Schema(
+            type=openapi.TYPE_ARRAY,
+            items=openapi.Schema(
+                type=openapi.TYPE_OBJECT,
+                properties={
+                    "id": openapi.Schema(type=openapi.TYPE_STRING),
+                    "name": openapi.Schema(type=openapi.TYPE_STRING),
+                },
+            ),
+        ),
+    },
+)
+ORGANIZATION_LIST_RESPONSE = openapi.Response(
+    description="Пагинированный список организаций v2.",
+    schema=openapi.Schema(
+        type=openapi.TYPE_OBJECT,
+        properties={
+            "success": openapi.Schema(type=openapi.TYPE_BOOLEAN),
+            "data": openapi.Schema(
+                type=openapi.TYPE_ARRAY,
+                items=ORGANIZATION_SCHEMA,
+            ),
+            "errors": openapi.Schema(
+                type=openapi.TYPE_ARRAY,
+                items=openapi.Schema(type=openapi.TYPE_OBJECT),
+                description="Список ошибок; null при успешном ответе.",
+            ),
+            "meta": openapi.Schema(
+                type=openapi.TYPE_OBJECT,
+                properties={
+                    "pagination": openapi.Schema(
+                        type=openapi.TYPE_OBJECT,
+                        properties={
+                            "page": openapi.Schema(type=openapi.TYPE_INTEGER),
+                            "page_size": openapi.Schema(type=openapi.TYPE_INTEGER),
+                            "total_count": openapi.Schema(type=openapi.TYPE_INTEGER),
+                            "total_pages": openapi.Schema(type=openapi.TYPE_INTEGER),
+                            "has_next": openapi.Schema(type=openapi.TYPE_BOOLEAN),
+                            "has_previous": openapi.Schema(type=openapi.TYPE_BOOLEAN),
+                        },
+                    ),
+                },
+            ),
+        },
+    ),
+)
+ORGANIZATION_DETAIL_RESPONSE = openapi.Response(
+    description="Карточка организации v2.",
+    schema=ORGANIZATION_SCHEMA,
+)
+
+
+class CachedReadOnlyMixin:
+    """Cache successful GET list/retrieve responses by full request path."""
+
+    cache_timeout = ORGANIZATIONS_API_CACHE_TIMEOUT_SECONDS
+    cache_key_prefix = "api:v2:organizations"
+
+    def _build_cache_key(self, request) -> str:
+        user_marker = "anonymous"
+        if request.user and request.user.is_authenticated:
+            user_marker = "authenticated"
+
+        raw_key = f"{request.method}:{request.get_full_path()}:{user_marker}"
+        digest = hashlib.md5(raw_key.encode(), usedforsecurity=False).hexdigest()
+        return f"{self.cache_key_prefix}:{digest}"
+
+    def _cached_response(self, request, producer) -> Response:
+        cache_key = self._build_cache_key(request)
+        cached_data = cache.get(cache_key)
+        if cached_data is not None:
+            response = Response(cached_data)
+            response["X-Cache"] = "HIT"
+            return response
+
+        response = producer()
+        if 200 <= response.status_code < 300:
+            cache.set(cache_key, response.data, timeout=self.cache_timeout)
+            response["X-Cache"] = "MISS"
+        return response
+
+
+class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet):
+    """Read-only API for canonical organizations."""
+
+    queryset = Organization.objects.select_related("data_snapshot").order_by(
+        "name", "uid"
+    )
+    serializer_class = OrganizationSerializer
+    permission_classes = [IsAuthenticated]
+    lookup_field = "uid"
+    filter_backends = [
+        filters.DjangoFilterBackend,
+        SearchFilter,
+        OrderingFilter,
+    ]
+    filterset_class = OrganizationFilter
+    search_fields = ["name", "inn", "kpp", "ogrn", "ogrip"]
+    ordering_fields = ["name", "inn", "kpp", "ogrn", "ogrip", "uid"]
+    ordering = ["name", "uid"]
+
+    def get_permissions(self):
+        if getattr(settings, "ORGANIZATIONS_V2_ALLOW_ANONYMOUS", False):
+            return [AllowAny()]
+        return super().get_permissions()
+
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.action != "list" or "has_registry" in self.request.query_params:
+            return queryset
+
+        filterset = OrganizationFilter(
+            data={"has_registry": "true"},
+            queryset=queryset,
+            request=self.request,
+        )
+        if filterset.is_valid():
+            return filterset.qs
+        return queryset
+
+    @swagger_auto_schema(
+        tags=[ORGANIZATIONS_TAG],
+        operation_id="v2_organizations_list",
+        operation_summary="Список организаций",
+        operation_description=(
+            "Возвращает канонический справочник организаций API v2. "
+            "По умолчанию показывает только организации с активным участием "
+            "в реестрах; передайте has_registry=false, чтобы снять это ограничение. "
+            "Поддерживает пагинацию, поиск по наименованию и реквизитам, фильтры "
+            "по реестрам и наличию данных по источникам."
+        ),
+        manual_parameters=ORGANIZATION_LIST_PARAMS,
+        responses={200: ORGANIZATION_LIST_RESPONSE},
+    )
+    def list(self, request, *args: Any, **kwargs: Any) -> Response:
+        return self._cached_response(
+            request,
+            lambda: self._list_with_enrichment(request, *args, **kwargs),
+        )
+
+    @swagger_auto_schema(
+        tags=[ORGANIZATIONS_TAG],
+        operation_id="v2_organizations_retrieve",
+        operation_summary="Карточка организации",
+        operation_description=(
+            "Возвращает одну организацию по UID с реестрами и данными источников. "
+            "Параметры data/data_sources и exclude_data/exclude_data_sources "
+            "позволяют запросить только нужные блоки данных."
+        ),
+        manual_parameters=ORGANIZATION_DETAIL_PARAMS,
+        responses={200: ORGANIZATION_DETAIL_RESPONSE, 404: "Организация не найдена"},
+    )
+    def retrieve(self, request, *args: Any, **kwargs: Any) -> Response:
+        return self._cached_response(
+            request,
+            lambda: self._retrieve_with_enrichment(request, *args, **kwargs),
+        )
+
+    def _list_with_enrichment(self, request, *args: Any, **kwargs: Any) -> Response:
+        queryset = self.filter_queryset(self.get_queryset())
+        data_sources = self._parse_data_sources(request)
+
+        page = self.paginate_queryset(queryset)
+        if page is not None:
+            organizations = list(page)
+            enrichment = self._build_missing_snapshot_enrichment(
+                organizations,
+                data_sources,
+            )
+            serializer = self.get_serializer(
+                organizations,
+                many=True,
+                context={
+                    **self.get_serializer_context(),
+                    "data_sources": data_sources,
+                    "enrichment": enrichment,
+                },
+            )
+            return self.get_paginated_response(serializer.data)
+
+        organizations = list(queryset)
+        enrichment = self._build_missing_snapshot_enrichment(
+            organizations,
+            data_sources,
+        )
+        serializer = self.get_serializer(
+            organizations,
+            many=True,
+            context={
+                **self.get_serializer_context(),
+                "data_sources": data_sources,
+                "enrichment": enrichment,
+            },
+        )
+        return Response(serializer.data)
+
+    def _retrieve_with_enrichment(
+        self,
+        request,
+        *args: Any,
+        **kwargs: Any,
+    ) -> Response:
+        organization = self.get_object()
+        data_sources = self._parse_data_sources(request)
+        enrichment = self._build_missing_snapshot_enrichment(
+            [organization],
+            data_sources,
+        )
+        serializer = self.get_serializer(
+            organization,
+            context={
+                **self.get_serializer_context(),
+                "data_sources": data_sources,
+                "enrichment": enrichment,
+            },
+        )
+        return Response(serializer.data)
+
+    @staticmethod
+    def _build_missing_snapshot_enrichment(
+        organizations: list[Organization],
+        data_sources: set[str] | None,
+    ) -> dict:
+        missing = [
+            organization
+            for organization in organizations
+            if not hasattr(organization, "data_snapshot")
+        ]
+        if not missing:
+            return {}
+        return OrganizationApiEnrichmentService.build_for(
+            missing,
+            data_sources=data_sources,
+        )
+
+    @staticmethod
+    def _parse_data_sources(request) -> set[str] | None:
+        included = _query_param_values(request, "data", "data_sources")
+        excluded = _query_param_values(request, "exclude_data", "exclude_data_sources")
+
+        unknown = (included | excluded) - API_DATA_SOURCE_KEY_SET
+        if unknown:
+            raise ValidationError(
+                {
+                    "data": (
+                        "Unknown data source(s): "
+                        + ", ".join(sorted(unknown))
+                        + ". Available sources: "
+                        + ", ".join(sorted(API_DATA_SOURCE_KEY_SET))
+                    )
+                }
+            )
+
+        if included:
+            return {
+                to_api_data_source(to_internal_data_source(source))
+                for source in included - excluded
+            }
+        if excluded:
+            return API_DATA_SOURCE_KEY_SET - excluded
+        return None
+
+
+def _query_param_values(request, *names: str) -> set[str]:
+    values: set[str] = set()
+    for name in names:
+        for raw_value in request.query_params.getlist(name):
+            values.update(
+                value.strip() for value in raw_value.split(",") if value.strip()
+            )
+    return values