diff --git a/src/apps/core/middleware.py b/src/apps/core/middleware.py index 44c4613..07e6e47 100644 --- a/src/apps/core/middleware.py +++ b/src/apps/core/middleware.py @@ -4,15 +4,20 @@ Core middleware components. Provides Request ID tracking and other cross-cutting concerns. """ +import json import logging import threading +import time import uuid +from django.db import connection from django.middleware.csrf import CsrfViewMiddleware +from django.template.response import ContentNotRenderedError from django.urls import Resolver404, resolve from django.utils.deprecation import MiddlewareMixin logger = logging.getLogger(__name__) +organization_metrics_logger = logging.getLogger("organizations.api.metrics") # Thread-local storage for request context _request_context = threading.local() @@ -170,3 +175,102 @@ class RequestLoggingMiddleware(MiddlewareMixin): }, ) return response + + +class OrganizationApiMetricsMiddleware: + """Emit focused request metrics for API endpoints that serve organizations.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if not self._should_log(request): + return self.get_response(request) + + db_query_count = 0 + db_duration_seconds = 0.0 + started_at = time.perf_counter() + response = None + error = None + + def execute_with_metrics(execute, sql, params, many, context): + nonlocal db_query_count, db_duration_seconds + + db_query_count += 1 + query_started_at = time.perf_counter() + try: + return execute(sql, params, many, context) + finally: + db_duration_seconds += time.perf_counter() - query_started_at + + try: + with connection.execute_wrapper(execute_with_metrics): + response = self.get_response(request) + return response + except Exception as exc: + error = exc.__class__.__name__ + raise + finally: + duration_seconds = time.perf_counter() - started_at + self._log_metrics( + request=request, + response=response, + error=error, + duration_seconds=duration_seconds, + db_query_count=db_query_count, + db_duration_seconds=db_duration_seconds, + ) + + @staticmethod + def _should_log(request) -> bool: + path = getattr(request, "path_info", "") or getattr(request, "path", "") + if not path.startswith("/api/"): + return False + return "organizations" in path.strip("/").split("/") + + @classmethod + def _log_metrics( + cls, + *, + request, + response, + error: str | None, + duration_seconds: float, + db_query_count: int, + db_duration_seconds: float, + ) -> None: + metrics = { + "request_id": getattr(request, "request_id", None), + "method": request.method, + "path": getattr(request, "path_info", "") or request.path, + "status_code": getattr(response, "status_code", 500), + "duration_ms": round(duration_seconds * 1000, 2), + "db_query_count": db_query_count, + "db_duration_ms": round(db_duration_seconds * 1000, 2), + "response_size_bytes": cls._response_size(response), + "cache": response.get("X-Cache") if response is not None else None, + "user_id": cls._user_id(request), + "query_keys": sorted(request.GET.keys()), + "error": error, + } + organization_metrics_logger.info( + "organization_api_metrics %s", + json.dumps(metrics, ensure_ascii=False, sort_keys=True), + extra={"organization_api_metrics": metrics}, + ) + + @staticmethod + def _response_size(response) -> int | None: + if response is None or getattr(response, "streaming", False): + return None + try: + return len(response.content) + except (AttributeError, ContentNotRenderedError): + return None + + @staticmethod + def _user_id(request) -> int | None: + user = getattr(request, "user", None) + if user is None or not getattr(user, "is_authenticated", False): + return None + return getattr(user, "id", None) diff --git a/src/organizations/apps.py b/src/organizations/apps.py index 08c6a8d..9bbb6c3 100644 --- a/src/organizations/apps.py +++ b/src/organizations/apps.py @@ -7,3 +7,6 @@ class OrganizationsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "organizations" verbose_name = "Организации" + + def ready(self) -> None: + import organizations.signals # noqa: F401 diff --git a/src/organizations/cache.py b/src/organizations/cache.py new file mode 100644 index 0000000..5023bb9 --- /dev/null +++ b/src/organizations/cache.py @@ -0,0 +1,59 @@ +"""Cache helpers for organizations API responses.""" + +from __future__ import annotations + +import time + +from django.core.cache import cache + +ORGANIZATION_API_CACHE_PREFIX = "api:v2:organizations" +ORGANIZATION_API_CACHE_VERSION_KEY = f"{ORGANIZATION_API_CACHE_PREFIX}:version" +DEFAULT_ORGANIZATION_API_CACHE_VERSION = 1 +DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS = 24 * 60 * 60 + + +def get_organization_api_cache_version() -> int: + """Return the current organizations API cache version.""" + version = cache.get(ORGANIZATION_API_CACHE_VERSION_KEY) + if version is None: + cache.add( + ORGANIZATION_API_CACHE_VERSION_KEY, + DEFAULT_ORGANIZATION_API_CACHE_VERSION, + timeout=None, + ) + version = cache.get( + ORGANIZATION_API_CACHE_VERSION_KEY, + DEFAULT_ORGANIZATION_API_CACHE_VERSION, + ) + + try: + return int(version) + except (TypeError, ValueError): + cache.set( + ORGANIZATION_API_CACHE_VERSION_KEY, + DEFAULT_ORGANIZATION_API_CACHE_VERSION, + timeout=None, + ) + return DEFAULT_ORGANIZATION_API_CACHE_VERSION + + +def invalidate_organization_api_cache() -> int: + """ + Invalidate organizations API responses by moving them to a new key version. + + Versioning avoids broad cache scans and works with cache backends that do not + support pattern deletion. + """ + new_version = _new_cache_version() + if cache.add(ORGANIZATION_API_CACHE_VERSION_KEY, new_version, timeout=None): + return new_version + + try: + return int(cache.incr(ORGANIZATION_API_CACHE_VERSION_KEY)) + except (TypeError, ValueError, NotImplementedError): + cache.set(ORGANIZATION_API_CACHE_VERSION_KEY, new_version, timeout=None) + return new_version + + +def _new_cache_version() -> int: + return time.time_ns() diff --git a/src/organizations/management/commands/refresh_organization_data_snapshots.py b/src/organizations/management/commands/refresh_organization_data_snapshots.py index aee0278..d16ccaa 100644 --- a/src/organizations/management/commands/refresh_organization_data_snapshots.py +++ b/src/organizations/management/commands/refresh_organization_data_snapshots.py @@ -6,6 +6,7 @@ import json from apps.core.management.commands.base import BaseAppCommand +from organizations.cache import invalidate_organization_api_cache from organizations.services import OrganizationDataSnapshotRefreshService @@ -36,6 +37,7 @@ class Command(BaseAppCommand): organization_uids=options.get("uids"), batch_size=options["batch_size"], ) + invalidate_organization_api_cache() rendered = json.dumps( { "processed": result.processed, diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index 82f96a2..9150bab 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -49,6 +49,13 @@ class OrganizationSerializer(serializers.ModelSerializer): enrichment = self.context.get("enrichment", {}).get(str(obj.uid)) if enrichment is None: return {} + data_sources = self.context.get("data_sources") + if data_sources is not None: + return { + source: enrichment.data_presence.get(source, []) + for source in data_sources + if source in enrichment.data_presence + } return enrichment.data_presence def get_data_sources(self, obj) -> list[dict[str, int | str]]: diff --git a/src/organizations/signals.py b/src/organizations/signals.py new file mode 100644 index 0000000..ebd7d42 --- /dev/null +++ b/src/organizations/signals.py @@ -0,0 +1,89 @@ +"""Invalidation hooks for organizations API cache.""" + +from __future__ import annotations + +from apps.parsers.models import ParserLoadLog +from django.db import transaction +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver +from registers.models import ( + Organization as RegistryOrganization, +) +from registers.models import ( + Register, + RegisterUpload, + RegistryMembershipPeriod, +) + +from organizations.cache import invalidate_organization_api_cache +from organizations.models import OrganizationDataSnapshot + +SOURCE_UPDATE_STATUSES = { + ParserLoadLog.Status.SUCCESS, + ParserLoadLog.Status.SKIPPED, +} + + +def _invalidate_on_commit() -> None: + transaction.on_commit(invalidate_organization_api_cache) + + +@receiver( + post_save, sender=ParserLoadLog, dispatch_uid="organizations_parser_load_save" +) +def invalidate_for_parser_load(sender, instance: ParserLoadLog, **kwargs) -> None: + """Invalidate when a parser source reaches a visible terminal state.""" + if instance.status in SOURCE_UPDATE_STATUSES: + _invalidate_on_commit() + + +@receiver(post_save, sender=Register, dispatch_uid="organizations_register_save") +@receiver(post_delete, sender=Register, dispatch_uid="organizations_register_delete") +@receiver( + post_save, + sender=RegistryOrganization, + dispatch_uid="organizations_registry_organization_save", +) +@receiver( + post_delete, + sender=RegistryOrganization, + dispatch_uid="organizations_registry_organization_delete", +) +@receiver( + post_save, + sender=RegistryMembershipPeriod, + dispatch_uid="organizations_registry_membership_save", +) +@receiver( + post_delete, + sender=RegistryMembershipPeriod, + dispatch_uid="organizations_registry_membership_delete", +) +@receiver( + post_save, + sender=OrganizationDataSnapshot, + dispatch_uid="organizations_data_snapshot_save", +) +@receiver( + post_delete, + sender=OrganizationDataSnapshot, + dispatch_uid="organizations_data_snapshot_delete", +) +def invalidate_for_registry_or_snapshot_change(sender, **kwargs) -> None: + """Invalidate for direct registry and snapshot writes.""" + _invalidate_on_commit() + + +@receiver( + post_save, sender=RegisterUpload, dispatch_uid="organizations_register_upload_save" +) +def invalidate_for_successful_register_upload( + sender, + instance: RegisterUpload, + created: bool, + **kwargs, +) -> None: + """Invalidate once a registry import has completed successfully.""" + if created or instance.import_status != RegisterUpload.ImportStatus.SUCCESS: + return + _invalidate_on_commit() diff --git a/src/organizations/tasks.py b/src/organizations/tasks.py index a9c12c8..dee8f5d 100644 --- a/src/organizations/tasks.py +++ b/src/organizations/tasks.py @@ -5,8 +5,8 @@ from __future__ import annotations import logging from celery import shared_task -from django.core.cache import cache +from organizations.cache import invalidate_organization_api_cache from organizations.services import OrganizationDataSnapshotRefreshService logger = logging.getLogger(__name__) @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) def refresh_all_organization_data_snapshots(batch_size: int = 100) -> dict: """Refresh all organization data snapshots for API v2.""" result = OrganizationDataSnapshotRefreshService.refresh(batch_size=batch_size) - cache.clear() + invalidate_organization_api_cache() payload = { "processed": result.processed, "created": result.created, @@ -39,7 +39,7 @@ def refresh_organization_data_snapshots_for_parser_batch( batch_id=batch_id, batch_size=batch_size, ) - cache.clear() + invalidate_organization_api_cache() payload = { "source": source, "batch_id": batch_id, diff --git a/src/organizations/views.py b/src/organizations/views.py index e391af0..5adcb11 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -23,11 +23,15 @@ from organizations.api_enrichment import ( to_api_data_source, to_internal_data_source, ) +from organizations.cache import ( + DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS, + ORGANIZATION_API_CACHE_PREFIX, + get_organization_api_cache_version, +) from organizations.filters import OrganizationFilter from organizations.models import Organization from organizations.serializers import OrganizationSerializer -ORGANIZATIONS_API_CACHE_TIMEOUT_SECONDS = 300 ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations") ORGANIZATION_DATA_SOURCE_KEYS = ", ".join(sorted(API_DATA_SOURCE_KEY_SET)) @@ -59,7 +63,9 @@ ORGANIZATION_DATA_PARAMS = [ description=( "Ограничить блок data одним или несколькими источниками. " f"Допустимые значения: {ORGANIZATION_DATA_SOURCE_KEYS}. " - "Можно передать несколько параметров или CSV-строку." + "Можно передать несколько параметров или CSV-строку. " + "На list endpoint блок data по умолчанию пустой; передайте этот " + "параметр, чтобы вернуть данные источников." ), ), _query_parameter( @@ -238,17 +244,28 @@ ORGANIZATION_DETAIL_RESPONSE = openapi.Response( class CachedReadOnlyMixin: """Cache successful GET list/retrieve responses by full request path.""" - cache_timeout = ORGANIZATIONS_API_CACHE_TIMEOUT_SECONDS - cache_key_prefix = "api:v2:organizations" + cache_timeout = DEFAULT_ORGANIZATION_API_CACHE_TIMEOUT_SECONDS + cache_key_prefix = ORGANIZATION_API_CACHE_PREFIX + + def _cache_timeout(self) -> int: + return getattr( + settings, + "ORGANIZATIONS_API_CACHE_TIMEOUT_SECONDS", + self.cache_timeout, + ) def _build_cache_key(self, request) -> str: user_marker = "anonymous" if request.user and request.user.is_authenticated: user_marker = "authenticated" - raw_key = f"{request.method}:{request.get_full_path()}:{user_marker}" + cache_version = get_organization_api_cache_version() + raw_key = ( + f"v{cache_version}:{request.method}:" + f"{request.get_full_path()}:{user_marker}" + ) digest = hashlib.md5(raw_key.encode(), usedforsecurity=False).hexdigest() - return f"{self.cache_key_prefix}:{digest}" + return f"{self.cache_key_prefix}:v{cache_version}:{digest}" def _cached_response(self, request, producer) -> Response: cache_key = self._build_cache_key(request) @@ -260,7 +277,7 @@ class CachedReadOnlyMixin: response = producer() if 200 <= response.status_code < 300: - cache.set(cache_key, response.data, timeout=self.cache_timeout) + cache.set(cache_key, response.data, timeout=self._cache_timeout()) response["X-Cache"] = "MISS" return response @@ -312,7 +329,9 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): "По умолчанию показывает только организации с активным участием " "в реестрах; передайте has_registry=false, чтобы снять это ограничение. " "Поддерживает пагинацию, поиск по наименованию и реквизитам, фильтры " - "по реестрам и наличию данных по источникам." + "по реестрам и наличию данных по источникам. Для list endpoint " + "тяжелый блок data по умолчанию пустой; передайте data/data_sources, " + "чтобы вернуть данные конкретных источников." ), manual_parameters=ORGANIZATION_LIST_PARAMS, responses={200: ORGANIZATION_LIST_RESPONSE}, @@ -343,7 +362,7 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): def _list_with_enrichment(self, request, *args: Any, **kwargs: Any) -> Response: queryset = self.filter_queryset(self.get_queryset()) - data_sources = self._parse_data_sources(request) + data_sources = self._parse_data_sources(request, default=set()) page = self.paginate_queryset(queryset) if page is not None: @@ -419,7 +438,11 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): ) @staticmethod - def _parse_data_sources(request) -> set[str] | None: + def _parse_data_sources( + request, + *, + default: set[str] | None = None, + ) -> set[str] | None: included = _query_param_values(request, "data", "data_sources") excluded = _query_param_values(request, "exclude_data", "exclude_data_sources") @@ -443,7 +466,7 @@ class OrganizationViewSet(CachedReadOnlyMixin, ReadOnlyModelViewSet): } if excluded: return API_DATA_SOURCE_KEY_SET - excluded - return None + return default def _query_param_values(request, *names: str) -> set[str]: diff --git a/src/settings/base.py b/src/settings/base.py index c6e17cd..ebd615f 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -171,6 +171,7 @@ MIDDLEWARE = [ "django.middleware.common.CommonMiddleware", "apps.core.middleware.ApiCsrfExemptMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", + "apps.core.middleware.OrganizationApiMetricsMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", ] @@ -381,6 +382,10 @@ LOGGING = { "format": "{levelname} {message}", "style": "{", }, + "metrics": { + "format": "{message}", + "style": "{", + }, }, "handlers": { "file": { @@ -389,6 +394,14 @@ LOGGING = { "filename": PROJECT_ROOT / "logs/django.log", "formatter": "verbose", }, + "organizations_api_metrics_file": { + "level": "INFO", + "class": "logging.handlers.RotatingFileHandler", + "filename": PROJECT_ROOT / "logs/organizations_api_metrics.log", + "maxBytes": 20 * 1024 * 1024, + "backupCount": 5, + "formatter": "metrics", + }, "console": { "level": "INFO", "class": "logging.StreamHandler", @@ -405,6 +418,11 @@ LOGGING = { "level": "INFO", "propagate": False, }, + "organizations.api.metrics": { + "handlers": ["organizations_api_metrics_file"], + "level": "INFO", + "propagate": False, + }, }, } diff --git a/src/settings/production.py b/src/settings/production.py index 1f4c842..03e8f82 100644 --- a/src/settings/production.py +++ b/src/settings/production.py @@ -95,6 +95,10 @@ LOGGING = { "format": "{levelname} {asctime} {module} {process:d} {thread:d} {message}", "style": "{", }, + "metrics": { + "format": "{message}", + "style": "{", + }, }, "handlers": { "console": { @@ -102,6 +106,14 @@ LOGGING = { "class": "logging.StreamHandler", "formatter": "verbose", }, + "organizations_api_metrics_file": { + "level": "INFO", + "class": "logging.handlers.RotatingFileHandler", + "filename": PROJECT_ROOT / "logs/organizations_api_metrics.log", + "maxBytes": 20 * 1024 * 1024, + "backupCount": 5, + "formatter": "metrics", + }, }, "root": { "handlers": ["console"], @@ -113,5 +125,10 @@ LOGGING = { "level": "INFO", "propagate": False, }, + "organizations.api.metrics": { + "handlers": ["organizations_api_metrics_file"], + "level": "INFO", + "propagate": False, + }, }, } diff --git a/tests/apps/core/test_middleware.py b/tests/apps/core/test_middleware.py index 3c2ccf9..b712db0 100644 --- a/tests/apps/core/test_middleware.py +++ b/tests/apps/core/test_middleware.py @@ -1,18 +1,22 @@ """Tests for core middleware""" +import json import logging from io import StringIO from apps.core.middleware import ( ApiCsrfExemptMiddleware, ApiSlashlessRouteMiddleware, + OrganizationApiMetricsMiddleware, RequestIDMiddleware, RequestLoggingMiddleware, get_request_id, ) +from django.conf import settings from django.http import HttpResponse from django.test import RequestFactory from django.urls import reverse +from organizations.models import Organization from rest_framework.test import APITestCase @@ -126,3 +130,70 @@ class ApiSlashlessRouteMiddlewareTest(APITestCase): self.middleware.process_request(request) self.assertEqual(request.path_info, "/admin/login") + + +class OrganizationApiMetricsMiddlewareTest(APITestCase): + def setUp(self): + self.factory = RequestFactory() + + def test_middleware_is_enabled_for_organization_endpoint_metrics(self): + self.assertIn( + "apps.core.middleware.OrganizationApiMetricsMiddleware", + settings.MIDDLEWARE, + ) + + def test_logs_organization_endpoint_metrics_without_query_values(self): + Organization.objects.create(name='ООО "Метрика"', inn="7707083893") + + def get_response(request): + Organization.objects.count() + response = HttpResponse("ok", status=200) + response["X-Cache"] = "MISS" + return response + + middleware = OrganizationApiMetricsMiddleware(get_response) + request = self.factory.get( + "/api/v2/organizations/", + {"page": "1", "page_size": "20", "search": "7707083893"}, + ) + request.request_id = "metrics-request" + request.user = type("AnonymousUser", (), {"is_authenticated": False})() + + with self.assertLogs("organizations.api.metrics", level="INFO") as captured: + response = middleware(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(captured.output), 1) + _, payload = captured.output[0].split("organization_api_metrics ", 1) + metrics = json.loads(payload) + + self.assertEqual(metrics["request_id"], "metrics-request") + self.assertEqual(metrics["method"], "GET") + self.assertEqual(metrics["path"], "/api/v2/organizations/") + self.assertEqual(metrics["status_code"], 200) + self.assertEqual(metrics["cache"], "MISS") + self.assertEqual(metrics["query_keys"], ["page", "page_size", "search"]) + self.assertGreaterEqual(metrics["db_query_count"], 1) + self.assertGreater(metrics["duration_ms"], 0) + self.assertGreater(metrics["response_size_bytes"], 0) + self.assertNotIn("7707083893", captured.output[0]) + + def test_ignores_non_organization_api_paths(self): + middleware = OrganizationApiMetricsMiddleware( + lambda request: HttpResponse("ok", status=200) + ) + request = self.factory.get("/api/v2/sources/") + request.request_id = "metrics-request" + request.user = type("AnonymousUser", (), {"is_authenticated": False})() + + logger = logging.getLogger("organizations.api.metrics") + stream = StringIO() + handler = logging.StreamHandler(stream) + logger.addHandler(handler) + try: + response = middleware(request) + finally: + logger.removeHandler(handler) + + self.assertEqual(response.status_code, 200) + self.assertEqual(stream.getvalue(), "") diff --git a/tests/apps/organizations/test_api_v2.py b/tests/apps/organizations/test_api_v2.py index 8df7d7e..c620766 100644 --- a/tests/apps/organizations/test_api_v2.py +++ b/tests/apps/organizations/test_api_v2.py @@ -11,7 +11,8 @@ from django.db import connection from django.test import override_settings from django.test.utils import CaptureQueriesContext from django.urls import reverse -from organizations.models import Organization +from organizations.cache import invalidate_organization_api_cache +from organizations.models import Organization, OrganizationDataSnapshot from organizations.services import OrganizationDataSnapshotRefreshService from rest_framework import status from rest_framework.test import APITestCase @@ -104,20 +105,23 @@ class OrganizationsApiV2Test(APITestCase): "exclude_data", ): self.assertIn(expected_name, list_parameters) - self.assertIn("по умолчанию true", list_parameters["has_registry"]["description"]) + self.assertIn( + "по умолчанию true", list_parameters["has_registry"]["description"] + ) self.assertIn( "industrial_products", list_parameters["data"]["description"], ) detail_parameters = { - parameter["name"]: parameter - for parameter in detail_operation["parameters"] + parameter["name"]: parameter for parameter in detail_operation["parameters"] } self.assertEqual(detail_parameters["uid"]["type"], "string") self.assertEqual(detail_parameters["uid"]["format"], "uuid") self.assertIn("data", detail_parameters) self.assertIn("exclude_data", detail_parameters) - self.assertIn("Пагинированный", list_operation["responses"]["200"]["description"]) + self.assertIn( + "Пагинированный", list_operation["responses"]["200"]["description"] + ) self.assertIn( "Карточка организации", detail_operation["responses"]["200"]["description"], @@ -132,7 +136,9 @@ class OrganizationsApiV2Test(APITestCase): ) response = self.client.get( - reverse("api_v2:organizations:organizations-detail", args=[organization.uid]) + reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ) ) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -148,13 +154,100 @@ class OrganizationsApiV2Test(APITestCase): ) response = self.client.get( - reverse("api_v2:organizations:organizations-detail", args=[organization.uid]) + reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ) ) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["name"], 'АКЦИОНЕРНОЕ ОБЩЕСТВО "СЕВЕРНЫЙ МОСТ"') self.assertEqual(response.data["normalized_name"], 'АО "Северный Мост"') + def test_list_omits_full_snapshot_data_by_default_but_keeps_summary(self): + organization = Organization.objects.create( + name='ООО "Легкий список"', + inn="7712345682", + kpp="771201005", + ogrn="1027700132200", + ) + OrganizationDataSnapshot.objects.create( + organization=organization, + data={ + "industrial": [{"id": 1}], + "fns_reports": [{"id": 2}, {"id": 3}], + }, + registries=[], + ) + + response = self.client.get( + reverse("api_v2:organizations:organizations-list"), + {"inn": organization.inn, "has_registry": "false"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + item = response.data["data"][0] + self.assertEqual(item["data"], {}) + self.assertEqual( + item["data_sources"], + [ + {"source": "fns_reports", "count": 2}, + {"source": "industrial", "count": 1}, + ], + ) + + def test_list_returns_snapshot_data_when_sources_are_requested(self): + organization = Organization.objects.create( + name='ООО "Явные данные"', + inn="7712345683", + kpp="771201006", + ogrn="1027700132201", + ) + OrganizationDataSnapshot.objects.create( + organization=organization, + data={ + "industrial": [{"id": 1}], + "fns_reports": [{"id": 2}], + }, + registries=[], + ) + + response = self.client.get( + reverse("api_v2:organizations:organizations-list"), + { + "inn": organization.inn, + "has_registry": "false", + "data": "industrial", + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.data["data"][0]["data"], + {"industrial": [{"id": 1}]}, + ) + + def test_detail_keeps_full_snapshot_data_by_default(self): + organization = Organization.objects.create( + name='ООО "Полная карточка"', + inn="7712345684", + kpp="771201007", + ogrn="1027700132202", + ) + OrganizationDataSnapshot.objects.create( + organization=organization, + data={"industrial": [{"id": 1}]}, + registries=[], + ) + + response = self.client.get( + reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ) + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["data"], {"industrial": [{"id": 1}]}) + def test_list_keeps_data_source_summary_when_data_payload_is_excluded(self): organization = Organization.objects.create( name='ООО "Сводка данных"', @@ -201,7 +294,9 @@ class OrganizationsApiV2Test(APITestCase): ) response = self.client.get( - reverse("api_v2:organizations:organizations-detail", args=[organization.uid]) + reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ) ) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -299,6 +394,70 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(different_query_response["X-Cache"], "MISS") self.assertEqual(different_query_response.data["data"][0]["inn"], "7744444444") + def test_list_response_cache_is_invalidated_by_version_bump(self): + Organization.objects.create( + name='ООО "Версия кеша"', + inn="7744444445", + kpp="774401002", + ogrn="1027700132445", + ) + url = reverse("api_v2:organizations:organizations-list") + params = {"inn": "7744444445", "has_registry": "false"} + + first_response = self.client.get(url, params) + second_response = self.client.get(url, params) + invalidate_organization_api_cache() + third_response = self.client.get(url, params) + + self.assertEqual(first_response.status_code, status.HTTP_200_OK) + self.assertEqual(first_response["X-Cache"], "MISS") + self.assertEqual(second_response["X-Cache"], "HIT") + self.assertEqual(third_response["X-Cache"], "MISS") + + def test_source_update_invalidates_organization_cache(self): + Organization.objects.create( + name='ООО "Источник сброса"', + inn="7744444446", + kpp="774401003", + ogrn="1027700132446", + ) + url = reverse("api_v2:organizations:organizations-list") + params = {"inn": "7744444446", "has_registry": "false"} + + first_response = self.client.get(url, params) + second_response = self.client.get(url, params) + with self.captureOnCommitCallbacks(execute=True): + ParserLoadLog.objects.create( + source=ParserLoadLog.Source.FSTEC, + batch_id=1, + status=ParserLoadLog.Status.SUCCESS, + ) + third_response = self.client.get(url, params) + + self.assertEqual(first_response["X-Cache"], "MISS") + self.assertEqual(second_response["X-Cache"], "HIT") + self.assertEqual(third_response["X-Cache"], "MISS") + + def test_registry_update_invalidates_organization_cache(self): + Organization.objects.create( + name='ООО "Реестр сброса"', + inn="7744444447", + kpp="774401004", + ogrn="1027700132447", + ) + url = reverse("api_v2:organizations:organizations-list") + params = {"inn": "7744444447", "has_registry": "false"} + + first_response = self.client.get(url, params) + second_response = self.client.get(url, params) + with self.captureOnCommitCallbacks(execute=True): + RegisterFactory(name="Реестр для сброса кеша") + third_response = self.client.get(url, params) + + self.assertEqual(first_response["X-Cache"], "MISS") + self.assertEqual(second_response["X-Cache"], "HIT") + self.assertEqual(third_response["X-Cache"], "MISS") + def test_retrieve_response_is_cached(self): organization = Organization.objects.create( name='ООО "Деталь"', @@ -306,7 +465,9 @@ class OrganizationsApiV2Test(APITestCase): kpp="775501001", ogrn="1027700132555", ) - url = reverse("api_v2:organizations:organizations-detail", args=[organization.uid]) + url = reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ) first_response = self.client.get(url) organization.name = 'ООО "Изменено"' @@ -417,7 +578,13 @@ class OrganizationsApiV2Test(APITestCase): response = self.client.get( reverse("api_v2:organizations:organizations-list"), - {"inn": organization.inn}, + { + "inn": organization.inn, + "data": ( + "industrial,industrial_products,manufactures,inspections," + "procurements,procurements_44fz,fstec,fns_reports" + ), + }, ) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -525,9 +692,7 @@ class OrganizationsApiV2Test(APITestCase): "period_end": 500, }, ) - self.assertTrue( - all(isinstance(value, list) for value in item["data"].values()) - ) + self.assertTrue(all(isinstance(value, list) for value in item["data"].values())) def test_filters_by_registry_and_has_registry(self): with_registry = Organization.objects.create( @@ -608,7 +773,9 @@ class OrganizationsApiV2Test(APITestCase): self.assertEqual(default_response.status_code, status.HTTP_200_OK) self.assertEqual(default_response.data["meta"]["pagination"]["total_count"], 1) - self.assertEqual(default_response.data["data"][0]["uid"], str(with_registry.uid)) + self.assertEqual( + default_response.data["data"][0]["uid"], str(with_registry.uid) + ) self.assertEqual( explicit_false_response.data["meta"]["pagination"]["total_count"], @@ -764,7 +931,9 @@ class OrganizationsApiV2Test(APITestCase): ) response = self.client.get( - reverse("api_v2:organizations:organizations-detail", args=[organization.uid]), + reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ), {"data": "unknown"}, ) @@ -823,7 +992,9 @@ class OrganizationsApiV2Test(APITestCase): OrganizationDataSnapshotRefreshService.refresh( organization_uids=[str(organization.uid)], ) - url = reverse("api_v2:organizations:organizations-detail", args=[organization.uid]) + url = reverse( + "api_v2:organizations:organizations-detail", args=[organization.uid] + ) with CaptureQueriesContext(connection) as captured: response = self.client.get( url, diff --git a/tests/apps/organizations/test_tasks.py b/tests/apps/organizations/test_tasks.py index 0183bd0..a60fe81 100644 --- a/tests/apps/organizations/test_tasks.py +++ b/tests/apps/organizations/test_tasks.py @@ -6,6 +6,7 @@ from django.apps import apps as django_apps from django.core.cache import cache from django.test import TestCase from django_celery_beat.models import PeriodicTask +from organizations.cache import get_organization_api_cache_version from organizations.models import Organization from organizations.tasks import refresh_all_organization_data_snapshots @@ -15,7 +16,7 @@ from tests.apps.parsers.factories import IndustrialCertificateRecordFactory class OrganizationSnapshotTasksTest(TestCase): """Checks Celery tasks that maintain API v2 organization snapshots.""" - def test_refresh_all_task_rebuilds_snapshots_and_clears_api_cache(self): + def test_refresh_all_task_rebuilds_snapshots_and_invalidates_api_cache(self): organization = Organization.objects.create( name='ООО "Снапшот"', inn="7800000401", @@ -26,14 +27,19 @@ class OrganizationSnapshotTasksTest(TestCase): ogrn=organization.ogrn, certificate_number="FULL-SNAPSHOT-CERT", ) - cache.set("api:v2:organizations:test", {"stale": True}, timeout=60) + cache.set("unrelated:test", {"keep": True}, timeout=60) + cache_version_before = get_organization_api_cache_version() result = refresh_all_organization_data_snapshots(batch_size=10) self.assertEqual(result["processed"], 1) self.assertEqual(result["created"], 1) self.assertEqual(result["updated"], 0) - self.assertIsNone(cache.get("api:v2:organizations:test")) + self.assertNotEqual( + get_organization_api_cache_version(), + cache_version_before, + ) + self.assertEqual(cache.get("unrelated:test"), {"keep": True}) snapshot = organization.data_snapshot self.assertEqual( snapshot.data["industrial"][0]["certificate_number"],