avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(organizations): cache and instrument API responses

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-05-14 16:14:45 +02:00
parent 6d1ec2e55c
commit 5fdd23ecc0
13 changed files with 603 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
tests/apps/core/test_middleware.py
View File

@@ -1,18 +1,22 @@
"""Tests for core middleware"""
import json
import logging
from io import StringIO
from apps.core.middleware import (
ApiCsrfExemptMiddleware,
ApiSlashlessRouteMiddleware,
OrganizationApiMetricsMiddleware,
RequestIDMiddleware,
RequestLoggingMiddleware,
get_request_id,
)
from django.conf import settings
from django.http import HttpResponse
from django.test import RequestFactory
from django.urls import reverse
from organizations.models import Organization
from rest_framework.test import APITestCase
@@ -126,3 +130,70 @@ class ApiSlashlessRouteMiddlewareTest(APITestCase):
self.middleware.process_request(request)
self.assertEqual(request.path_info, "/admin/login")
class OrganizationApiMetricsMiddlewareTest(APITestCase):
def setUp(self):
self.factory = RequestFactory()
def test_middleware_is_enabled_for_organization_endpoint_metrics(self):
self.assertIn(
"apps.core.middleware.OrganizationApiMetricsMiddleware",
settings.MIDDLEWARE,
)
def test_logs_organization_endpoint_metrics_without_query_values(self):
Organization.objects.create(name='ООО "Метрика"', inn="7707083893")
def get_response(request):
Organization.objects.count()
response = HttpResponse("ok", status=200)
response["X-Cache"] = "MISS"
return response
middleware = OrganizationApiMetricsMiddleware(get_response)
request = self.factory.get(
"/api/v2/organizations/",
{"page": "1", "page_size": "20", "search": "7707083893"},
)
request.request_id = "metrics-request"
request.user = type("AnonymousUser", (), {"is_authenticated": False})()
with self.assertLogs("organizations.api.metrics", level="INFO") as captured:
response = middleware(request)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(captured.output), 1)
_, payload = captured.output[0].split("organization_api_metrics ", 1)
metrics = json.loads(payload)
self.assertEqual(metrics["request_id"], "metrics-request")
self.assertEqual(metrics["method"], "GET")
self.assertEqual(metrics["path"], "/api/v2/organizations/")
self.assertEqual(metrics["status_code"], 200)
self.assertEqual(metrics["cache"], "MISS")
self.assertEqual(metrics["query_keys"], ["page", "page_size", "search"])
self.assertGreaterEqual(metrics["db_query_count"], 1)
self.assertGreater(metrics["duration_ms"], 0)
self.assertGreater(metrics["response_size_bytes"], 0)
self.assertNotIn("7707083893", captured.output[0])
def test_ignores_non_organization_api_paths(self):
middleware = OrganizationApiMetricsMiddleware(
lambda request: HttpResponse("ok", status=200)
)
request = self.factory.get("/api/v2/sources/")
request.request_id = "metrics-request"
request.user = type("AnonymousUser", (), {"is_authenticated": False})()
logger = logging.getLogger("organizations.api.metrics")
stream = StringIO()
handler = logging.StreamHandler(stream)
logger.addHandler(handler)
try:
response = middleware(request)
finally:
logger.removeHandler(handler)
self.assertEqual(response.status_code, 200)
self.assertEqual(stream.getvalue(), "")

203
tests/apps/organizations/test_api_v2.py
View File

@@ -11,7 +11,8 @@ from django.db import connection
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from django.urls import reverse
from organizations.models import Organization
from organizations.cache import invalidate_organization_api_cache
from organizations.models import Organization, OrganizationDataSnapshot
from organizations.services import OrganizationDataSnapshotRefreshService
from rest_framework import status
from rest_framework.test import APITestCase
@@ -104,20 +105,23 @@ class OrganizationsApiV2Test(APITestCase):
"exclude_data",
):
self.assertIn(expected_name, list_parameters)
self.assertIn("по умолчанию true", list_parameters["has_registry"]["description"])
self.assertIn(
"по умолчанию true", list_parameters["has_registry"]["description"]
)
self.assertIn(
"industrial_products",
list_parameters["data"]["description"],
)
detail_parameters = {
parameter["name"]: parameter
for parameter in detail_operation["parameters"]
parameter["name"]: parameter for parameter in detail_operation["parameters"]
}
self.assertEqual(detail_parameters["uid"]["type"], "string")
self.assertEqual(detail_parameters["uid"]["format"], "uuid")
self.assertIn("data", detail_parameters)
self.assertIn("exclude_data", detail_parameters)
self.assertIn("Пагинированный", list_operation["responses"]["200"]["description"])
self.assertIn(
"Пагинированный", list_operation["responses"]["200"]["description"]
)
self.assertIn(
"Карточка организации",
detail_operation["responses"]["200"]["description"],
@@ -132,7 +136,9 @@ class OrganizationsApiV2Test(APITestCase):
)
response = self.client.get(
reverse("api_v2:organizations:organizations-detail", args=[organization.uid])
reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
)
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -148,13 +154,100 @@ class OrganizationsApiV2Test(APITestCase):
)
response = self.client.get(
reverse("api_v2:organizations:organizations-detail", args=[organization.uid])
reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
)
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["name"], 'АКЦИОНЕРНОЕ ОБЩЕСТВО "СЕВЕРНЫЙ МОСТ"')
self.assertEqual(response.data["normalized_name"], 'АО "Северный Мост"')
def test_list_omits_full_snapshot_data_by_default_but_keeps_summary(self):
organization = Organization.objects.create(
name='ООО "Легкий список"',
inn="7712345682",
kpp="771201005",
ogrn="1027700132200",
)
OrganizationDataSnapshot.objects.create(
organization=organization,
data={
"industrial": [{"id": 1}],
"fns_reports": [{"id": 2}, {"id": 3}],
},
registries=[],
)
response = self.client.get(
reverse("api_v2:organizations:organizations-list"),
{"inn": organization.inn, "has_registry": "false"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
item = response.data["data"][0]
self.assertEqual(item["data"], {})
self.assertEqual(
item["data_sources"],
[
{"source": "fns_reports", "count": 2},
{"source": "industrial", "count": 1},
],
)
def test_list_returns_snapshot_data_when_sources_are_requested(self):
organization = Organization.objects.create(
name='ООО "Явные данные"',
inn="7712345683",
kpp="771201006",
ogrn="1027700132201",
)
OrganizationDataSnapshot.objects.create(
organization=organization,
data={
"industrial": [{"id": 1}],
"fns_reports": [{"id": 2}],
},
registries=[],
)
response = self.client.get(
reverse("api_v2:organizations:organizations-list"),
{
"inn": organization.inn,
"has_registry": "false",
"data": "industrial",
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.data["data"][0]["data"],
{"industrial": [{"id": 1}]},
)
def test_detail_keeps_full_snapshot_data_by_default(self):
organization = Organization.objects.create(
name='ООО "Полная карточка"',
inn="7712345684",
kpp="771201007",
ogrn="1027700132202",
)
OrganizationDataSnapshot.objects.create(
organization=organization,
data={"industrial": [{"id": 1}]},
registries=[],
)
response = self.client.get(
reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
)
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["data"], {"industrial": [{"id": 1}]})
def test_list_keeps_data_source_summary_when_data_payload_is_excluded(self):
organization = Organization.objects.create(
name='ООО "Сводка данных"',
@@ -201,7 +294,9 @@ class OrganizationsApiV2Test(APITestCase):
)
response = self.client.get(
reverse("api_v2:organizations:organizations-detail", args=[organization.uid])
reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
)
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -299,6 +394,70 @@ class OrganizationsApiV2Test(APITestCase):
self.assertEqual(different_query_response["X-Cache"], "MISS")
self.assertEqual(different_query_response.data["data"][0]["inn"], "7744444444")
def test_list_response_cache_is_invalidated_by_version_bump(self):
Organization.objects.create(
name='ООО "Версия кеша"',
inn="7744444445",
kpp="774401002",
ogrn="1027700132445",
)
url = reverse("api_v2:organizations:organizations-list")
params = {"inn": "7744444445", "has_registry": "false"}
first_response = self.client.get(url, params)
second_response = self.client.get(url, params)
invalidate_organization_api_cache()
third_response = self.client.get(url, params)
self.assertEqual(first_response.status_code, status.HTTP_200_OK)
self.assertEqual(first_response["X-Cache"], "MISS")
self.assertEqual(second_response["X-Cache"], "HIT")
self.assertEqual(third_response["X-Cache"], "MISS")
def test_source_update_invalidates_organization_cache(self):
Organization.objects.create(
name='ООО "Источник сброса"',
inn="7744444446",
kpp="774401003",
ogrn="1027700132446",
)
url = reverse("api_v2:organizations:organizations-list")
params = {"inn": "7744444446", "has_registry": "false"}
first_response = self.client.get(url, params)
second_response = self.client.get(url, params)
with self.captureOnCommitCallbacks(execute=True):
ParserLoadLog.objects.create(
source=ParserLoadLog.Source.FSTEC,
batch_id=1,
status=ParserLoadLog.Status.SUCCESS,
)
third_response = self.client.get(url, params)
self.assertEqual(first_response["X-Cache"], "MISS")
self.assertEqual(second_response["X-Cache"], "HIT")
self.assertEqual(third_response["X-Cache"], "MISS")
def test_registry_update_invalidates_organization_cache(self):
Organization.objects.create(
name='ООО "Реестр сброса"',
inn="7744444447",
kpp="774401004",
ogrn="1027700132447",
)
url = reverse("api_v2:organizations:organizations-list")
params = {"inn": "7744444447", "has_registry": "false"}
first_response = self.client.get(url, params)
second_response = self.client.get(url, params)
with self.captureOnCommitCallbacks(execute=True):
RegisterFactory(name="Реестр для сброса кеша")
third_response = self.client.get(url, params)
self.assertEqual(first_response["X-Cache"], "MISS")
self.assertEqual(second_response["X-Cache"], "HIT")
self.assertEqual(third_response["X-Cache"], "MISS")
def test_retrieve_response_is_cached(self):
organization = Organization.objects.create(
name='ООО "Деталь"',
@@ -306,7 +465,9 @@ class OrganizationsApiV2Test(APITestCase):
kpp="775501001",
ogrn="1027700132555",
)
url = reverse("api_v2:organizations:organizations-detail", args=[organization.uid])
url = reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
)
first_response = self.client.get(url)
organization.name = 'ООО "Изменено"'
@@ -417,7 +578,13 @@ class OrganizationsApiV2Test(APITestCase):
response = self.client.get(
reverse("api_v2:organizations:organizations-list"),
{"inn": organization.inn},
{
"inn": organization.inn,
"data": (
"industrial,industrial_products,manufactures,inspections,"
"procurements,procurements_44fz,fstec,fns_reports"
),
},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -525,9 +692,7 @@ class OrganizationsApiV2Test(APITestCase):
"period_end": 500,
},
)
self.assertTrue(
all(isinstance(value, list) for value in item["data"].values())
)
self.assertTrue(all(isinstance(value, list) for value in item["data"].values()))
def test_filters_by_registry_and_has_registry(self):
with_registry = Organization.objects.create(
@@ -608,7 +773,9 @@ class OrganizationsApiV2Test(APITestCase):
self.assertEqual(default_response.status_code, status.HTTP_200_OK)
self.assertEqual(default_response.data["meta"]["pagination"]["total_count"], 1)
self.assertEqual(default_response.data["data"][0]["uid"], str(with_registry.uid))
self.assertEqual(
default_response.data["data"][0]["uid"], str(with_registry.uid)
)
self.assertEqual(
explicit_false_response.data["meta"]["pagination"]["total_count"],
@@ -764,7 +931,9 @@ class OrganizationsApiV2Test(APITestCase):
)
response = self.client.get(
reverse("api_v2:organizations:organizations-detail", args=[organization.uid]),
reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
),
{"data": "unknown"},
)
@@ -823,7 +992,9 @@ class OrganizationsApiV2Test(APITestCase):
OrganizationDataSnapshotRefreshService.refresh(
organization_uids=[str(organization.uid)],
)
url = reverse("api_v2:organizations:organizations-detail", args=[organization.uid])
url = reverse(
"api_v2:organizations:organizations-detail", args=[organization.uid]
)
with CaptureQueriesContext(connection) as captured:
response = self.client.get(
url,

12
tests/apps/organizations/test_tasks.py
View File

@@ -6,6 +6,7 @@ from django.apps import apps as django_apps
from django.core.cache import cache
from django.test import TestCase
from django_celery_beat.models import PeriodicTask
from organizations.cache import get_organization_api_cache_version
from organizations.models import Organization
from organizations.tasks import refresh_all_organization_data_snapshots
@@ -15,7 +16,7 @@ from tests.apps.parsers.factories import IndustrialCertificateRecordFactory
class OrganizationSnapshotTasksTest(TestCase):
"""Checks Celery tasks that maintain API v2 organization snapshots."""
def test_refresh_all_task_rebuilds_snapshots_and_clears_api_cache(self):
def test_refresh_all_task_rebuilds_snapshots_and_invalidates_api_cache(self):
organization = Organization.objects.create(
name='ООО "Снапшот"',
inn="7800000401",
@@ -26,14 +27,19 @@ class OrganizationSnapshotTasksTest(TestCase):
ogrn=organization.ogrn,
certificate_number="FULL-SNAPSHOT-CERT",
)
cache.set("api:v2:organizations:test", {"stale": True}, timeout=60)
cache.set("unrelated:test", {"keep": True}, timeout=60)
cache_version_before = get_organization_api_cache_version()
result = refresh_all_organization_data_snapshots(batch_size=10)
self.assertEqual(result["processed"], 1)
self.assertEqual(result["created"], 1)
self.assertEqual(result["updated"], 0)
self.assertIsNone(cache.get("api:v2:organizations:test"))
self.assertNotEqual(
get_organization_api_cache_version(),
cache_version_before,
)
self.assertEqual(cache.get("unrelated:test"), {"keep": True})
snapshot = organization.data_snapshot
self.assertEqual(
snapshot.data["industrial"][0]["certificate_number"],