fix: exclude infrastructure probes from API throttling #41

Merged
avm merged 1 commits from codex/health-probes-main into main 2026-09-16 10:49:16 +03:00
7 changed files with 169 additions and 210 deletions

View File

@@ -45,14 +45,10 @@ jobs:
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Free Docker space
- name: Inspect Docker space
run: |
set -euo pipefail
docker system df || true
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker system prune --all --force --volumes || true
docker system df || true
- name: Run quality in golden image
env:
@@ -104,8 +100,6 @@ jobs:
}
if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1 || ! image_usable; then
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker buildx build \
-f ./docker/Dockerfile \
--target ci-deps-base \
@@ -182,15 +176,11 @@ jobs:
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Free Docker build space
- name: Inspect Docker build space
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }}
run: |
set -euo pipefail
docker system df || true
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker system prune --all --force --volumes || true
docker system df || true
- name: Build and push dev images
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }}
@@ -274,8 +264,6 @@ jobs:
return 0
fi
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker buildx build \
-f ./docker/Dockerfile \
--target "${target}" \

View File

@@ -1,192 +0,0 @@
name: Deploy Customer Main
on:
push:
branches:
- main
workflow_dispatch:
concurrency:
group: mostovik-backend-customer-main-${{ github.ref }}
cancel-in-progress: true
env:
CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru
CUSTOMER_REGISTRY_NAMESPACE: avm
CUSTOMER_WEB_IMAGE: mostovik-backend-web
CUSTOMER_CELERY_IMAGE: mostovik-backend-celery
CUSTOMER_DEPLOY_USER: ecos
CUSTOMER_DEPLOY_HOST: 10.0.10.174
CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121
CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml
CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh
jobs:
deploy:
name: Build, Push, Deploy
runs-on: [backend-docker]
timeout-minutes: 90
steps:
- name: Ensure main branch
run: |
set -euo pipefail
if [ "${GITHUB_REF_NAME}" != "main" ]; then
echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2
exit 1
fi
- name: Checkout code
run: |
set -euo pipefail
REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Validate deploy inputs
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
GITEA_TOKEN: ${{ gitea.token }}
CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
run: |
set -euo pipefail
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
home_dir="${HOME:-/root}"
if [ -z "${registry_user}" ]; then
echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2
exit 1
fi
if [ -z "${registry_password}" ]; then
echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2
exit 1
fi
if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \
&& [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \
&& [ -z "${DEPLOY_SSH_KEY:-}" ] \
&& [ ! -f "${home_dir}/.ssh/ci-key" ] \
&& [ ! -f "/root/.ssh/ci-key" ]; then
echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2
exit 1
fi
- name: Setup Docker Buildx
run: |
set -euo pipefail
if ! docker buildx inspect customer-builder >/dev/null 2>&1; then
docker buildx create --name customer-builder --driver docker-container --use
else
docker buildx use customer-builder
fi
docker buildx inspect --bootstrap
- name: Build and push customer images
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
GITEA_TOKEN: ${{ gitea.token }}
run: |
set -euo pipefail
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)"
registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}"
web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}"
celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}"
printf '%s' "${registry_password}" \
| docker login "${CUSTOMER_REGISTRY_HOST}" \
-u "${registry_user}" \
--password-stdin
docker buildx build \
--platform linux/amd64 \
-f ./docker/Dockerfile \
--target runtime-web \
--build-arg INSTALL_DEV=false \
--push \
-t "${web_ref}:latest" \
-t "${web_ref}:${sha_short}" \
.
docker buildx build \
--platform linux/amd64 \
-f ./docker/Dockerfile \
--target runtime-celery \
--build-arg INSTALL_DEV=false \
--push \
-t "${celery_ref}:latest" \
-t "${celery_ref}:${sha_short}" \
.
- name: Deploy customer stack
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
GITEA_TOKEN: ${{ gitea.token }}
CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
run: |
set -euo pipefail
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
case "${registry_user}" in
*[!A-Za-z0-9._@-]*)
echo "Registry user contains unsupported characters" >&2
exit 1
;;
esac
home_dir="${HOME:-/root}"
mkdir -p "${home_dir}/.ssh"
key_path="${home_dir}/.ssh/customer_deploy_key"
if [ -f "/root/.ssh/ci-key" ]; then
cp "/root/.ssh/ci-key" "${key_path}"
elif [ -f "${home_dir}/.ssh/ci-key" ]; then
cp "${home_dir}/.ssh/ci-key" "${key_path}"
elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then
printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}"
elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then
printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}"
elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then
printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}"
else
echo "Customer deploy SSH key is unavailable" >&2
exit 1
fi
chmod 600 "${key_path}"
ssh_common=(
-i "${key_path}"
-o BatchMode=yes
-o IdentitiesOnly=yes \
-o StrictHostKeyChecking=no
-o UserKnownHostsFile=/dev/null
-o GlobalKnownHostsFile=/dev/null
-o LogLevel=ERROR
-o ConnectTimeout=15
)
proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}"
remote_command="set -euo pipefail
flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'"
ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true"
printf '%s' "${registry_password}" \
| ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \
"docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin"
ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}"

View File

@@ -0,0 +1,5 @@
# Use only with the verified deployed image and a views-only application diff.
ARG HEALTH_BASE_IMAGE
FROM ${HEALTH_BASE_IMAGE}
COPY --chown=0:0 --chmod=0644 src/apps/core/views.py /app/src/apps/core/views.py

36
docs/health-probes.md Normal file
View File

@@ -0,0 +1,36 @@
# Health probes
`GET /health/live/` and `GET /health/ready/` are unauthenticated infrastructure
probes and do not consume API rate limits. Liveness reports the running process;
readiness checks the database and still returns HTTP 503 when it is unavailable.
The comprehensive `/health/` endpoint and ordinary API endpoints retain their
configured throttling.
## Internal-main health hotfix
The internal-main Docker probe runs every 10 seconds. Applying the shared
anonymous limit of 100 requests/hour to it causes false HTTP 429 failures.
Customer deployment workflows have been removed. Do not rerun historical
customer workflows: they execute the configuration from their original commit.
For this views-only fix, `docker/Dockerfile.health-hotfix` can build a web image
from the exact deployed `repository@sha256:digest`, preserving its dependencies,
startup command and all other application files. Before using this recipe,
verify that the base image matches the source baseline and that the application
diff contains only the probe changes in `src/apps/core/views.py`. Label the new
image with the committed source revision and publish an immutable digest.
The normal internal-main backend release requires `--refresh-data` and replaces
the database/media clone. It must not be used for this health-only fix. Instead,
under the existing release lock, verify that the candidate has no pending
migrations, keep a private copy of the current manifest, and change only
`MOSTOVIK_BACKEND_WEB_IMAGE`. Recreate only `mostovik-web` with `--no-deps` and the
already-pulled image. Accept the manifest atomically after HTTP probes and Docker
health succeed; restore the previous web image and manifest if they fail.
Preserve database/media selections, Redis state and every other service image.
Acceptance includes repeated probes beyond the anonymous request allowance,
database-failure regression coverage, and a control proving that ordinary
endpoints are still throttled. Do not disable throttling globally or clear Redis
to make the healthcheck pass.

View File

@@ -167,6 +167,7 @@ class LivenessView(APIView):
permission_classes = [AllowAny]
authentication_classes = []
throttle_classes = []
@swagger_auto_schema(
tags=[HEALTH_TAG],
@@ -174,7 +175,7 @@ class LivenessView(APIView):
operation_description="Возвращает 200 если приложение запущено.",
responses={
200: "Приложение запущено",
**ErrorResponses.PUBLIC,
500: CommonResponses.SERVER_ERROR,
},
)
def get(self, request: Request) -> Response:
@@ -191,6 +192,7 @@ class ReadinessView(APIView):
permission_classes = [AllowAny]
authentication_classes = []
throttle_classes = []
@swagger_auto_schema(
tags=[HEALTH_TAG],
@@ -201,7 +203,7 @@ class ReadinessView(APIView):
responses={
200: "Приложение готово обрабатывать запросы",
503: CommonResponses.SERVICE_UNAVAILABLE,
**ErrorResponses.PUBLIC,
500: CommonResponses.SERVER_ERROR,
},
)
def get(self, request: Request) -> Response:

View File

@@ -3,18 +3,32 @@
import sys
import types
from datetime import timedelta
from unittest.mock import patch
from apps.core import views as core_views
from apps.core.views import HealthCheckView
from django.urls import reverse
from django.core.cache.backends.locmem import LocMemCache
from django.urls import path, reverse
from django.utils import timezone
from drf_yasg import openapi
from drf_yasg.generators import OpenAPISchemaGenerator
from rest_framework import status
from rest_framework.test import APIRequestFactory, APITestCase
from rest_framework.throttling import AnonRateThrottle
from rest_framework.views import APIView
from tests.apps.user.factories import UserFactory
from tests.utils.fixtures import fake
class _RejectAllThrottle:
def allow_request(self, request, view):
return False
def wait(self):
return None
class HealthCheckViewTest(APITestCase):
"""Tests for HealthCheckView"""
@@ -23,6 +37,12 @@ class HealthCheckViewTest(APITestCase):
url = reverse("core:health")
self.assertEqual(url, "/health/")
def test_detailed_health_check_keeps_global_throttling(self):
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(reverse("core:health"))
self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
def test_health_check_success(self):
"""Test health check returns healthy status"""
url = reverse("core:health")
@@ -228,6 +248,13 @@ class LivenessViewTest(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "alive")
def test_liveness_bypasses_global_throttling(self):
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(reverse("core:liveness"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "alive")
class ReadinessViewTest(APITestCase):
"""Tests for ReadinessView"""
@@ -245,6 +272,13 @@ class ReadinessViewTest(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "ready")
def test_readiness_bypasses_global_throttling(self):
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(reverse("core:readiness"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "ready")
def test_readiness_returns_not_ready_on_db_error(self):
original_connection = core_views.connection
@@ -262,6 +296,7 @@ class ReadinessViewTest(APITestCase):
try:
core_views.connection = _BrokenConnection()
url = reverse("core:readiness")
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(url)
finally:
core_views.connection = original_connection
@@ -270,6 +305,65 @@ class ReadinessViewTest(APITestCase):
self.assertEqual(response.data["status"], "not_ready")
class HealthProbeThrottlingTest(APITestCase):
def test_openapi_keeps_rate_limit_only_on_detailed_health(self):
generator = OpenAPISchemaGenerator(
info=openapi.Info(title="Health probes", default_version="v1"),
patterns=[
path("health/", HealthCheckView.as_view()),
path("health/live/", core_views.LivenessView.as_view()),
path("health/ready/", core_views.ReadinessView.as_view()),
],
)
schema = generator.get_schema(request=None, public=True)
responses = {
f"{schema.base_path.rstrip('/')}{url}": item["get"].responses
for url, item in schema.paths.items()
}
self.assertNotIn("429", responses["/health/live/"])
self.assertNotIn("429", responses["/health/ready/"])
self.assertIn("503", responses["/health/ready/"])
self.assertIn("429", responses["/health/"])
def test_probes_do_not_consume_or_obey_exhausted_anonymous_quota(self):
probe_cache = LocMemCache(self.id(), {})
probe_cache.clear()
self.addCleanup(probe_cache.clear)
class OnePerMinuteAnonThrottle(AnonRateThrottle):
rate = "1/min"
cache = probe_cache
with (
patch.object(APIView, "throttle_classes", [OnePerMinuteAnonThrottle]),
patch.object(
HealthCheckView, "_check_database", return_value={"status": "up"}
),
patch.object(
HealthCheckView, "_check_redis", return_value={"status": "up"}
),
):
for quota_exhausted in (False, True):
for _ in range(3):
for name, expected_status in (
("liveness", "alive"),
("readiness", "ready"),
):
with self.subTest(probe=name, quota_exhausted=quota_exhausted):
response = self.client.get(reverse(f"core:{name}"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], expected_status)
if not quota_exhausted:
response = self.client.get(reverse("core:health"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.client.get(reverse("core:health"))
self.assertEqual(
response.status_code, status.HTTP_429_TOO_MANY_REQUESTS
)
class APIVersioningURLTest(APITestCase):
"""Tests for API versioning URL structure"""

View File

@@ -0,0 +1,26 @@
from pathlib import Path
import yaml
WORKFLOWS = Path(__file__).resolve().parents[1] / ".gitea" / "workflows"
def test_workflows_do_not_deploy_to_center():
assert not (WORKFLOWS / "deploy-customer-main.yml").exists()
for path in WORKFLOWS.glob("*.y*ml"):
workflow = yaml.safe_load(path.read_text())
assert "deploy_customer_main" not in workflow["jobs"]
content = path.read_text()
for forbidden in ("CUSTOMER_DEPLOY_", "10.0.10.174", "/ecos/release-service"):
assert forbidden not in content, (path.name, forbidden)
def test_ci_does_not_prune_shared_runner_resources():
for path in WORKFLOWS.glob("*.y*ml"):
content = path.read_text()
for command in (
"docker system prune",
"docker builder prune",
"docker buildx prune",
):
assert command not in content, (path.name, command)