From fe1e5f79bd48b668896baa87c1b6ca316d0b2802 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Wed, 16 Sep 2026 09:39:56 +0200 Subject: [PATCH] fix: exclude infrastructure probes from API throttling --- .gitea/workflows/ci-cd.yml | 16 +- .gitea/workflows/deploy-customer-main.yml | 192 ---------------------- docker/Dockerfile.health-hotfix | 5 + docs/health-probes.md | 36 ++++ src/apps/core/views.py | 6 +- tests/apps/core/test_views.py | 98 ++++++++++- tests/test_deployment_boundaries.py | 26 +++ 7 files changed, 169 insertions(+), 210 deletions(-) delete mode 100644 .gitea/workflows/deploy-customer-main.yml create mode 100644 docker/Dockerfile.health-hotfix create mode 100644 docs/health-probes.md create mode 100644 tests/test_deployment_boundaries.py diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index b4a74f3..92314bf 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -45,14 +45,10 @@ jobs: git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - name: Free Docker space + - name: Inspect Docker space run: | set -euo pipefail docker system df || true - docker buildx prune --all --force || true - docker builder prune --all --force || true - docker system prune --all --force --volumes || true - docker system df || true - name: Run quality in golden image env: @@ -104,8 +100,6 @@ jobs: } if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1 || ! image_usable; then - docker buildx prune --all --force || true - docker builder prune --all --force || true docker buildx build \ -f ./docker/Dockerfile \ --target ci-deps-base \ @@ -182,15 +176,11 @@ jobs: git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - name: Free Docker build space + - name: Inspect Docker build space if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }} run: | set -euo pipefail docker system df || true - docker buildx prune --all --force || true - docker builder prune --all --force || true - docker system prune --all --force --volumes || true - docker system df || true - name: Build and push dev images if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }} @@ -274,8 +264,6 @@ jobs: return 0 fi - docker buildx prune --all --force || true - docker builder prune --all --force || true docker buildx build \ -f ./docker/Dockerfile \ --target "${target}" \ diff --git a/.gitea/workflows/deploy-customer-main.yml b/.gitea/workflows/deploy-customer-main.yml deleted file mode 100644 index d0b88c1..0000000 --- a/.gitea/workflows/deploy-customer-main.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: Deploy Customer Main - -on: - push: - branches: - - main - workflow_dispatch: - -concurrency: - group: mostovik-backend-customer-main-${{ github.ref }} - cancel-in-progress: true - -env: - CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru - CUSTOMER_REGISTRY_NAMESPACE: avm - CUSTOMER_WEB_IMAGE: mostovik-backend-web - CUSTOMER_CELERY_IMAGE: mostovik-backend-celery - CUSTOMER_DEPLOY_USER: ecos - CUSTOMER_DEPLOY_HOST: 10.0.10.174 - CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121 - CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml - CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh - -jobs: - deploy: - name: Build, Push, Deploy - runs-on: [backend-docker] - timeout-minutes: 90 - - steps: - - name: Ensure main branch - run: | - set -euo pipefail - if [ "${GITHUB_REF_NAME}" != "main" ]; then - echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2 - exit 1 - fi - - - name: Checkout code - run: | - set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Validate deploy inputs - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} - CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - home_dir="${HOME:-/root}" - - if [ -z "${registry_user}" ]; then - echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2 - exit 1 - fi - if [ -z "${registry_password}" ]; then - echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2 - exit 1 - fi - if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \ - && [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \ - && [ -z "${DEPLOY_SSH_KEY:-}" ] \ - && [ ! -f "${home_dir}/.ssh/ci-key" ] \ - && [ ! -f "/root/.ssh/ci-key" ]; then - echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2 - exit 1 - fi - - - name: Setup Docker Buildx - run: | - set -euo pipefail - if ! docker buildx inspect customer-builder >/dev/null 2>&1; then - docker buildx create --name customer-builder --driver docker-container --use - else - docker buildx use customer-builder - fi - docker buildx inspect --bootstrap - - - name: Build and push customer images - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)" - registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}" - web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}" - celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}" - - printf '%s' "${registry_password}" \ - | docker login "${CUSTOMER_REGISTRY_HOST}" \ - -u "${registry_user}" \ - --password-stdin - - docker buildx build \ - --platform linux/amd64 \ - -f ./docker/Dockerfile \ - --target runtime-web \ - --build-arg INSTALL_DEV=false \ - --push \ - -t "${web_ref}:latest" \ - -t "${web_ref}:${sha_short}" \ - . - - docker buildx build \ - --platform linux/amd64 \ - -f ./docker/Dockerfile \ - --target runtime-celery \ - --build-arg INSTALL_DEV=false \ - --push \ - -t "${celery_ref}:latest" \ - -t "${celery_ref}:${sha_short}" \ - . - - - name: Deploy customer stack - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} - CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - case "${registry_user}" in - *[!A-Za-z0-9._@-]*) - echo "Registry user contains unsupported characters" >&2 - exit 1 - ;; - esac - - home_dir="${HOME:-/root}" - mkdir -p "${home_dir}/.ssh" - key_path="${home_dir}/.ssh/customer_deploy_key" - if [ -f "/root/.ssh/ci-key" ]; then - cp "/root/.ssh/ci-key" "${key_path}" - elif [ -f "${home_dir}/.ssh/ci-key" ]; then - cp "${home_dir}/.ssh/ci-key" "${key_path}" - elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then - printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" - elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then - printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}" - elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then - printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}" - else - echo "Customer deploy SSH key is unavailable" >&2 - exit 1 - fi - chmod 600 "${key_path}" - - ssh_common=( - -i "${key_path}" - -o BatchMode=yes - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o GlobalKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o ConnectTimeout=15 - ) - proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}" - remote_command="set -euo pipefail - flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'" - - ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true" - printf '%s' "${registry_password}" \ - | ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \ - "docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin" - ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}" diff --git a/docker/Dockerfile.health-hotfix b/docker/Dockerfile.health-hotfix new file mode 100644 index 0000000..e11e44d --- /dev/null +++ b/docker/Dockerfile.health-hotfix @@ -0,0 +1,5 @@ +# Use only with the verified deployed image and a views-only application diff. +ARG HEALTH_BASE_IMAGE +FROM ${HEALTH_BASE_IMAGE} + +COPY --chown=0:0 --chmod=0644 src/apps/core/views.py /app/src/apps/core/views.py diff --git a/docs/health-probes.md b/docs/health-probes.md new file mode 100644 index 0000000..48d5b5f --- /dev/null +++ b/docs/health-probes.md @@ -0,0 +1,36 @@ +# Health probes + +`GET /health/live/` and `GET /health/ready/` are unauthenticated infrastructure +probes and do not consume API rate limits. Liveness reports the running process; +readiness checks the database and still returns HTTP 503 when it is unavailable. +The comprehensive `/health/` endpoint and ordinary API endpoints retain their +configured throttling. + +## Internal-main health hotfix + +The internal-main Docker probe runs every 10 seconds. Applying the shared +anonymous limit of 100 requests/hour to it causes false HTTP 429 failures. + +Customer deployment workflows have been removed. Do not rerun historical +customer workflows: they execute the configuration from their original commit. + +For this views-only fix, `docker/Dockerfile.health-hotfix` can build a web image +from the exact deployed `repository@sha256:digest`, preserving its dependencies, +startup command and all other application files. Before using this recipe, +verify that the base image matches the source baseline and that the application +diff contains only the probe changes in `src/apps/core/views.py`. Label the new +image with the committed source revision and publish an immutable digest. + +The normal internal-main backend release requires `--refresh-data` and replaces +the database/media clone. It must not be used for this health-only fix. Instead, +under the existing release lock, verify that the candidate has no pending +migrations, keep a private copy of the current manifest, and change only +`MOSTOVIK_BACKEND_WEB_IMAGE`. Recreate only `mostovik-web` with `--no-deps` and the +already-pulled image. Accept the manifest atomically after HTTP probes and Docker +health succeed; restore the previous web image and manifest if they fail. +Preserve database/media selections, Redis state and every other service image. + +Acceptance includes repeated probes beyond the anonymous request allowance, +database-failure regression coverage, and a control proving that ordinary +endpoints are still throttled. Do not disable throttling globally or clear Redis +to make the healthcheck pass. diff --git a/src/apps/core/views.py b/src/apps/core/views.py index 5f8bbe7..99308e7 100644 --- a/src/apps/core/views.py +++ b/src/apps/core/views.py @@ -167,6 +167,7 @@ class LivenessView(APIView): permission_classes = [AllowAny] authentication_classes = [] + throttle_classes = [] @swagger_auto_schema( tags=[HEALTH_TAG], @@ -174,7 +175,7 @@ class LivenessView(APIView): operation_description="Возвращает 200 если приложение запущено.", responses={ 200: "Приложение запущено", - **ErrorResponses.PUBLIC, + 500: CommonResponses.SERVER_ERROR, }, ) def get(self, request: Request) -> Response: @@ -191,6 +192,7 @@ class ReadinessView(APIView): permission_classes = [AllowAny] authentication_classes = [] + throttle_classes = [] @swagger_auto_schema( tags=[HEALTH_TAG], @@ -201,7 +203,7 @@ class ReadinessView(APIView): responses={ 200: "Приложение готово обрабатывать запросы", 503: CommonResponses.SERVICE_UNAVAILABLE, - **ErrorResponses.PUBLIC, + 500: CommonResponses.SERVER_ERROR, }, ) def get(self, request: Request) -> Response: diff --git a/tests/apps/core/test_views.py b/tests/apps/core/test_views.py index 2d688a0..3b9c0c4 100644 --- a/tests/apps/core/test_views.py +++ b/tests/apps/core/test_views.py @@ -3,18 +3,32 @@ import sys import types from datetime import timedelta +from unittest.mock import patch from apps.core import views as core_views from apps.core.views import HealthCheckView -from django.urls import reverse +from django.core.cache.backends.locmem import LocMemCache +from django.urls import path, reverse from django.utils import timezone +from drf_yasg import openapi +from drf_yasg.generators import OpenAPISchemaGenerator from rest_framework import status from rest_framework.test import APIRequestFactory, APITestCase +from rest_framework.throttling import AnonRateThrottle +from rest_framework.views import APIView from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake +class _RejectAllThrottle: + def allow_request(self, request, view): + return False + + def wait(self): + return None + + class HealthCheckViewTest(APITestCase): """Tests for HealthCheckView""" @@ -23,6 +37,12 @@ class HealthCheckViewTest(APITestCase): url = reverse("core:health") self.assertEqual(url, "/health/") + def test_detailed_health_check_keeps_global_throttling(self): + with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]): + response = self.client.get(reverse("core:health")) + + self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) + def test_health_check_success(self): """Test health check returns healthy status""" url = reverse("core:health") @@ -228,6 +248,13 @@ class LivenessViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["status"], "alive") + def test_liveness_bypasses_global_throttling(self): + with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]): + response = self.client.get(reverse("core:liveness")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["status"], "alive") + class ReadinessViewTest(APITestCase): """Tests for ReadinessView""" @@ -245,6 +272,13 @@ class ReadinessViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["status"], "ready") + def test_readiness_bypasses_global_throttling(self): + with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]): + response = self.client.get(reverse("core:readiness")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["status"], "ready") + def test_readiness_returns_not_ready_on_db_error(self): original_connection = core_views.connection @@ -262,7 +296,8 @@ class ReadinessViewTest(APITestCase): try: core_views.connection = _BrokenConnection() url = reverse("core:readiness") - response = self.client.get(url) + with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]): + response = self.client.get(url) finally: core_views.connection = original_connection @@ -270,6 +305,65 @@ class ReadinessViewTest(APITestCase): self.assertEqual(response.data["status"], "not_ready") +class HealthProbeThrottlingTest(APITestCase): + def test_openapi_keeps_rate_limit_only_on_detailed_health(self): + generator = OpenAPISchemaGenerator( + info=openapi.Info(title="Health probes", default_version="v1"), + patterns=[ + path("health/", HealthCheckView.as_view()), + path("health/live/", core_views.LivenessView.as_view()), + path("health/ready/", core_views.ReadinessView.as_view()), + ], + ) + schema = generator.get_schema(request=None, public=True) + responses = { + f"{schema.base_path.rstrip('/')}{url}": item["get"].responses + for url, item in schema.paths.items() + } + + self.assertNotIn("429", responses["/health/live/"]) + self.assertNotIn("429", responses["/health/ready/"]) + self.assertIn("503", responses["/health/ready/"]) + self.assertIn("429", responses["/health/"]) + + def test_probes_do_not_consume_or_obey_exhausted_anonymous_quota(self): + probe_cache = LocMemCache(self.id(), {}) + probe_cache.clear() + self.addCleanup(probe_cache.clear) + + class OnePerMinuteAnonThrottle(AnonRateThrottle): + rate = "1/min" + cache = probe_cache + + with ( + patch.object(APIView, "throttle_classes", [OnePerMinuteAnonThrottle]), + patch.object( + HealthCheckView, "_check_database", return_value={"status": "up"} + ), + patch.object( + HealthCheckView, "_check_redis", return_value={"status": "up"} + ), + ): + for quota_exhausted in (False, True): + for _ in range(3): + for name, expected_status in ( + ("liveness", "alive"), + ("readiness", "ready"), + ): + with self.subTest(probe=name, quota_exhausted=quota_exhausted): + response = self.client.get(reverse(f"core:{name}")) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["status"], expected_status) + + if not quota_exhausted: + response = self.client.get(reverse("core:health")) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response = self.client.get(reverse("core:health")) + self.assertEqual( + response.status_code, status.HTTP_429_TOO_MANY_REQUESTS + ) + + class APIVersioningURLTest(APITestCase): """Tests for API versioning URL structure""" diff --git a/tests/test_deployment_boundaries.py b/tests/test_deployment_boundaries.py new file mode 100644 index 0000000..a02d9da --- /dev/null +++ b/tests/test_deployment_boundaries.py @@ -0,0 +1,26 @@ +from pathlib import Path + +import yaml + +WORKFLOWS = Path(__file__).resolve().parents[1] / ".gitea" / "workflows" + + +def test_workflows_do_not_deploy_to_center(): + assert not (WORKFLOWS / "deploy-customer-main.yml").exists() + for path in WORKFLOWS.glob("*.y*ml"): + workflow = yaml.safe_load(path.read_text()) + assert "deploy_customer_main" not in workflow["jobs"] + content = path.read_text() + for forbidden in ("CUSTOMER_DEPLOY_", "10.0.10.174", "/ecos/release-service"): + assert forbidden not in content, (path.name, forbidden) + + +def test_ci_does_not_prune_shared_runner_resources(): + for path in WORKFLOWS.glob("*.y*ml"): + content = path.read_text() + for command in ( + "docker system prune", + "docker builder prune", + "docker buildx prune", + ): + assert command not in content, (path.name, command)