feat: add sanitized internal main deployment

.gitea/workflows/ci-cd.yml

@@ -1,47 +1,47 @@
-name: CI/CD Pipeline
+name: State Corp Backend CI/CD
 
 on:
   push:
     branches:
-      - main
-      - develop
       - dev
+      # Add main only after the first manual main restore/rollback drill passes.
   pull_request:
     branches:
-      - main
-      - develop
       - dev
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: state-corp-backend-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHON_VERSION: "3.11"
-  REGISTRY_HOST: "registry.dev.nii-ecos.ru"
-  REGISTRY_NAMESPACE: "${{ github.repository_owner }}"
-  WEB_IMAGE: "state-corp-backend-web"
-  CELERY_IMAGE: "state-corp-backend-celery"
+  REGISTRY_HOST: registry.dev.nii-ecos.ru
+  REGISTRY_NAMESPACE: avm
+  WEB_IMAGE: state-corp-backend-web
+  CELERY_IMAGE: state-corp-backend-celery
 
 jobs:
-  lint:
-    name: Code Quality Checks
+  quality:
+    name: Quality gate
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
 
     steps:
-      - name: Checkout code
-        run: |
-          set -euo pipefail
-          REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|")
-          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
-          git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
-          git checkout "${GITHUB_SHA}"
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
 
       - name: Install Python and uv
         run: |
           set -euo pipefail
-          PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")"
-          PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")"
-          printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env
+          project_python_version="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")"
+          python_bin="$(./scripts/ensure-ci-python.sh "${project_python_version}")"
+          printf 'PYTHON_BIN=%s\n' "${python_bin}" > .ci-python-env
 
-      - name: Create virtual environment and install dependencies
+      - name: Install dependencies
         run: |
           set -euo pipefail
           . ./.ci-python-env
@@ -56,55 +56,14 @@ jobs:
             --no-managed-python \
             --no-python-downloads
 
-      - name: Run Ruff linting
+      - name: Lint and formatting
         run: |
           set -euo pipefail
           . .venv/bin/activate
           ruff check src tests scripts
-
-      - name: Run Ruff formatting check
-        run: |
-          set -euo pipefail
-          . .venv/bin/activate
           ruff format src tests scripts --check
 
-  test:
-    name: Run Tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-
-    steps:
-      - name: Checkout code
-        run: |
-          set -euo pipefail
-          REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|")
-          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
-          git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
-          git checkout "${GITHUB_SHA}"
-
-      - name: Install Python and uv
-        run: |
-          set -euo pipefail
-          PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")"
-          PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")"
-          printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env
-
-      - name: Create virtual environment and install dependencies
-        run: |
-          set -euo pipefail
-          . ./.ci-python-env
-          "${PYTHON_BIN}" -m venv .venv
-          . .venv/bin/activate
-          python -m pip install --upgrade pip uv
-          uv sync \
-            --dev \
-            --frozen \
-            --active \
-            --python "${PYTHON_BIN}" \
-            --no-managed-python \
-            --no-python-downloads
-
-      - name: Run pytest suite
+      - name: Tests
         env:
           DJANGO_SETTINGS_MODULE: settings.test
           SECRET_KEY: test-secret-key-for-ci
@@ -113,126 +72,227 @@ jobs:
           export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}"
           .venv/bin/python -m pytest tests -q
 
-  build_push:
-    name: Build and Push Dev Images
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    needs: [lint, test]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image')
+  build:
+    name: Build linux/amd64 images once
+    runs-on: [backend-docker]
+    timeout-minutes: 60
+    needs: [quality]
+    if: >-
+      github.event_name != 'pull_request' &&
+      (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
+    outputs:
+      celery_ref: ${{ steps.image_refs.outputs.celery_ref }}
+      celery_tag_ref: ${{ steps.image_refs.outputs.celery_tag_ref }}
+      run_id: ${{ steps.image_refs.outputs.run_id }}
+      web_ref: ${{ steps.image_refs.outputs.web_ref }}
+      web_tag_ref: ${{ steps.image_refs.outputs.web_tag_ref }}
 
     steps:
-      - name: Checkout code
-        run: |
-          set -euo pipefail
-          REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|")
-          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
-          git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
-          git checkout "${GITHUB_SHA}"
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
 
-      - name: Build and push images
+      - name: Validate registry credentials
         env:
-          REGISTRY_USER: ${{ secrets.REGISTRY_USERNAME }}
           REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
         run: |
           set -euo pipefail
-          if [ -z "${REGISTRY_USER:-}" ] || [ -z "${REGISTRY_PASSWORD:-}" ]; then
-            echo "Registry credentials are not configured" >&2
+          [ -n "${REGISTRY_USERNAME:-}" ] || {
+            echo 'Registry username is not configured.' >&2
             exit 1
-          fi
+          }
+          [ -n "${REGISTRY_PASSWORD:-}" ] || {
+            echo 'Registry password/token is not configured.' >&2
+            exit 1
+          }
 
-          SHA_SHORT="$(printf '%s' "${GITHUB_SHA}" | cut -c1-7)"
-          BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-          REGISTRY="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
-          WEB_REF="${REGISTRY}/${WEB_IMAGE}"
-          CELERY_REF="${REGISTRY}/${CELERY_IMAGE}"
+      - name: Prepare Buildx
+        run: |
+          set -euo pipefail
+          builder_name="state-corp-backend-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}"
+          docker buildx create \
+            --driver docker-container \
+            --name "${builder_name}" \
+            --use
+          docker buildx inspect --bootstrap
+          printf 'BUILDER_NAME=%s\n' "${builder_name}" >> "${GITHUB_ENV}"
 
-          echo "${REGISTRY_PASSWORD}" \
+      - name: Build and push immutable images
+        id: image_refs
+        env:
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+        run: |
+          set -euo pipefail
+
+          registry_path="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
+          web_image="${registry_path}/${WEB_IMAGE}"
+          celery_image="${registry_path}/${CELERY_IMAGE}"
+          short_sha="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)"
+          branch="${GITHUB_REF_NAME}"
+          immutable_tag="${branch}-${short_sha}"
+          build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+          printf '%s' "${REGISTRY_PASSWORD}" \
             | docker login "${REGISTRY_HOST}" \
-              -u "${REGISTRY_USER}" \
+              --username "${REGISTRY_USERNAME}" \
               --password-stdin
 
-          if ! docker buildx inspect state-corp-builder >/dev/null 2>&1; then
-            docker buildx create --name state-corp-builder --use
-          else
-            docker buildx use state-corp-builder
-          fi
-          docker buildx inspect --bootstrap
-
           docker buildx build \
-            -f ./docker/Dockerfile \
+            --platform linux/amd64 \
+            --file ./docker/Dockerfile \
             --target runtime-web \
             --build-arg INSTALL_DEV=false \
             --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
             --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \
-            --label "org.opencontainers.image.created=${BUILD_TIME}" \
-            --tag "${WEB_REF}:dev-${SHA_SHORT}" \
-            --tag "${WEB_REF}:dev" \
+            --label "org.opencontainers.image.created=${build_time}" \
+            --tag "${web_image}:${immutable_tag}" \
+            --metadata-file web-metadata.json \
             --push \
             .
 
           docker buildx build \
-            -f ./docker/Dockerfile \
+            --platform linux/amd64 \
+            --file ./docker/Dockerfile \
             --target runtime-celery \
             --build-arg INSTALL_DEV=false \
             --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
             --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \
-            --label "org.opencontainers.image.created=${BUILD_TIME}" \
-            --tag "${CELERY_REF}:dev-${SHA_SHORT}" \
-            --tag "${CELERY_REF}:dev" \
+            --label "org.opencontainers.image.created=${build_time}" \
+            --tag "${celery_image}:${immutable_tag}" \
+            --metadata-file celery-metadata.json \
             --push \
             .
 
-  deploy_dev:
-    name: Deploy Dev via Compose
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    needs: [build_push]
-    if: needs.build_push.result == 'success'
+          metadata_digest() {
+            sed -n \
+              's/^[[:space:]]*"containerimage.digest":[[:space:]]*"\([^"]*\)".*/\1/p' \
+              "$1" \
+              | head -n 1
+          }
 
-    steps:
-      - name: Checkout code
+          web_digest="$(metadata_digest web-metadata.json)"
+          celery_digest="$(metadata_digest celery-metadata.json)"
+          digest_pattern='^sha256:[a-f0-9]{64}$'
+          [[ "${web_digest}" =~ ${digest_pattern} ]] || {
+            echo 'Buildx did not return a valid web image digest.' >&2
+            exit 1
+          }
+          [[ "${celery_digest}" =~ ${digest_pattern} ]] || {
+            echo 'Buildx did not return a valid Celery image digest.' >&2
+            exit 1
+          }
+
+          release_run_id="${GITHUB_RUN_ID:-${GITHUB_SHA}}-${GITHUB_RUN_ATTEMPT:-1}"
+          {
+            printf 'web_ref=%s@%s\n' "${web_image}" "${web_digest}"
+            printf 'celery_ref=%s@%s\n' "${celery_image}" "${celery_digest}"
+            printf 'web_tag_ref=%s:%s\n' "${web_image}" "${immutable_tag}"
+            printf 'celery_tag_ref=%s:%s\n' "${celery_image}" "${immutable_tag}"
+            printf 'run_id=%s\n' "${release_run_id}"
+          } >> "${GITHUB_OUTPUT}"
+
+      - name: Remove Buildx builder
+        if: always()
         run: |
-          set -euo pipefail
-          REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|")
-          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
-          git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
-          git checkout "${GITHUB_SHA}"
-
-      - name: Deploy prebuilt images via SSH
-        env:
-          DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
-          DEPLOY_USER: ${{ secrets.DEPLOY_USER }}
-          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-          REGISTRY_HOST: ${{ secrets.REGISTRY_HOST }}
-          HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
-        run: |
-          set -euo pipefail
-
-          if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then
-            echo "Skip dev deploy for ${GITHUB_REF}"
-            exit 0
+          if [ -n "${BUILDER_NAME:-}" ]; then
+            docker buildx rm "${BUILDER_NAME}" || true
           fi
 
-          case "${HEAD_COMMIT_MESSAGE:-}" in
-            *"#no_deploy"* | *"#no_image"*)
-              echo "Skip dev deploy because commit message disables deploy or image build"
-              exit 0
-              ;;
-          esac
+  deploy_dev:
+    name: Release dev
+    runs-on: [backend-docker]
+    timeout-minutes: 20
+    needs: [build]
+    if: needs.build.result == 'success' && github.ref == 'refs/heads/dev'
 
-          short_sha="$(printf '%s' "${GITHUB_SHA}" | cut -c1-7)"
-          image_tag="dev-${short_sha}"
-          mkdir -p ~/.ssh
-          printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > ~/.ssh/ecos_deploy_key
-          chmod 0600 ~/.ssh/ecos_deploy_key
-          ssh-keyscan -H "${DEPLOY_HOST}" >> ~/.ssh/known_hosts 2>/dev/null
-          tmp_current="$(mktemp)"
-          ssh -i ~/.ssh/ecos_deploy_key "${DEPLOY_USER}@${DEPLOY_HOST}" 'cat /opt/ecos-dev/releases/current.env' > "${tmp_current}"
-          grep -v '^STATE_CORP_BACKEND_' "${tmp_current}" > "${tmp_current}.new"
-          cat >> "${tmp_current}.new" <<EOF
-          STATE_CORP_BACKEND_WEB_IMAGE=${REGISTRY_HOST}/avm/state-corp-backend-web:${image_tag}
-          STATE_CORP_BACKEND_CELERY_IMAGE=${REGISTRY_HOST}/avm/state-corp-backend-celery:${image_tag}
-          EOF
-          scp -i ~/.ssh/ecos_deploy_key "${tmp_current}.new" "${DEPLOY_USER}@${DEPLOY_HOST}:/tmp/current.env"
-          ssh -i ~/.ssh/ecos_deploy_key "${DEPLOY_USER}@${DEPLOY_HOST}" 'cat /tmp/current.env > /opt/ecos-dev/releases/current.env && rm -f /tmp/current.env && /opt/ecos-dev/deploy.sh state-corp-backend'
-          ssh -i ~/.ssh/ecos_deploy_key "${DEPLOY_USER}@${DEPLOY_HOST}" 'cd /opt/ecos-dev && docker compose --env-file runtime.env --env-file releases/current.env -f compose.yml run --rm --no-deps state-corp-web python src/manage.py migrate --noinput'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Release exact images
+        env:
+          RELEASE_HOST: ${{ secrets.DEPLOY_HOST }}
+          RELEASE_KNOWN_HOSTS_B64: ${{ secrets.INTERNAL_KNOWN_HOSTS_B64 }}
+          RELEASE_SERVICE_PATH: /opt/ecos-dev/release-service
+          RELEASE_SSH_KEY_B64: ${{ secrets.INTERNAL_DEPLOY_SSH_KEY_B64 }}
+          RELEASE_USER: ${{ secrets.DEPLOY_USER }}
+        run: |
+          ./scripts/ci/release-service.sh \
+            state-corp-backend \
+            "${{ needs.build.outputs.run_id }}" \
+            "${{ needs.build.outputs.web_ref }}" \
+            "${{ needs.build.outputs.celery_ref }}"
+
+      - name: Smoke dev backend
+        env:
+          APP_BASE_URL: ${{ secrets.APP_BASE_URL }}
+        run: ./scripts/ci/smoke.sh "${APP_BASE_URL%/}/health/ready/"
+
+  deploy_main_internal:
+    name: Refresh and release internal main
+    runs-on: [backend-docker]
+    timeout-minutes: 60
+    needs: [build]
+    if: needs.build.result == 'success' && github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Refresh cloned data and release exact images
+        env:
+          RELEASE_HOST: ${{ secrets.MAIN_DEPLOY_HOST }}
+          RELEASE_KNOWN_HOSTS_B64: ${{ secrets.INTERNAL_KNOWN_HOSTS_B64 }}
+          RELEASE_SERVICE_PATH: /opt/ecos-main/release-service
+          RELEASE_SSH_KEY_B64: ${{ secrets.INTERNAL_DEPLOY_SSH_KEY_B64 }}
+          RELEASE_USER: ${{ secrets.DEPLOY_USER }}
+        run: |
+          ./scripts/ci/release-service.sh \
+            state-corp-backend \
+            "${{ needs.build.outputs.run_id }}" \
+            "${{ needs.build.outputs.web_ref }}" \
+            "${{ needs.build.outputs.celery_ref }}" \
+            --refresh-data
+
+      - name: Smoke internal main backend
+        env:
+          INTERNAL_MAIN_STATE_CORP_BACKEND_URL: ${{ secrets.INTERNAL_MAIN_STATE_CORP_BACKEND_URL }}
+        run: ./scripts/ci/smoke.sh "${INTERNAL_MAIN_STATE_CORP_BACKEND_URL%/}/health/ready/"
+
+  deploy_main_customer:
+    name: Release customer main
+    runs-on: [backend-docker]
+    timeout-minutes: 30
+    needs: [build, deploy_main_internal]
+    if: >-
+      needs.build.result == 'success' &&
+      needs.deploy_main_internal.result == 'success' &&
+      github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Release the same exact images without refresh
+        env:
+          RELEASE_HOST: 10.0.10.174
+          RELEASE_KNOWN_HOSTS_B64: ${{ secrets.CUSTOMER_KNOWN_HOSTS_B64 }}
+          RELEASE_PROXY: ecos-proxy@10.10.0.121
+          RELEASE_SERVICE_PATH: /ecos/release-service
+          RELEASE_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
+          RELEASE_USER: ecos
+        run: |
+          ./scripts/ci/release-service.sh \
+            state-corp-backend \
+            "${{ needs.build.outputs.run_id }}" \
+            "${{ needs.build.outputs.web_ref }}" \
+            "${{ needs.build.outputs.celery_ref }}"

.gitea/workflows/deploy-customer-main.yml

@@ -1,192 +0,0 @@
-name: Deploy Customer Main
-
-on:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-
-concurrency:
-  group: state-corp-backend-customer-main-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru
-  CUSTOMER_REGISTRY_NAMESPACE: avm
-  CUSTOMER_WEB_IMAGE: state-corp-backend-web
-  CUSTOMER_CELERY_IMAGE: state-corp-backend-celery
-  CUSTOMER_DEPLOY_USER: ecos
-  CUSTOMER_DEPLOY_HOST: 10.0.10.174
-  CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121
-  CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml
-  CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh
-
-jobs:
-  deploy:
-    name: Build, Push, Deploy
-    runs-on: [backend-docker]
-    timeout-minutes: 90
-
-    steps:
-      - name: Ensure main branch
-        run: |
-          set -euo pipefail
-          if [ "${GITHUB_REF_NAME}" != "main" ]; then
-            echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2
-            exit 1
-          fi
-
-      - name: Checkout code
-        run: |
-          set -euo pipefail
-          REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|")
-          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
-          git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
-          git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
-
-      - name: Validate deploy inputs
-        env:
-          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-          GITEA_TOKEN: ${{ gitea.token }}
-          CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
-          CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
-          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-        run: |
-          set -euo pipefail
-
-          registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
-          registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
-          home_dir="${HOME:-/root}"
-
-          if [ -z "${registry_user}" ]; then
-            echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2
-            exit 1
-          fi
-          if [ -z "${registry_password}" ]; then
-            echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2
-            exit 1
-          fi
-          if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \
-            && [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \
-            && [ -z "${DEPLOY_SSH_KEY:-}" ] \
-            && [ ! -f "${home_dir}/.ssh/ci-key" ] \
-            && [ ! -f "/root/.ssh/ci-key" ]; then
-            echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2
-            exit 1
-          fi
-
-      - name: Setup Docker Buildx
-        run: |
-          set -euo pipefail
-          if ! docker buildx inspect customer-builder >/dev/null 2>&1; then
-            docker buildx create --name customer-builder --driver docker-container --use
-          else
-            docker buildx use customer-builder
-          fi
-          docker buildx inspect --bootstrap
-
-      - name: Build and push customer images
-        env:
-          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-          GITEA_TOKEN: ${{ gitea.token }}
-        run: |
-          set -euo pipefail
-
-          registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
-          registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
-          sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)"
-          registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}"
-          web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}"
-          celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}"
-
-          printf '%s' "${registry_password}" \
-            | docker login "${CUSTOMER_REGISTRY_HOST}" \
-              -u "${registry_user}" \
-              --password-stdin
-
-          docker buildx build \
-            --platform linux/amd64 \
-            -f ./docker/Dockerfile \
-            --target runtime-web \
-            --build-arg INSTALL_DEV=false \
-            --push \
-            -t "${web_ref}:latest" \
-            -t "${web_ref}:${sha_short}" \
-            .
-
-          docker buildx build \
-            --platform linux/amd64 \
-            -f ./docker/Dockerfile \
-            --target runtime-celery \
-            --build-arg INSTALL_DEV=false \
-            --push \
-            -t "${celery_ref}:latest" \
-            -t "${celery_ref}:${sha_short}" \
-            .
-
-      - name: Deploy customer stack
-        env:
-          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
-          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
-          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
-          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
-          GITEA_TOKEN: ${{ gitea.token }}
-          CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
-          CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
-          DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
-        run: |
-          set -euo pipefail
-
-          registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
-          registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
-          case "${registry_user}" in
-            *[!A-Za-z0-9._@-]*)
-              echo "Registry user contains unsupported characters" >&2
-              exit 1
-              ;;
-          esac
-
-          home_dir="${HOME:-/root}"
-          mkdir -p "${home_dir}/.ssh"
-          key_path="${home_dir}/.ssh/customer_deploy_key"
-          if [ -f "/root/.ssh/ci-key" ]; then
-            cp "/root/.ssh/ci-key" "${key_path}"
-          elif [ -f "${home_dir}/.ssh/ci-key" ]; then
-            cp "${home_dir}/.ssh/ci-key" "${key_path}"
-          elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then
-            printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}"
-          elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then
-            printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}"
-          elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then
-            printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}"
-          else
-            echo "Customer deploy SSH key is unavailable" >&2
-            exit 1
-          fi
-          chmod 600 "${key_path}"
-
-          ssh_common=(
-            -i "${key_path}"
-            -o BatchMode=yes
-            -o IdentitiesOnly=yes \
-            -o StrictHostKeyChecking=no
-            -o UserKnownHostsFile=/dev/null
-            -o GlobalKnownHostsFile=/dev/null
-            -o LogLevel=ERROR
-            -o ConnectTimeout=15
-          )
-          proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}"
-          remote_command="set -euo pipefail
-             flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'"
-
-          ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true"
-          printf '%s' "${registry_password}" \
-            | ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \
-                "docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin"
-          ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}"