ci: use reusable golden images

2026-04-28 20:34:05 +02:00
parent 29e4fa8e97
commit 77d84b9778
3 changed files with 254 additions and 120 deletions
--- a/.gitea/workflows/ci-cd.yml
+++ b/.gitea/workflows/ci-cd.yml
@@ -14,7 +14,7 @@ on:
  workflow_dispatch:
    inputs:
      manual_action:
-        description: "Manual action: noop, cleanup_dev_database, or dokploy_start"
+        description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start"
        required: true
        default: "noop"
      dokploy_target:
@@ -37,6 +37,10 @@ env:
  REGISTRY_NAMESPACE: "${{ github.repository_owner }}"
  WEB_IMAGE: "mostovik-backend-web"
  CELERY_IMAGE: "mostovik-backend-celery"
+  CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden"
+  WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden"
+  CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden"
+  GOLDEN_TAG: "py311-uv0.7.2"
  DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF"
  DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA"
  DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv"
@@ -63,9 +67,95 @@ jobs:
            echo "- manual_action=cleanup_dev_database"
            echo "- cleanup_confirm=CLEAN_DEV_DB"
            echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat."
+            echo "For base image refresh run with manual_action=build_golden_images."
            echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat."
          } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"

+  build_golden_images:
+    name: Build Golden Images
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    if: |
+      github.event_name == 'workflow_dispatch' &&
+      github.ref == 'refs/heads/dev' &&
+      github.event.inputs.manual_action == 'build_golden_images'
+
+    steps:
+      - name: Checkout code
+        run: |
+          set -euo pipefail
+          REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|")
+          BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
+          git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
+          git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
+
+      - name: Build and push golden images
+        env:
+          GITEA_TOKEN: ${{ gitea.token }}
+          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
+          CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}"
+          WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}"
+          CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}"
+          REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}"
+          REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}"
+
+          unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY
+          export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}"
+          export no_proxy="${no_proxy:-},${REGISTRY_HOST}"
+
+          if [ -z "${REGISTRY_PASSWORD}" ]; then
+            echo "REGISTRY_TOKEN secret is not set and gitea.token fallback is empty" >&2
+            exit 1
+          fi
+
+          echo "${REGISTRY_PASSWORD}" \
+            | docker login "${REGISTRY_HOST}" \
+              -u "${REGISTRY_USER}" \
+              --password-stdin
+
+          if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then
+            docker buildx create --name mostovik-builder --use
+          else
+            docker buildx use mostovik-builder
+          fi
+          docker buildx inspect --bootstrap
+
+          docker buildx prune --all --force || true
+          docker builder prune --all --force || true
+
+          build_golden() {
+            local target="$1"
+            local ref="$2"
+            local build_args=()
+            if [ "${target}" = "celery-deps-base" ]; then
+              build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}")
+            fi
+            docker buildx build \
+              -f ./docker/Dockerfile \
+              --target "${target}" \
+              "${build_args[@]}" \
+              --push \
+              -t "${ref}:${GOLDEN_TAG}" \
+              -t "${ref}:latest" \
+              .
+          }
+
+          build_golden "ci-deps-base" "${CI_GOLDEN_REF}"
+          build_golden "web-deps-base" "${WEB_GOLDEN_REF}"
+          build_golden "celery-deps-base" "${CELERY_GOLDEN_REF}"
+
+          {
+            echo "Golden images pushed:"
+            echo "- ${CI_GOLDEN_REF}:${GOLDEN_TAG}"
+            echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}"
+            echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}"
+          } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+
  quality:
    name: Quality Gate
    runs-on: ubuntu-latest
@@ -81,94 +171,74 @@ jobs:
          git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" .
          git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"

-      - name: Install Python and uv
+      - name: Run quality in golden image
+        env:
+          GITEA_TOKEN: ${{ gitea.token }}
+          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }}
+          SKIP_LINT: ${{ contains(github.event.head_commit.message, '#no_lint') }}
+          SKIP_TEST: ${{ contains(github.event.head_commit.message, '#no_test') }}
        run: |
          set -euo pipefail
-          rm -rf .venv .ci-bin .ci-python-env
-          CLEAN_PATH=""
-          IFS=: read -r -a PATH_PARTS <<< "${PATH}"
-          for path_part in "${PATH_PARTS[@]}"; do
-            case "${path_part}" in
-              .venv/bin|*/.venv/bin)
-                continue
-                ;;
-            esac
-            CLEAN_PATH="${CLEAN_PATH:+${CLEAN_PATH}:}${path_part}"
-          done
-          export PATH="${CLEAN_PATH}"
-          hash -r
-          PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")"
-          PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")"
-          case "${PYTHON_BIN}" in
-            .venv/*|*/.venv/*)
-              echo "Refusing to use project virtualenv as base Python: ${PYTHON_BIN}" >&2
-              exit 1
-              ;;
-          esac
-          "${PYTHON_BIN}" --version

-          printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env
-          mkdir -p .ci-bin
-          if command -v uv >/dev/null 2>&1; then
-            cp "$(command -v uv)" .ci-bin/uv
-          elif ! curl -LsSf --connect-timeout 5 --max-time 60 --retry 1 "https://astral.sh/uv/${UV_VERSION}/install.sh" | env UV_INSTALL_DIR="${PWD}/.ci-bin" sh; then
-            curl -LsSf --connect-timeout 5 --max-time 60 --retry 1 "https://astral.sh/uv/install.sh" | env UV_INSTALL_DIR="${PWD}/.ci-bin" sh
+          REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
+          CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}"
+          REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}"
+          REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}"
+
+          unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY
+          export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}"
+          export no_proxy="${no_proxy:-},${REGISTRY_HOST}"
+
+          if [ -z "${REGISTRY_PASSWORD}" ]; then
+            echo "REGISTRY_TOKEN secret is not set and gitea.token fallback is empty" >&2
+            exit 1
          fi
-          .ci-bin/uv --version

-      - name: Create virtual environment and install dependencies
-        run: |
-          set -euo pipefail
-          . ./.ci-python-env
-          case "${PYTHON_BIN}" in
-            .venv/*|*/.venv/*)
-              echo "Refusing to create venv from project virtualenv: ${PYTHON_BIN}" >&2
-              exit 1
-              ;;
-          esac
-          "${PYTHON_BIN}" -m venv --without-pip .venv
-          . .venv/bin/activate
-          .ci-bin/uv sync \
-            --dev \
-            --frozen \
-            --active \
-            --python "${PWD}/.venv/bin/python" \
-            --no-managed-python \
-            --no-python-downloads
+          echo "${REGISTRY_PASSWORD}" \
+            | docker login "${REGISTRY_HOST}" \
+              -u "${REGISTRY_USER}" \
+              --password-stdin

-      - name: Run Ruff linting
-        if: ${{ !contains(github.event.head_commit.message, '#no_lint') }}
-        run: |
-          set -euo pipefail
-          . .venv/bin/activate
-          ruff check src
+          if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then
+            docker buildx create --name mostovik-builder --use
+          else
+            docker buildx use mostovik-builder
+          fi
+          docker buildx inspect --bootstrap

-      - name: Run Ruff formatting check
-        if: ${{ !contains(github.event.head_commit.message, '#no_lint') }}
-        run: |
-          set -euo pipefail
-          . .venv/bin/activate
-          ruff format src --check
+          if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1; then
+            docker buildx prune --all --force || true
+            docker builder prune --all --force || true
+            docker buildx build \
+              -f ./docker/Dockerfile \
+              --target ci-deps-base \
+              --push \
+              -t "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \
+              -t "${CI_GOLDEN_REF}:latest" \
+              .
+          fi

-      - name: Run regular pytest suite
-        if: ${{ !contains(github.event.head_commit.message, '#no_test') }}
-        env:
-          DJANGO_SETTINGS_MODULE: settings.test
-          SECRET_KEY: test-secret-key-for-ci
-        run: |
-          set -euo pipefail
-          export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}"
-          .venv/bin/python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q
-
-      - name: Run API inventory pytest suite
-        if: ${{ !contains(github.event.head_commit.message, '#no_test') }}
-        env:
-          DJANGO_SETTINGS_MODULE: settings.test
-          SECRET_KEY: test-secret-key-for-ci
-        run: |
-          set -euo pipefail
-          export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}"
-          .venv/bin/python -m pytest tests/test_api_inventory_e2e.py -q
+          docker run --rm \
+            -v "${PWD}:/workspace" \
+            -w /workspace \
+            -e DJANGO_SETTINGS_MODULE=settings.test \
+            -e SECRET_KEY=test-secret-key-for-ci \
+            -e SKIP_LINT="${SKIP_LINT}" \
+            -e SKIP_TEST="${SKIP_TEST}" \
+            "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \
+            bash -lc '
+              set -euo pipefail
+              export PYTHONPATH="/workspace/src:${PYTHONPATH:-}"
+              if [ "${SKIP_LINT}" != "true" ]; then
+                ruff check src
+                ruff format src --check
+              fi
+              if [ "${SKIP_TEST}" != "true" ]; then
+                python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q
+                python -m pytest tests/test_api_inventory_e2e.py -q
+              fi
+            '

  build_push:
    name: Build and Push Images
@@ -218,6 +288,8 @@ jobs:
          REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
          WEB_REF="${REGISTRY_PATH}/${WEB_IMAGE}"
          CELERY_REF="${REGISTRY_PATH}/${CELERY_IMAGE}"
+          WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}"
+          CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}"
          REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}"
          REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}"

@@ -255,15 +327,40 @@ jobs:
          fi
          docker buildx inspect --bootstrap

+          ensure_golden() {
+            local target="$1"
+            local ref="$2"
+            local build_args=()
+            if [ "${target}" = "celery-deps-base" ]; then
+              build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}")
+            fi
+            if docker buildx imagetools inspect "${ref}:${GOLDEN_TAG}" >/dev/null 2>&1; then
+              return 0
+            fi
+
+            docker buildx prune --all --force || true
+            docker builder prune --all --force || true
+            docker buildx build \
+              -f ./docker/Dockerfile \
+              --target "${target}" \
+              "${build_args[@]}" \
+              --push \
+              -t "${ref}:${GOLDEN_TAG}" \
+              -t "${ref}:latest" \
+              .
+          }
+
+          ensure_golden "web-deps-base" "${WEB_GOLDEN_REF}"
+          ensure_golden "celery-deps-base" "${CELERY_GOLDEN_REF}"
+
          docker buildx build \
            -f ./docker/Dockerfile \
            --target runtime-web \
            --build-arg INSTALL_DEV=false \
+            --build-arg GOLDEN_WEB_IMAGE="${WEB_GOLDEN_REF}:${GOLDEN_TAG}" \
            --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
            --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \
-            --output type=image,push=true,oci-mediatypes=false \
-            --provenance=false \
-            --sbom=false \
+            --push \
            "${WEB_TAGS[@]}" \
            .

@@ -271,16 +368,18 @@ jobs:
            -f ./docker/Dockerfile \
            --target runtime-celery \
            --build-arg INSTALL_DEV=false \
+            --build-arg GOLDEN_CELERY_IMAGE="${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" \
            --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
            --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \
-            --output type=image,push=true,oci-mediatypes=false \
-            --provenance=false \
-            --sbom=false \
+            --push \
            "${CELERY_TAGS[@]}" \
            .

          {
            echo "Registry API: ${REGISTRY_API_URL}"
+            echo "Golden images:"
+            echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}"
+            echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}"
            echo "Pushed images:"
            echo "- ${WEB_REF}:${BRANCH_TAG}"
            echo "- ${WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}"