From 77d84b9778edfdf61093bc12d44d80f447fcff3e Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 28 Apr 2026 20:34:05 +0200 Subject: [PATCH] ci: use reusable golden images --- .gitea/workflows/ci-cd.yml | 273 +++++++++++++++++++++++++------------ .pre-commit-config.yaml | 12 +- docker/Dockerfile | 89 ++++++++---- 3 files changed, 254 insertions(+), 120 deletions(-) diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 9b0f357..cfb402e 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -14,7 +14,7 @@ on: workflow_dispatch: inputs: manual_action: - description: "Manual action: noop, cleanup_dev_database, or dokploy_start" + description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start" required: true default: "noop" dokploy_target: @@ -37,6 +37,10 @@ env: REGISTRY_NAMESPACE: "${{ github.repository_owner }}" WEB_IMAGE: "mostovik-backend-web" CELERY_IMAGE: "mostovik-backend-celery" + CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden" + WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden" + CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden" + GOLDEN_TAG: "py311-uv0.7.2" DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" @@ -63,9 +67,95 @@ jobs: echo "- manual_action=cleanup_dev_database" echo "- cleanup_confirm=CLEAN_DEV_DB" echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat." + echo "For base image refresh run with manual_action=build_golden_images." echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat." } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + build_golden_images: + name: Build Golden Images + runs-on: ubuntu-latest + timeout-minutes: 60 + if: | + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/dev' && + github.event.inputs.manual_action == 'build_golden_images' + + steps: + - name: Checkout code + run: | + set -euo pipefail + REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") + BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . + git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" + + - name: Build and push golden images + env: + GITEA_TOKEN: ${{ gitea.token }} + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -euo pipefail + + REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}" + WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}" + CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}" + REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" + REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" + + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY + export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" + export no_proxy="${no_proxy:-},${REGISTRY_HOST}" + + if [ -z "${REGISTRY_PASSWORD}" ]; then + echo "REGISTRY_TOKEN secret is not set and gitea.token fallback is empty" >&2 + exit 1 + fi + + echo "${REGISTRY_PASSWORD}" \ + | docker login "${REGISTRY_HOST}" \ + -u "${REGISTRY_USER}" \ + --password-stdin + + if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then + docker buildx create --name mostovik-builder --use + else + docker buildx use mostovik-builder + fi + docker buildx inspect --bootstrap + + docker buildx prune --all --force || true + docker builder prune --all --force || true + + build_golden() { + local target="$1" + local ref="$2" + local build_args=() + if [ "${target}" = "celery-deps-base" ]; then + build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}") + fi + docker buildx build \ + -f ./docker/Dockerfile \ + --target "${target}" \ + "${build_args[@]}" \ + --push \ + -t "${ref}:${GOLDEN_TAG}" \ + -t "${ref}:latest" \ + . + } + + build_golden "ci-deps-base" "${CI_GOLDEN_REF}" + build_golden "web-deps-base" "${WEB_GOLDEN_REF}" + build_golden "celery-deps-base" "${CELERY_GOLDEN_REF}" + + { + echo "Golden images pushed:" + echo "- ${CI_GOLDEN_REF}:${GOLDEN_TAG}" + echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}" + echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" + } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + quality: name: Quality Gate runs-on: ubuntu-latest @@ -81,94 +171,74 @@ jobs: git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - name: Install Python and uv + - name: Run quality in golden image + env: + GITEA_TOKEN: ${{ gitea.token }} + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} + SKIP_LINT: ${{ contains(github.event.head_commit.message, '#no_lint') }} + SKIP_TEST: ${{ contains(github.event.head_commit.message, '#no_test') }} run: | set -euo pipefail - rm -rf .venv .ci-bin .ci-python-env - CLEAN_PATH="" - IFS=: read -r -a PATH_PARTS <<< "${PATH}" - for path_part in "${PATH_PARTS[@]}"; do - case "${path_part}" in - .venv/bin|*/.venv/bin) - continue - ;; - esac - CLEAN_PATH="${CLEAN_PATH:+${CLEAN_PATH}:}${path_part}" - done - export PATH="${CLEAN_PATH}" - hash -r - PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" - PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")" - case "${PYTHON_BIN}" in - .venv/*|*/.venv/*) - echo "Refusing to use project virtualenv as base Python: ${PYTHON_BIN}" >&2 - exit 1 - ;; - esac - "${PYTHON_BIN}" --version - printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env - mkdir -p .ci-bin - if command -v uv >/dev/null 2>&1; then - cp "$(command -v uv)" .ci-bin/uv - elif ! curl -LsSf --connect-timeout 5 --max-time 60 --retry 1 "https://astral.sh/uv/${UV_VERSION}/install.sh" | env UV_INSTALL_DIR="${PWD}/.ci-bin" sh; then - curl -LsSf --connect-timeout 5 --max-time 60 --retry 1 "https://astral.sh/uv/install.sh" | env UV_INSTALL_DIR="${PWD}/.ci-bin" sh + REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}" + REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" + REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" + + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY + export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" + export no_proxy="${no_proxy:-},${REGISTRY_HOST}" + + if [ -z "${REGISTRY_PASSWORD}" ]; then + echo "REGISTRY_TOKEN secret is not set and gitea.token fallback is empty" >&2 + exit 1 fi - .ci-bin/uv --version - - name: Create virtual environment and install dependencies - run: | - set -euo pipefail - . ./.ci-python-env - case "${PYTHON_BIN}" in - .venv/*|*/.venv/*) - echo "Refusing to create venv from project virtualenv: ${PYTHON_BIN}" >&2 - exit 1 - ;; - esac - "${PYTHON_BIN}" -m venv --without-pip .venv - . .venv/bin/activate - .ci-bin/uv sync \ - --dev \ - --frozen \ - --active \ - --python "${PWD}/.venv/bin/python" \ - --no-managed-python \ - --no-python-downloads + echo "${REGISTRY_PASSWORD}" \ + | docker login "${REGISTRY_HOST}" \ + -u "${REGISTRY_USER}" \ + --password-stdin - - name: Run Ruff linting - if: ${{ !contains(github.event.head_commit.message, '#no_lint') }} - run: | - set -euo pipefail - . .venv/bin/activate - ruff check src + if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then + docker buildx create --name mostovik-builder --use + else + docker buildx use mostovik-builder + fi + docker buildx inspect --bootstrap - - name: Run Ruff formatting check - if: ${{ !contains(github.event.head_commit.message, '#no_lint') }} - run: | - set -euo pipefail - . .venv/bin/activate - ruff format src --check + if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1; then + docker buildx prune --all --force || true + docker builder prune --all --force || true + docker buildx build \ + -f ./docker/Dockerfile \ + --target ci-deps-base \ + --push \ + -t "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ + -t "${CI_GOLDEN_REF}:latest" \ + . + fi - - name: Run regular pytest suite - if: ${{ !contains(github.event.head_commit.message, '#no_test') }} - env: - DJANGO_SETTINGS_MODULE: settings.test - SECRET_KEY: test-secret-key-for-ci - run: | - set -euo pipefail - export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" - .venv/bin/python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q - - - name: Run API inventory pytest suite - if: ${{ !contains(github.event.head_commit.message, '#no_test') }} - env: - DJANGO_SETTINGS_MODULE: settings.test - SECRET_KEY: test-secret-key-for-ci - run: | - set -euo pipefail - export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" - .venv/bin/python -m pytest tests/test_api_inventory_e2e.py -q + docker run --rm \ + -v "${PWD}:/workspace" \ + -w /workspace \ + -e DJANGO_SETTINGS_MODULE=settings.test \ + -e SECRET_KEY=test-secret-key-for-ci \ + -e SKIP_LINT="${SKIP_LINT}" \ + -e SKIP_TEST="${SKIP_TEST}" \ + "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ + bash -lc ' + set -euo pipefail + export PYTHONPATH="/workspace/src:${PYTHONPATH:-}" + if [ "${SKIP_LINT}" != "true" ]; then + ruff check src + ruff format src --check + fi + if [ "${SKIP_TEST}" != "true" ]; then + python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q + python -m pytest tests/test_api_inventory_e2e.py -q + fi + ' build_push: name: Build and Push Images @@ -218,6 +288,8 @@ jobs: REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" WEB_REF="${REGISTRY_PATH}/${WEB_IMAGE}" CELERY_REF="${REGISTRY_PATH}/${CELERY_IMAGE}" + WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}" + CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}" REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" @@ -255,15 +327,40 @@ jobs: fi docker buildx inspect --bootstrap + ensure_golden() { + local target="$1" + local ref="$2" + local build_args=() + if [ "${target}" = "celery-deps-base" ]; then + build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}") + fi + if docker buildx imagetools inspect "${ref}:${GOLDEN_TAG}" >/dev/null 2>&1; then + return 0 + fi + + docker buildx prune --all --force || true + docker builder prune --all --force || true + docker buildx build \ + -f ./docker/Dockerfile \ + --target "${target}" \ + "${build_args[@]}" \ + --push \ + -t "${ref}:${GOLDEN_TAG}" \ + -t "${ref}:latest" \ + . + } + + ensure_golden "web-deps-base" "${WEB_GOLDEN_REF}" + ensure_golden "celery-deps-base" "${CELERY_GOLDEN_REF}" + docker buildx build \ -f ./docker/Dockerfile \ --target runtime-web \ --build-arg INSTALL_DEV=false \ + --build-arg GOLDEN_WEB_IMAGE="${WEB_GOLDEN_REF}:${GOLDEN_TAG}" \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --output type=image,push=true,oci-mediatypes=false \ - --provenance=false \ - --sbom=false \ + --push \ "${WEB_TAGS[@]}" \ . @@ -271,16 +368,18 @@ jobs: -f ./docker/Dockerfile \ --target runtime-celery \ --build-arg INSTALL_DEV=false \ + --build-arg GOLDEN_CELERY_IMAGE="${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --output type=image,push=true,oci-mediatypes=false \ - --provenance=false \ - --sbom=false \ + --push \ "${CELERY_TAGS[@]}" \ . { echo "Registry API: ${REGISTRY_API_URL}" + echo "Golden images:" + echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}" + echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" echo "Pushed images:" echo "- ${WEB_REF}:${BRANCH_TAG}" echo "- ${WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a327cb9..235c435 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,13 +46,6 @@ repos: - id: check-merge-conflict - id: detect-private-key - - repo: https://github.com/hadolint/hadolint - rev: v2.12.0 - hooks: - - id: hadolint-docker - name: hadolint dockerfiles - files: ^docker/.*Dockerfile.*$ - - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.11.0.1 hooks: @@ -61,6 +54,11 @@ repos: - repo: local hooks: + - id: hadolint-docker + name: hadolint dockerfiles + entry: bash -c 'for file in "$@"; do docker run --rm -i ghcr.io/hadolint/hadolint:latest hadolint - < "$file"; done' -- + language: system + files: ^docker/.*Dockerfile.*$ - id: django-check-migrations name: django check migrations entry: ./scripts/check-migrations.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 6dd4b20..76bf1c8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,11 @@ -FROM python:3.11-slim-bookworm AS base +ARG PYTHON_IMAGE=python:3.11-slim-bookworm +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.7.2 +ARG GOLDEN_WEB_IMAGE=web-deps-base +ARG GOLDEN_CELERY_IMAGE=celery-deps-base + +FROM ${UV_IMAGE} AS uv-bin + +FROM ${PYTHON_IMAGE} AS base ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ @@ -11,13 +18,10 @@ WORKDIR /app RUN groupadd -r appgroup && useradd -r -g appgroup -m appuser -# Install uv binary. -COPY --from=ghcr.io/astral-sh/uv:0.7.2 /uv /uvx /usr/local/bin/ +COPY --from=uv-bin /uv /uvx /usr/local/bin/ -FROM base AS builder - -ARG INSTALL_DEV=false +FROM base AS builder-base # hadolint ignore=DL3008 RUN apt-get update \ @@ -34,14 +38,29 @@ RUN apt-get update \ COPY pyproject.toml uv.lock ./ -RUN if [ "${INSTALL_DEV}" = "true" ]; then \ - uv sync --frozen --no-install-project --dev; \ - else \ - uv sync --frozen --no-install-project; \ - fi + +FROM builder-base AS prod-deps-base + +RUN uv sync --frozen --no-install-project -FROM base AS runtime-base +FROM builder-base AS ci-deps-build + +RUN uv sync --frozen --no-install-project --dev + + +FROM base AS ci-deps-base + +COPY --from=ci-deps-build /app/.venv /app/.venv + +ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONPATH=/workspace/src \ + DJANGO_SETTINGS_MODULE=settings.test + + +FROM ci-deps-base AS web-deps-base + +USER root # hadolint ignore=DL3008 RUN apt-get update \ @@ -54,13 +73,8 @@ RUN apt-get update \ zlib1g \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/.venv /app/.venv -COPY src/ ./src/ -COPY docker/scripts/ ./docker/scripts/ - RUN mkdir -p logs media staticfiles input/fns input/fns/processed input/fns/failed src/static \ - && chmod +x /app/docker/scripts/*.sh \ - && chown -R appuser:appgroup /app + && chown -R appuser:appgroup logs media staticfiles input src/static ENV PATH="/app/.venv/bin:${PATH}" \ PYTHONPATH=/app/src \ @@ -94,13 +108,7 @@ ENV PATH="/app/.venv/bin:${PATH}" \ USER appuser -FROM runtime-base AS runtime-web - -EXPOSE 8000 -CMD ["/app/docker/scripts/start-web.sh"] - - -FROM runtime-base AS runtime-celery +FROM ${GOLDEN_WEB_IMAGE} AS celery-deps-base USER root @@ -129,7 +137,36 @@ RUN apt-get update \ ENV PLAYWRIGHT_BROWSERS_PATH=/app/.playwright RUN python -m playwright install chromium \ - && chown -R appuser:appgroup /app + && chown -R appuser:appgroup /app/.playwright + +USER appuser + + +FROM ${GOLDEN_WEB_IMAGE} AS runtime-web + +WORKDIR /app +USER root +COPY src/ ./src/ +COPY docker/scripts/ ./docker/scripts/ +RUN mkdir -p logs media staticfiles input/fns input/fns/processed input/fns/failed src/static \ + && chmod +x /app/docker/scripts/*.sh \ + && chown -R appuser:appgroup logs media staticfiles input src/static docker/scripts + +USER appuser + +EXPOSE 8000 +CMD ["/app/docker/scripts/start-web.sh"] + + +FROM ${GOLDEN_CELERY_IMAGE} AS runtime-celery + +WORKDIR /app +USER root +COPY src/ ./src/ +COPY docker/scripts/ ./docker/scripts/ +RUN mkdir -p logs media staticfiles input/fns input/fns/processed input/fns/failed src/static \ + && chmod +x /app/docker/scripts/*.sh \ + && chown -R appuser:appgroup logs media staticfiles input src/static docker/scripts USER appuser