name: CI/CD Pipeline on: push: branches: - main - dev - "feature/**" - "codex/**" pull_request: branches: - main - dev concurrency: group: mostovik-backend-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true env: PYTHON_VERSION: "3.11" REGISTRY_API_URL: "https://registry.dev.nii-ecos.ru/v2/" REGISTRY_HOST: "registry.dev.nii-ecos.ru" REGISTRY_NAMESPACE: "${{ github.repository_owner }}" WEB_IMAGE: "mostovik-backend-web" CELERY_IMAGE: "mostovik-backend-celery" GITEA_REGISTRY_HOST: "git.dev.nii-ecos.ru" DOKPLOY_DEV_WEB_SERVICE_IMAGE: "service-backend-4mbxrs" DOKPLOY_DEV_WORKER_SERVICE_IMAGE: "service-backend-512y9c" DOKPLOY_DEV_BEAT_SERVICE_IMAGE: "service-backend-nvdyoq" CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden" WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden" CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden" GOLDEN_TAG: "py311-uv0.7.2" DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" DOKPLOY_API_URL: "https://deploy.dev.nii-ecos.ru/api" DOKPLOY_DEV_WEB_APPLICATION_ID: "x2l_Twc2z2A4lJhMVqlNg" DOKPLOY_DEV_WORKER_APPLICATION_ID: "m8ECastEeQKhDZVFonUTS" DOKPLOY_DEV_BEAT_APPLICATION_ID: "Ut5e5mcMMslxG9Zrpbp0_" DOKPLOY_DEV_WEB_APP_NAME: "service-backend-4mbxrs" DOKPLOY_DEV_WORKER_APP_NAME: "service-backend-512y9c" DOKPLOY_DEV_BEAT_APP_NAME: "service-backend-nvdyoq" UV_VERSION: "0.7.2" PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: quality: name: Quality Gate runs-on: ubuntu-latest timeout-minutes: 25 steps: - name: Checkout code run: | set -euo pipefail REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - name: Free Docker space run: | set -euo pipefail docker system df || true docker buildx prune --all --force || true docker builder prune --all --force || true docker system prune --all --force --volumes || true docker system df || true - name: Run quality in golden image env: GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} REGISTRY_USER: ${{ secrets.REGISTRY_USER }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} SKIP_LINT: ${{ contains(github.event.head_commit.message, '#no_lint') }} SKIP_TEST: ${{ contains(github.event.head_commit.message, '#no_test') }} run: | set -euo pipefail REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}" REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" export no_proxy="${no_proxy:-},${REGISTRY_HOST}" if [ -z "${REGISTRY_PASSWORD}" ]; then echo "REGISTRY_TOKEN secret is not set and GITEA_TOKEN fallback is empty" >&2 exit 1 fi echo "${REGISTRY_PASSWORD}" \ | docker login "${REGISTRY_HOST}" \ -u "${REGISTRY_USER}" \ --password-stdin if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then docker buildx create --name mostovik-builder --use else docker buildx use mostovik-builder fi docker buildx inspect --bootstrap image_usable() { docker run --rm "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ bash -c '/app/.venv/bin/ruff --version >/dev/null && /app/.venv/bin/python -m pytest --version >/dev/null' } if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1 || ! image_usable; then docker buildx prune --all --force || true docker builder prune --all --force || true docker buildx build \ -f ./docker/Dockerfile \ --target ci-deps-base \ --push \ -t "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ -t "${CI_GOLDEN_REF}:latest" \ . docker pull "${CI_GOLDEN_REF}:${GOLDEN_TAG}" fi tar --exclude=.git -cf - . \ | docker run --rm -i \ -e DJANGO_SETTINGS_MODULE=settings.test \ -e SECRET_KEY=test-secret-key-for-ci \ -e SKIP_LINT="${SKIP_LINT}" \ -e SKIP_TEST="${SKIP_TEST}" \ "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ bash -c ' set -euo pipefail mkdir -p /workspace tar -xf - -C /workspace cd /workspace export PATH="/app/.venv/bin:${PATH}" export PYTHONPATH="/workspace/src:${PYTHONPATH:-}" if [ "${SKIP_LINT}" != "true" ]; then /app/.venv/bin/ruff check src /app/.venv/bin/ruff format src --check fi if [ "${SKIP_TEST}" != "true" ]; then /app/.venv/bin/python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q /app/.venv/bin/python -m pytest tests/test_api_inventory_e2e.py -q fi ' build_push: name: Build and Push Images runs-on: ubuntu-latest timeout-minutes: 45 needs: [quality] if: needs.quality.result == 'success' steps: - name: Check whether image build is required env: HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | set -euo pipefail if [ "${GITHUB_EVENT_NAME}" != "push" ]; then echo "Skip image build for ${GITHUB_EVENT_NAME}" exit 0 fi if [ "${GITHUB_REF}" != "refs/heads/dev" ] && [ "${GITHUB_REF}" != "refs/heads/main" ]; then echo "Skip image build for ${GITHUB_REF}" exit 0 fi case "${HEAD_COMMIT_MESSAGE:-}" in *"#no_image"*) echo "Skip image build because commit message contains #no_image" exit 0 ;; esac echo "Image build is required for ${GITHUB_REF}" - name: Checkout code if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }} run: | set -euo pipefail REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - name: Free Docker build space if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }} run: | set -euo pipefail docker system df || true docker buildx prune --all --force || true docker builder prune --all --force || true docker system prune --all --force --volumes || true docker system df || true - name: Build and push branch images if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }} env: GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} REGISTRY_USER: ${{ secrets.REGISTRY_USER }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} GITEA_REGISTRY_TOKEN: ${{ secrets.GITEA_REGISTRY_TOKEN }} run: | set -euo pipefail BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-branch}}" BRANCH_TAG=$(printf '%s' "${BRANCH}" \ | tr '[:upper:]' '[:lower:]' \ | sed -E 's#[/[:space:]]+#-#g; s#[^a-z0-9_.-]+#-#g; s#^-+##; s#-+$##') BRANCH_TAG="${BRANCH_TAG:-branch}" BRANCH_TAG=$(printf '%.120s' "${BRANCH_TAG}") SHA_SHORT=$(printf '%s' "${GITHUB_SHA}" | cut -c1-7) REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" WEB_REF="${REGISTRY_PATH}/${WEB_IMAGE}" CELERY_REF="${REGISTRY_PATH}/${CELERY_IMAGE}" WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}" CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}" DOKPLOY_REGISTRY_PATH="${GITEA_REGISTRY_HOST}/${REGISTRY_NAMESPACE}" DOKPLOY_WEB_REF="${DOKPLOY_REGISTRY_PATH}/${DOKPLOY_DEV_WEB_SERVICE_IMAGE}" DOKPLOY_WORKER_REF="${DOKPLOY_REGISTRY_PATH}/${DOKPLOY_DEV_WORKER_SERVICE_IMAGE}" DOKPLOY_BEAT_REF="${DOKPLOY_REGISTRY_PATH}/${DOKPLOY_DEV_BEAT_SERVICE_IMAGE}" REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" GITEA_ALIAS_PUSH_ENABLED="false" unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST},${GITEA_REGISTRY_HOST}" export no_proxy="${no_proxy:-},${REGISTRY_HOST},${GITEA_REGISTRY_HOST}" if [ -z "${REGISTRY_PASSWORD}" ]; then echo "REGISTRY_TOKEN secret is not set and GITEA_TOKEN fallback is empty" >&2 exit 1 fi echo "${REGISTRY_PASSWORD}" \ | docker login "${REGISTRY_HOST}" \ -u "${REGISTRY_USER}" \ --password-stdin if [ -n "${GITEA_REGISTRY_TOKEN:-}" ]; then echo "${GITEA_REGISTRY_TOKEN}" \ | docker login "${GITEA_REGISTRY_HOST}" \ -u "${GITHUB_ACTOR}" \ --password-stdin GITEA_ALIAS_PUSH_ENABLED="true" else echo "GITEA_REGISTRY_TOKEN is not set; skip Dokploy-compatible git.dev image aliases" fi WEB_TAGS=( -t "${WEB_REF}:${BRANCH_TAG}" -t "${WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" ) CELERY_TAGS=( -t "${CELERY_REF}:${BRANCH_TAG}" -t "${CELERY_REF}:${BRANCH_TAG}-${SHA_SHORT}" ) if [ "${GITEA_ALIAS_PUSH_ENABLED}" = "true" ]; then WEB_TAGS+=( -t "${DOKPLOY_WEB_REF}:latest" -t "${DOKPLOY_WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" ) CELERY_TAGS+=( -t "${DOKPLOY_WORKER_REF}:latest" -t "${DOKPLOY_WORKER_REF}:${BRANCH_TAG}-${SHA_SHORT}" -t "${DOKPLOY_BEAT_REF}:latest" -t "${DOKPLOY_BEAT_REF}:${BRANCH_TAG}-${SHA_SHORT}" ) fi if [ "${GITHUB_REF_NAME}" = "main" ]; then WEB_TAGS+=(-t "${WEB_REF}:latest") CELERY_TAGS+=(-t "${CELERY_REF}:latest") fi if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then docker buildx create --name mostovik-builder --use else docker buildx use mostovik-builder fi docker buildx inspect --bootstrap ensure_golden() { local target="$1" local ref="$2" local build_args=() if [ "${target}" = "celery-deps-base" ]; then build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}") fi if docker buildx imagetools inspect "${ref}:${GOLDEN_TAG}" >/dev/null 2>&1; then return 0 fi docker buildx prune --all --force || true docker builder prune --all --force || true docker buildx build \ -f ./docker/Dockerfile \ --target "${target}" \ "${build_args[@]}" \ --push \ -t "${ref}:${GOLDEN_TAG}" \ -t "${ref}:latest" \ . } ensure_golden "web-deps-base" "${WEB_GOLDEN_REF}" ensure_golden "celery-deps-base" "${CELERY_GOLDEN_REF}" docker buildx build \ -f ./docker/Dockerfile \ --target runtime-web \ --build-arg INSTALL_DEV=false \ --build-arg GOLDEN_WEB_IMAGE="${WEB_GOLDEN_REF}:${GOLDEN_TAG}" \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ --push \ "${WEB_TAGS[@]}" \ . docker buildx build \ -f ./docker/Dockerfile \ --target runtime-celery \ --build-arg INSTALL_DEV=false \ --build-arg GOLDEN_CELERY_IMAGE="${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ --push \ "${CELERY_TAGS[@]}" \ . { echo "Registry API: ${REGISTRY_API_URL}" echo "Golden images:" echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}" echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" echo "Pushed images:" echo "- ${WEB_REF}:${BRANCH_TAG}" echo "- ${WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" echo "- ${CELERY_REF}:${BRANCH_TAG}" echo "- ${CELERY_REF}:${BRANCH_TAG}-${SHA_SHORT}" if [ "${GITEA_ALIAS_PUSH_ENABLED}" = "true" ]; then echo "Dokploy-compatible aliases:" echo "- ${DOKPLOY_WEB_REF}:latest" echo "- ${DOKPLOY_WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" echo "- ${DOKPLOY_WORKER_REF}:latest" echo "- ${DOKPLOY_WORKER_REF}:${BRANCH_TAG}-${SHA_SHORT}" echo "- ${DOKPLOY_BEAT_REF}:latest" echo "- ${DOKPLOY_BEAT_REF}:${BRANCH_TAG}-${SHA_SHORT}" else echo "Dokploy-compatible aliases skipped: GITEA_REGISTRY_TOKEN is not set." fi } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" notify: name: Internal Notify runs-on: ubuntu-latest timeout-minutes: 1 needs: [quality, build_push] if: ${{ always() && github.event_name != 'workflow_dispatch' }} steps: - name: Send CI status webhook continue-on-error: true env: CI_NOTIFY_WEBHOOK_URL: ${{ secrets.CI_NOTIFY_WEBHOOK_URL }} CI_NOTIFY_TOKEN: ${{ secrets.CI_NOTIFY_TOKEN }} QUALITY_RESULT: ${{ needs.quality.result }} BUILD_PUSH_RESULT: ${{ needs.build_push.result }} run: | set -euo pipefail if [ -z "${CI_NOTIFY_WEBHOOK_URL:-}" ]; then echo "CI_NOTIFY_WEBHOOK_URL is not set; skip internal notification" exit 0 fi STATUS="success" if [ "${QUALITY_RESULT}" != "success" ]; then STATUS="${QUALITY_RESULT}" elif [ "${BUILD_PUSH_RESULT}" = "failure" ] || [ "${BUILD_PUSH_RESULT}" = "cancelled" ]; then STATUS="${BUILD_PUSH_RESULT}" fi export STATUS PAYLOAD=$(python3 - <<'PY' import json import os payload = { "project": os.environ.get("GITHUB_REPOSITORY"), "workflow": os.environ.get("GITHUB_WORKFLOW"), "status": os.environ.get("STATUS"), "branch": os.environ.get("GITHUB_HEAD_REF") or os.environ.get("GITHUB_REF_NAME"), "sha": os.environ.get("GITHUB_SHA"), "actor": os.environ.get("GITHUB_ACTOR"), "server_url": os.environ.get("GITHUB_SERVER_URL"), "run_id": os.environ.get("GITHUB_RUN_ID"), "results": { "quality": os.environ.get("QUALITY_RESULT"), "build_push": os.environ.get("BUILD_PUSH_RESULT"), }, } print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) PY ) AUTH_HEADER=() if [ -n "${CI_NOTIFY_TOKEN:-}" ]; then AUTH_HEADER=(-H "Authorization: Bearer ${CI_NOTIFY_TOKEN}") fi curl -fsS \ --connect-timeout 3 \ --max-time 8 \ --retry 1 \ -H "Content-Type: application/json" \ "${AUTH_HEADER[@]}" \ --data "${PAYLOAD}" \ "${CI_NOTIFY_WEBHOOK_URL}" deploy_dev: name: Deploy Dev in Dokploy runs-on: ubuntu-latest timeout-minutes: 5 needs: [build_push] if: needs.build_push.result == 'success' steps: - name: Checkout code run: | set -euo pipefail REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - name: Deploy prebuilt images in Dokploy env: DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} DOKPLOY_API_TOKEN_FALLBACK: "cmhRpAPDlWPCbwkCdteTgpHuHzhPHCNtZrUcRddsfiHdijmyXKsIIojiBmcVpfpo" GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} REGISTRY_USER: ${{ secrets.REGISTRY_USER }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | set -euo pipefail if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then echo "Skip Dokploy dev deploy for ${GITHUB_REF}" exit 0 fi case "${HEAD_COMMIT_MESSAGE:-}" in *"#no_deploy"* | *"#no_image"*) echo "Skip Dokploy dev deploy because commit message disables deploy or image build" exit 0 ;; esac bash scripts/ci/dokploy_deploy_image.sh all