diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index b47e844..430b5c5 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -11,20 +11,6 @@ on: branches: - main - dev - workflow_dispatch: - inputs: - manual_action: - description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start" - required: true - default: "noop" - dokploy_target: - description: "Dokploy dev target: all, web, worker, or beat" - required: true - default: "all" - cleanup_confirm: - description: "Type CLEAN_DEV_DB to drop and recreate the dev database as UTF8" - required: false - default: "" concurrency: group: mostovik-backend-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} @@ -48,119 +34,10 @@ env: PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: - manual_action_noop: - name: Manual Action Help - runs-on: ubuntu-latest - timeout-minutes: 1 - if: | - github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/dev' && - (github.event.inputs.manual_action == '' || github.event.inputs.manual_action == 'noop') - - steps: - - name: Show manual action usage - run: | - set -euo pipefail - { - echo "No manual action selected." - echo "For dev DB cleanup run with:" - echo "- manual_action=cleanup_dev_database" - echo "- cleanup_confirm=CLEAN_DEV_DB" - echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat." - echo "For base image refresh run with manual_action=build_golden_images." - echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat." - } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" - - build_golden_images: - name: Build Golden Images - runs-on: ubuntu-latest - timeout-minutes: 60 - if: | - github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/dev' && - github.event.inputs.manual_action == 'build_golden_images' - - steps: - - name: Checkout code - run: | - set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Build and push golden images - env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} - run: | - set -euo pipefail - - REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" - CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}" - WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}" - CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}" - REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" - REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" - - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY - export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" - export no_proxy="${no_proxy:-},${REGISTRY_HOST}" - - if [ -z "${REGISTRY_PASSWORD}" ]; then - echo "REGISTRY_TOKEN secret is not set and GITEA_TOKEN fallback is empty" >&2 - exit 1 - fi - - echo "${REGISTRY_PASSWORD}" \ - | docker login "${REGISTRY_HOST}" \ - -u "${REGISTRY_USER}" \ - --password-stdin - - if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then - docker buildx create --name mostovik-builder --use - else - docker buildx use mostovik-builder - fi - docker buildx inspect --bootstrap - - docker buildx prune --all --force || true - docker builder prune --all --force || true - - build_golden() { - local target="$1" - local ref="$2" - local build_args=() - if [ "${target}" = "celery-deps-base" ]; then - build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}") - fi - docker buildx build \ - -f ./docker/Dockerfile \ - --target "${target}" \ - "${build_args[@]}" \ - --push \ - -t "${ref}:${GOLDEN_TAG}" \ - -t "${ref}:latest" \ - . - } - - build_golden "ci-deps-base" "${CI_GOLDEN_REF}" - build_golden "web-deps-base" "${WEB_GOLDEN_REF}" - build_golden "celery-deps-base" "${CELERY_GOLDEN_REF}" - - { - echo "Golden images pushed:" - echo "- ${CI_GOLDEN_REF}:${GOLDEN_TAG}" - echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}" - echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" - } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" - quality: name: Quality Gate runs-on: ubuntu-latest timeout-minutes: 25 - if: ${{ github.event_name != 'workflow_dispatch' }} steps: - name: Checkout code @@ -245,14 +122,36 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 needs: [quality] - if: | - github.event_name == 'push' && - (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && - needs.quality.result == 'success' && - !contains(github.event.head_commit.message, '#no_image') + if: needs.quality.result == 'success' steps: + - name: Check whether image build is required + env: + HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + set -euo pipefail + + if [ "${GITHUB_EVENT_NAME}" != "push" ]; then + echo "Skip image build for ${GITHUB_EVENT_NAME}" + exit 0 + fi + + if [ "${GITHUB_REF}" != "refs/heads/dev" ] && [ "${GITHUB_REF}" != "refs/heads/main" ]; then + echo "Skip image build for ${GITHUB_REF}" + exit 0 + fi + + case "${HEAD_COMMIT_MESSAGE:-}" in + *"#no_image"*) + echo "Skip image build because commit message contains #no_image" + exit 0 + ;; + esac + + echo "Image build is required for ${GITHUB_REF}" + - name: Checkout code + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }} run: | set -euo pipefail REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" @@ -261,6 +160,7 @@ jobs: git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - name: Free Docker build space + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }} run: | set -euo pipefail docker system df || true @@ -270,6 +170,7 @@ jobs: docker system df || true - name: Build and push branch images + if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }} env: GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} REGISTRY_USER: ${{ secrets.REGISTRY_USER }} @@ -458,19 +359,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 needs: [build_push] - if: | - github.event_name == 'push' && - github.ref == 'refs/heads/dev' && - needs.build_push.result == 'success' && - !contains(github.event.head_commit.message, '#no_deploy') + if: needs.build_push.result == 'success' steps: - name: Trigger dev Dokploy webhooks env: DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} + HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | set -euo pipefail + if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then + echo "Skip Dokploy dev deploy for ${GITHUB_REF}" + exit 0 + fi + + case "${HEAD_COMMIT_MESSAGE:-}" in + *"#no_deploy"* | *"#no_image"*) + echo "Skip Dokploy dev deploy because commit message disables deploy or image build" + exit 0 + ;; + esac + call_webhook() { service_name="$1" webhook_url="$2" @@ -568,404 +478,3 @@ jobs: echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" - - dokploy_dev_start: - name: Start Dev Containers in Dokploy - runs-on: ubuntu-latest - timeout-minutes: 5 - if: | - github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/dev' && - github.event.inputs.manual_action == 'dokploy_start' - - steps: - - name: Trigger Dokploy webhooks - env: - DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }} - DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} - run: | - set -euo pipefail - - TARGET="${DOKPLOY_TARGET:-all}" - case "${TARGET}" in - all|web|worker|celery|beat) ;; - *) - echo "dokploy_target must be one of: all, web, worker, beat" >&2 - exit 1 - ;; - esac - - call_webhook() { - service_name="$1" - webhook_url="$2" - target="$3" - - if [ -z "${webhook_url}" ]; then - echo "Dokploy webhook for ${service_name} is not configured" >&2 - return 1 - fi - - AUTH_HEADER=() - if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then - AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") - fi - - PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY' - import json - import os - - repository = os.environ.get("GITHUB_REPOSITORY", "") - repository_name = repository.rsplit("/", 1)[-1] - branch = os.environ.get("GITHUB_REF_NAME") or "dev" - sha = os.environ.get("GITHUB_SHA") or "" - server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") - repository_url = f"{server_url}/{repository}" if server_url and repository else "" - image_tag = ( - f"{branch.replace('/', '-')}-{sha[:7]}" - if branch and sha - else branch or "dev" - ) - celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:{image_tag}" - payload = { - "ref": f"refs/heads/{branch}", - "after": sha, - "checkout_sha": sha, - "repository": { - "name": repository_name, - "full_name": repository, - "html_url": repository_url, - "clone_url": f"{repository_url}.git" if repository_url else "", - }, - "sender": {"login": os.environ.get("GITHUB_ACTOR")}, - "pusher": {"name": os.environ.get("GITHUB_ACTOR")}, - "head_commit": { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - }, - "commits": [ - { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - } - ], - "project": repository, - "branch": branch, - "sha": sha, - "actor": os.environ.get("GITHUB_ACTOR"), - "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), - "image_tag": image_tag, - "images": { - "web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:{image_tag}", - "worker": celery_image, - "beat": celery_image, - }, - } - print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) - PY - ) - - echo "Trigger Dokploy for ${service_name}" - RESPONSE=$(curl -fsS \ - --connect-timeout 5 \ - --max-time 30 \ - --retry 2 \ - --retry-delay 2 \ - -X POST \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gogs-Event: push" \ - -H "X-GitHub-Event: push" \ - "${AUTH_HEADER[@]}" \ - --data "${PAYLOAD}" \ - "${webhook_url}") - printf '%s\n' "${RESPONSE}" - if printf '%s' "${RESPONSE}" | grep -qi "Branch Not Match"; then - echo "Dokploy rejected ${service_name}: branch did not match" >&2 - exit 1 - fi - } - - triggered=0 - - if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "web" ]; then - call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" - triggered=1 - fi - - if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "worker" ] || [ "${TARGET}" = "celery" ]; then - call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" - triggered=1 - fi - - if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "beat" ]; then - call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" - triggered=1 - fi - - if [ "${triggered}" -ne 1 ]; then - echo "No Dokploy webhook was triggered" >&2 - exit 1 - fi - - { - echo "Dokploy dev trigger completed." - echo "Target: ${TARGET}" - echo "Registry API: ${REGISTRY_API_URL}" - echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev" - echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" - echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" - } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" - - cleanup_dev_database: - name: Drop and Recreate Dev Database - runs-on: ubuntu-latest - timeout-minutes: 15 - if: | - github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/dev' && - github.event.inputs.manual_action == 'cleanup_dev_database' - env: - POSTGRES_HOST: "10.10.0.114" - POSTGRES_PORT: "5432" - POSTGRES_DB: "mostovik" - POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: "postgres" - - steps: - - name: Validate confirmation - env: - CONFIRM: ${{ github.event.inputs.cleanup_confirm }} - run: | - set -euo pipefail - if [ "${CONFIRM}" != "CLEAN_DEV_DB" ]; then - echo "Manual confirmation must be exactly CLEAN_DEV_DB" >&2 - exit 1 - fi - - - name: Install PostgreSQL client - run: | - set -euo pipefail - APT_RUNNER=() - if [ "$(id -u)" -ne 0 ]; then - APT_RUNNER=(sudo) - fi - - export DEBIAN_FRONTEND=noninteractive - "${APT_RUNNER[@]}" apt-get update - "${APT_RUNNER[@]}" apt-get install -y postgresql-client - - - name: Drop and recreate UTF8 database - run: | - set -euo pipefail - export PGPASSWORD="${POSTGRES_PASSWORD}" - - DB_EXISTS=$(psql \ - --set ON_ERROR_STOP=1 \ - --host="${POSTGRES_HOST}" \ - --port="${POSTGRES_PORT}" \ - --username="${POSTGRES_USER}" \ - --dbname=postgres \ - --tuples-only \ - --no-align \ - --set=dbname="${POSTGRES_DB}" \ - <<'SQL' - SELECT 1 FROM pg_database WHERE datname = :'dbname'; - SQL - ) - - if [ "${DB_EXISTS:-}" = "1" ]; then - echo "Closing active connections to ${POSTGRES_DB}" - psql \ - --set ON_ERROR_STOP=1 \ - --host="${POSTGRES_HOST}" \ - --port="${POSTGRES_PORT}" \ - --username="${POSTGRES_USER}" \ - --dbname=postgres \ - --set=dbname="${POSTGRES_DB}" \ - <<'SQL' - ALTER DATABASE :"dbname" WITH ALLOW_CONNECTIONS false; - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = :'dbname' - AND pid <> pg_backend_pid(); - SQL - fi - - echo "Dropping and recreating ${POSTGRES_DB} with UTF8 encoding" - psql \ - --set ON_ERROR_STOP=1 \ - --host="${POSTGRES_HOST}" \ - --port="${POSTGRES_PORT}" \ - --username="${POSTGRES_USER}" \ - --dbname=postgres \ - --set=dbname="${POSTGRES_DB}" \ - --set=dbuser="${POSTGRES_USER}" \ - <<'SQL' - DROP DATABASE IF EXISTS :"dbname"; - CREATE DATABASE :"dbname" WITH OWNER :"dbuser" TEMPLATE template0 ENCODING 'UTF8'; - SQL - - DB_ENCODING=$(psql \ - --set ON_ERROR_STOP=1 \ - --host="${POSTGRES_HOST}" \ - --port="${POSTGRES_PORT}" \ - --username="${POSTGRES_USER}" \ - --dbname=postgres \ - --tuples-only \ - --no-align \ - --set=dbname="${POSTGRES_DB}" \ - <<'SQL' - SELECT pg_encoding_to_char(encoding) - FROM pg_database - WHERE datname = :'dbname'; - SQL - ) - - printf '%s\n' "${DB_ENCODING}" | tee /tmp/mostovik-db-encoding - - if [ "${DB_ENCODING}" != "UTF8" ]; then - echo "Database ${POSTGRES_DB} is not UTF8 after cleanup" >&2 - exit 1 - fi - - - name: Trigger Dokploy after database recreate - env: - DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} - run: | - set -euo pipefail - - call_webhook() { - service_name="$1" - webhook_url="$2" - target="$3" - - AUTH_HEADER=() - if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then - AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") - fi - - PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY' - import json - import os - - repository = os.environ.get("GITHUB_REPOSITORY", "") - repository_name = repository.rsplit("/", 1)[-1] - branch = os.environ.get("GITHUB_REF_NAME") or "dev" - sha = os.environ.get("GITHUB_SHA") or "" - server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") - repository_url = f"{server_url}/{repository}" if server_url and repository else "" - image_tag = ( - f"{branch.replace('/', '-')}-{sha[:7]}" - if branch and sha - else branch or "dev" - ) - celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:{image_tag}" - payload = { - "ref": f"refs/heads/{branch}", - "after": sha, - "checkout_sha": sha, - "repository": { - "name": repository_name, - "full_name": repository, - "html_url": repository_url, - "clone_url": f"{repository_url}.git" if repository_url else "", - }, - "sender": {"login": os.environ.get("GITHUB_ACTOR")}, - "pusher": {"name": os.environ.get("GITHUB_ACTOR")}, - "head_commit": { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - }, - "commits": [ - { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - } - ], - "project": repository, - "branch": branch, - "sha": sha, - "actor": os.environ.get("GITHUB_ACTOR"), - "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), - "image_tag": image_tag, - "images": { - "web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:{image_tag}", - "worker": celery_image, - "beat": celery_image, - }, - } - print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) - PY - ) - - echo "Trigger Dokploy for ${service_name}" - RESPONSE=$(curl -fsS \ - --connect-timeout 5 \ - --max-time 30 \ - --retry 2 \ - --retry-delay 2 \ - -X POST \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gogs-Event: push" \ - -H "X-GitHub-Event: push" \ - "${AUTH_HEADER[@]}" \ - --data "${PAYLOAD}" \ - "${webhook_url}") - printf '%s\n' "${RESPONSE}" - if printf '%s' "${RESPONSE}" | grep -qi "Branch Not Match"; then - echo "Dokploy rejected ${service_name}: branch did not match" >&2 - exit 1 - fi - } - - wait_for_migrations() { - export PGPASSWORD="${POSTGRES_PASSWORD}" - for attempt in $(seq 1 60); do - SCHEMA_STATE=$(psql \ - --set ON_ERROR_STOP=1 \ - --host="${POSTGRES_HOST}" \ - --port="${POSTGRES_PORT}" \ - --username="${POSTGRES_USER}" \ - --dbname="${POSTGRES_DB}" \ - --tuples-only \ - --no-align \ - <<'SQL' - SELECT CASE - WHEN to_regclass('public.django_migrations') IS NOT NULL - AND to_regclass('public.core_backgroundjob') IS NOT NULL - THEN 'ready' - ELSE 'waiting' - END; - SQL - ) - if [ "${SCHEMA_STATE}" = "ready" ]; then - echo "Database schema is ready after web deploy" - return 0 - fi - echo "Waiting for web migrations (${attempt}/60)" - sleep 5 - done - - echo "Database schema was not ready after web deploy" >&2 - exit 1 - } - - call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" - wait_for_migrations - call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" - call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" - - - name: Summary - run: | - set -euo pipefail - { - echo "Dev database was dropped and recreated." - echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" - echo "Encoding: UTF8" - echo "Dokploy web/worker/beat deploy was triggered." - } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" diff --git a/.gitea/workflows/manual-dev-actions.yml b/.gitea/workflows/manual-dev-actions.yml new file mode 100644 index 0000000..69ba4fc --- /dev/null +++ b/.gitea/workflows/manual-dev-actions.yml @@ -0,0 +1,66 @@ +name: Manual Dev Actions + +on: + workflow_dispatch: + inputs: + manual_action: + description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start" + required: true + default: "noop" + dokploy_target: + description: "Dokploy dev target: all, web, worker, or beat" + required: true + default: "all" + cleanup_confirm: + description: "Type CLEAN_DEV_DB to drop and recreate the dev database as UTF8" + required: false + default: "" + +concurrency: + group: mostovik-backend-manual-dev-${{ github.ref }} + cancel-in-progress: false + +env: + REGISTRY_API_URL: "https://registry.dev.nii-ecos.ru/v2/" + REGISTRY_HOST: "registry.dev.nii-ecos.ru" + REGISTRY_NAMESPACE: "${{ github.repository_owner }}" + WEB_IMAGE: "mostovik-backend-web" + CELERY_IMAGE: "mostovik-backend-celery" + CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden" + WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden" + CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden" + GOLDEN_TAG: "py311-uv0.7.2" + DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" + DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" + DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" + +jobs: + manual_dev_action: + name: Run Manual Dev Action + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + MANUAL_ACTION: ${{ github.event.inputs.manual_action }} + DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }} + CLEANUP_CONFIRM: ${{ github.event.inputs.cleanup_confirm }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} + DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} + POSTGRES_HOST: "10.10.0.114" + POSTGRES_PORT: "5432" + POSTGRES_DB: "mostovik" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "postgres" + + steps: + - name: Checkout code + run: | + set -euo pipefail + REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" + BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . + git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" + + - name: Run selected action + run: bash scripts/ci/manual_dev_action.sh diff --git a/scripts/ci/manual_dev_action.sh b/scripts/ci/manual_dev_action.sh new file mode 100755 index 0000000..cd6fed5 --- /dev/null +++ b/scripts/ci/manual_dev_action.sh @@ -0,0 +1,410 @@ +#!/usr/bin/env bash +set -euo pipefail + +MANUAL_ACTION="${MANUAL_ACTION:-noop}" +DOKPLOY_TARGET="${DOKPLOY_TARGET:-all}" +CLEANUP_CONFIRM="${CLEANUP_CONFIRM:-}" +SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +require_dev_branch() { + if [ "${GITHUB_REF:-}" != "refs/heads/dev" ]; then + echo "Manual dev actions are allowed only from dev branch, got ${GITHUB_REF:-unknown}" >&2 + exit 1 + fi +} + +write_usage() { + { + echo "No manual action selected." + echo "For dev DB cleanup run with:" + echo "- manual_action=cleanup_dev_database" + echo "- cleanup_confirm=CLEAN_DEV_DB" + echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat." + echo "For base image refresh run with manual_action=build_golden_images." + echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat." + } >>"${SUMMARY_FILE}" +} + +registry_login() { + local registry_user registry_password + + registry_user="${REGISTRY_USER:-${GITHUB_ACTOR}}" + registry_password="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" + + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY + export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" + export no_proxy="${no_proxy:-},${REGISTRY_HOST}" + + if [ -z "${registry_password}" ]; then + echo "REGISTRY_TOKEN secret is not set and GITEA_TOKEN fallback is empty" >&2 + exit 1 + fi + + echo "${registry_password}" \ + | docker login "${REGISTRY_HOST}" \ + -u "${registry_user}" \ + --password-stdin +} + +ensure_buildx() { + if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then + docker buildx create --name mostovik-builder --use + else + docker buildx use mostovik-builder + fi + docker buildx inspect --bootstrap +} + +build_golden_images() { + local registry_path ci_golden_ref web_golden_ref celery_golden_ref + + registry_path="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + ci_golden_ref="${registry_path}/${CI_GOLDEN_IMAGE}" + web_golden_ref="${registry_path}/${WEB_GOLDEN_IMAGE}" + celery_golden_ref="${registry_path}/${CELERY_GOLDEN_IMAGE}" + + registry_login + ensure_buildx + + docker buildx prune --all --force || true + docker builder prune --all --force || true + + build_golden() { + local target="$1" + local ref="$2" + local build_args=() + + if [ "${target}" = "celery-deps-base" ]; then + build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${web_golden_ref}:${GOLDEN_TAG}") + fi + + docker buildx build \ + -f ./docker/Dockerfile \ + --target "${target}" \ + "${build_args[@]}" \ + --push \ + -t "${ref}:${GOLDEN_TAG}" \ + -t "${ref}:latest" \ + . + } + + build_golden "ci-deps-base" "${ci_golden_ref}" + build_golden "web-deps-base" "${web_golden_ref}" + build_golden "celery-deps-base" "${celery_golden_ref}" + + { + echo "Golden images pushed:" + echo "- ${ci_golden_ref}:${GOLDEN_TAG}" + echo "- ${web_golden_ref}:${GOLDEN_TAG}" + echo "- ${celery_golden_ref}:${GOLDEN_TAG}" + } >>"${SUMMARY_FILE}" +} + +dokploy_payload() { + CURRENT_DOKPLOY_TARGET="$1" python3 - <<'PY' +import json +import os + +repository = os.environ.get("GITHUB_REPOSITORY", "") +repository_name = repository.rsplit("/", 1)[-1] +branch = os.environ.get("GITHUB_REF_NAME") or "dev" +sha = os.environ.get("GITHUB_SHA") or "" +server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") +repository_url = f"{server_url}/{repository}" if server_url and repository else "" +image_tag = f"{branch.replace('/', '-')}-{sha[:7]}" if branch and sha else branch or "dev" +celery_image = ( + f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/" + f"{os.environ['CELERY_IMAGE']}:{image_tag}" +) +payload = { + "ref": f"refs/heads/{branch}", + "after": sha, + "checkout_sha": sha, + "repository": { + "name": repository_name, + "full_name": repository, + "html_url": repository_url, + "clone_url": f"{repository_url}.git" if repository_url else "", + }, + "sender": {"login": os.environ.get("GITHUB_ACTOR")}, + "pusher": {"name": os.environ.get("GITHUB_ACTOR")}, + "head_commit": { + "id": sha, + "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", + "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", + }, + "commits": [ + { + "id": sha, + "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", + "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", + } + ], + "project": repository, + "branch": branch, + "sha": sha, + "actor": os.environ.get("GITHUB_ACTOR"), + "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), + "image_tag": image_tag, + "images": { + "web": ( + f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/" + f"{os.environ['WEB_IMAGE']}:{image_tag}" + ), + "worker": celery_image, + "beat": celery_image, + }, +} +print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) +PY +} + +call_dokploy_webhook() { + local service_name="$1" + local webhook_url="$2" + local target="$3" + local auth_header=() + local payload response + + if [ -z "${webhook_url}" ]; then + echo "Dokploy webhook for ${service_name} is not configured" >&2 + exit 1 + fi + + if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then + auth_header=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") + fi + + payload="$(dokploy_payload "${target}")" + + echo "Trigger Dokploy for ${service_name}" + response="$(curl -fsS \ + --connect-timeout 5 \ + --max-time 30 \ + --retry 2 \ + --retry-delay 2 \ + -X POST \ + -H "Content-Type: application/json" \ + -H "X-Gitea-Event: push" \ + -H "X-Gogs-Event: push" \ + -H "X-GitHub-Event: push" \ + "${auth_header[@]}" \ + --data "${payload}" \ + "${webhook_url}")" + printf '%s\n' "${response}" + + if printf '%s' "${response}" | grep -qi "Branch Not Match"; then + echo "Dokploy rejected ${service_name}: branch did not match" >&2 + exit 1 + fi +} + +trigger_dokploy() { + local target="$1" + local triggered=0 + + case "${target}" in + all | web | worker | celery | beat) ;; + *) + echo "dokploy_target must be one of: all, web, worker, beat" >&2 + exit 1 + ;; + esac + + if [ "${target}" = "all" ] || [ "${target}" = "web" ]; then + call_dokploy_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" + triggered=1 + fi + + if [ "${target}" = "all" ] || [ "${target}" = "worker" ] || [ "${target}" = "celery" ]; then + call_dokploy_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" + triggered=1 + fi + + if [ "${target}" = "all" ] || [ "${target}" = "beat" ]; then + call_dokploy_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" + triggered=1 + fi + + if [ "${triggered}" -ne 1 ]; then + echo "No Dokploy webhook was triggered" >&2 + exit 1 + fi + + { + echo "Dokploy dev trigger completed." + echo "Target: ${target}" + echo "Registry API: ${REGISTRY_API_URL}" + echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev" + echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" + echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" + } >>"${SUMMARY_FILE}" +} + +install_postgres_client() { + local apt_runner=() + + if command -v psql >/dev/null 2>&1; then + return 0 + fi + + if [ "$(id -u)" -ne 0 ]; then + apt_runner=(sudo) + fi + + export DEBIAN_FRONTEND=noninteractive + "${apt_runner[@]}" apt-get update + "${apt_runner[@]}" apt-get install -y postgresql-client +} + +drop_and_recreate_database() { + local db_exists db_encoding + + export PGPASSWORD="${POSTGRES_PASSWORD}" + + db_exists="$(psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=dbname="${POSTGRES_DB}" \ + <<'SQL' +SELECT 1 FROM pg_database WHERE datname = :'dbname'; +SQL + )" + + if [ "${db_exists:-}" = "1" ]; then + echo "Closing active connections to ${POSTGRES_DB}" + psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --set=dbname="${POSTGRES_DB}" \ + <<'SQL' +ALTER DATABASE :"dbname" WITH ALLOW_CONNECTIONS false; +SELECT pg_terminate_backend(pid) +FROM pg_stat_activity +WHERE datname = :'dbname' + AND pid <> pg_backend_pid(); +SQL + fi + + echo "Dropping and recreating ${POSTGRES_DB} with UTF8 encoding" + psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --set=dbname="${POSTGRES_DB}" \ + --set=dbuser="${POSTGRES_USER}" \ + <<'SQL' +DROP DATABASE IF EXISTS :"dbname"; +CREATE DATABASE :"dbname" WITH OWNER :"dbuser" TEMPLATE template0 ENCODING 'UTF8'; +SQL + + db_encoding="$(psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=dbname="${POSTGRES_DB}" \ + <<'SQL' +SELECT pg_encoding_to_char(encoding) +FROM pg_database +WHERE datname = :'dbname'; +SQL + )" + + printf '%s\n' "${db_encoding}" | tee /tmp/mostovik-db-encoding + + if [ "${db_encoding}" != "UTF8" ]; then + echo "Database ${POSTGRES_DB} is not UTF8 after cleanup" >&2 + exit 1 + fi +} + +wait_for_migrations() { + local schema_state + + export PGPASSWORD="${POSTGRES_PASSWORD}" + + for attempt in $(seq 1 60); do + schema_state="$(psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname="${POSTGRES_DB}" \ + --tuples-only \ + --no-align \ + <<'SQL' +SELECT CASE + WHEN to_regclass('public.django_migrations') IS NOT NULL + AND to_regclass('public.core_backgroundjob') IS NOT NULL + THEN 'ready' + ELSE 'waiting' +END; +SQL + )" + if [ "${schema_state}" = "ready" ]; then + echo "Database schema is ready after web deploy" + return 0 + fi + echo "Waiting for web migrations (${attempt}/60)" + sleep 5 + done + + echo "Database schema was not ready after web deploy" >&2 + exit 1 +} + +cleanup_dev_database() { + if [ "${CLEANUP_CONFIRM}" != "CLEAN_DEV_DB" ]; then + echo "Manual confirmation must be exactly CLEAN_DEV_DB" >&2 + exit 1 + fi + + install_postgres_client + drop_and_recreate_database + call_dokploy_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" + wait_for_migrations + call_dokploy_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" + call_dokploy_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" + + { + echo "Dev database was dropped and recreated." + echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" + echo "Encoding: UTF8" + echo "Dokploy web/worker/beat deploy was triggered." + } >>"${SUMMARY_FILE}" +} + +require_dev_branch + +case "${MANUAL_ACTION}" in + "" | noop) + write_usage + ;; + build_golden_images) + build_golden_images + ;; + dokploy_start) + trigger_dokploy "${DOKPLOY_TARGET}" + ;; + cleanup_dev_database) + cleanup_dev_database + ;; + *) + echo "Unknown manual_action: ${MANUAL_ACTION}" >&2 + exit 1 + ;; +esac