avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: split manual dev actions from push pipeline
Some checks failed
CI/CD Pipeline / Quality Gate (push) Failing after 1s
Details
CI/CD Pipeline / Build and Push Images (push) Has been skipped
Details
CI/CD Pipeline / Deploy Dev in Dokploy (push) Has been skipped
Details
CI/CD Pipeline / Internal Notify (push) Successful in 1s
Details

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-04-28 21:18:43 +02:00
parent 760929972b
commit 3cd799f365
3 changed files with 519 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

577
.gitea/workflows/ci-cd.yml
View File

@@ -11,20 +11,6 @@ on:
branches:
- main
- dev
workflow_dispatch:
inputs:
manual_action:
description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start"
required: true
default: "noop"
dokploy_target:
description: "Dokploy dev target: all, web, worker, or beat"
required: true
default: "all"
cleanup_confirm:
description: "Type CLEAN_DEV_DB to drop and recreate the dev database as UTF8"
required: false
default: ""
concurrency:
group: mostovik-backend-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
@@ -48,119 +34,10 @@ env:
PIP_DISABLE_PIP_VERSION_CHECK: "1"
jobs:
manual_action_noop:
name: Manual Action Help
runs-on: ubuntu-latest
timeout-minutes: 1
if: |
github.event_name == 'workflow_dispatch' &&
github.ref == 'refs/heads/dev' &&
(github.event.inputs.manual_action == '' || github.event.inputs.manual_action == 'noop')
steps:
- name: Show manual action usage
run: |
set -euo pipefail
{
echo "No manual action selected."
echo "For dev DB cleanup run with:"
echo "- manual_action=cleanup_dev_database"
echo "- cleanup_confirm=CLEAN_DEV_DB"
echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat."
echo "For base image refresh run with manual_action=build_golden_images."
echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat."
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
build_golden_images:
name: Build Golden Images
runs-on: ubuntu-latest
timeout-minutes: 60
if: |
github.event_name == 'workflow_dispatch' &&
github.ref == 'refs/heads/dev' &&
github.event.inputs.manual_action == 'build_golden_images'
steps:
- name: Checkout code
run: |
set -euo pipefail
REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Build and push golden images
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }}
run: |
set -euo pipefail
REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}"
WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}"
CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}"
REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}"
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}"
unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY
export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}"
export no_proxy="${no_proxy:-},${REGISTRY_HOST}"
if [ -z "${REGISTRY_PASSWORD}" ]; then
echo "REGISTRY_TOKEN secret is not set and GITEA_TOKEN fallback is empty" >&2
exit 1
fi
echo "${REGISTRY_PASSWORD}" \
| docker login "${REGISTRY_HOST}" \
-u "${REGISTRY_USER}" \
--password-stdin
if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then
docker buildx create --name mostovik-builder --use
else
docker buildx use mostovik-builder
fi
docker buildx inspect --bootstrap
docker buildx prune --all --force || true
docker builder prune --all --force || true
build_golden() {
local target="$1"
local ref="$2"
local build_args=()
if [ "${target}" = "celery-deps-base" ]; then
build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}")
fi
docker buildx build \
-f ./docker/Dockerfile \
--target "${target}" \
"${build_args[@]}" \
--push \
-t "${ref}:${GOLDEN_TAG}" \
-t "${ref}:latest" \
.
}
build_golden "ci-deps-base" "${CI_GOLDEN_REF}"
build_golden "web-deps-base" "${WEB_GOLDEN_REF}"
build_golden "celery-deps-base" "${CELERY_GOLDEN_REF}"
{
echo "Golden images pushed:"
echo "- ${CI_GOLDEN_REF}:${GOLDEN_TAG}"
echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}"
echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}"
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
quality:
name: Quality Gate
runs-on: ubuntu-latest
timeout-minutes: 25
if: ${{ github.event_name != 'workflow_dispatch' }}
steps:
- name: Checkout code
@@ -245,14 +122,36 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
needs: [quality]
if: |
github.event_name == 'push' &&
(github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') &&
needs.quality.result == 'success' &&
!contains(github.event.head_commit.message, '#no_image')
if: needs.quality.result == 'success'
steps:
- name: Check whether image build is required
env:
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |
set -euo pipefail
if [ "${GITHUB_EVENT_NAME}" != "push" ]; then
echo "Skip image build for ${GITHUB_EVENT_NAME}"
exit 0
fi
if [ "${GITHUB_REF}" != "refs/heads/dev" ] && [ "${GITHUB_REF}" != "refs/heads/main" ]; then
echo "Skip image build for ${GITHUB_REF}"
exit 0
fi
case "${HEAD_COMMIT_MESSAGE:-}" in
*"#no_image"*)
echo "Skip image build because commit message contains #no_image"
exit 0
;;
esac
echo "Image build is required for ${GITHUB_REF}"
- name: Checkout code
if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }}
run: |
set -euo pipefail
REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
@@ -261,6 +160,7 @@ jobs:
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Free Docker build space
if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }}
run: |
set -euo pipefail
docker system df || true
@@ -270,6 +170,7 @@ jobs:
docker system df || true
- name: Build and push branch images
if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && !contains(github.event.head_commit.message, '#no_image') }}
env:
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
@@ -458,19 +359,28 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
needs: [build_push]
if: |
github.event_name == 'push' &&
github.ref == 'refs/heads/dev' &&
needs.build_push.result == 'success' &&
!contains(github.event.head_commit.message, '#no_deploy')
if: needs.build_push.result == 'success'
steps:
- name: Trigger dev Dokploy webhooks
env:
DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }}
HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |
set -euo pipefail
if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then
echo "Skip Dokploy dev deploy for ${GITHUB_REF}"
exit 0
fi
case "${HEAD_COMMIT_MESSAGE:-}" in
*"#no_deploy"* | *"#no_image"*)
echo "Skip Dokploy dev deploy because commit message disables deploy or image build"
exit 0
;;
esac
call_webhook() {
service_name="$1"
webhook_url="$2"
@@ -568,404 +478,3 @@ jobs:
echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev"
echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev"
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
dokploy_dev_start:
name: Start Dev Containers in Dokploy
runs-on: ubuntu-latest
timeout-minutes: 5
if: |
github.event_name == 'workflow_dispatch' &&
github.ref == 'refs/heads/dev' &&
github.event.inputs.manual_action == 'dokploy_start'
steps:
- name: Trigger Dokploy webhooks
env:
DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }}
DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }}
run: |
set -euo pipefail
TARGET="${DOKPLOY_TARGET:-all}"
case "${TARGET}" in
all|web|worker|celery|beat) ;;
*)
echo "dokploy_target must be one of: all, web, worker, beat" >&2
exit 1
;;
esac
call_webhook() {
service_name="$1"
webhook_url="$2"
target="$3"
if [ -z "${webhook_url}" ]; then
echo "Dokploy webhook for ${service_name} is not configured" >&2
return 1
fi
AUTH_HEADER=()
if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then
AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}")
fi
PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY'
import json
import os
repository = os.environ.get("GITHUB_REPOSITORY", "")
repository_name = repository.rsplit("/", 1)[-1]
branch = os.environ.get("GITHUB_REF_NAME") or "dev"
sha = os.environ.get("GITHUB_SHA") or ""
server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
repository_url = f"{server_url}/{repository}" if server_url and repository else ""
image_tag = (
f"{branch.replace('/', '-')}-{sha[:7]}"
if branch and sha
else branch or "dev"
)
celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:{image_tag}"
payload = {
"ref": f"refs/heads/{branch}",
"after": sha,
"checkout_sha": sha,
"repository": {
"name": repository_name,
"full_name": repository,
"html_url": repository_url,
"clone_url": f"{repository_url}.git" if repository_url else "",
},
"sender": {"login": os.environ.get("GITHUB_ACTOR")},
"pusher": {"name": os.environ.get("GITHUB_ACTOR")},
"head_commit": {
"id": sha,
"message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}",
"url": f"{repository_url}/commit/{sha}" if repository_url and sha else "",
},
"commits": [
{
"id": sha,
"message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}",
"url": f"{repository_url}/commit/{sha}" if repository_url and sha else "",
}
],
"project": repository,
"branch": branch,
"sha": sha,
"actor": os.environ.get("GITHUB_ACTOR"),
"target": os.environ.get("CURRENT_DOKPLOY_TARGET"),
"image_tag": image_tag,
"images": {
"web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:{image_tag}",
"worker": celery_image,
"beat": celery_image,
},
}
print(json.dumps(payload, ensure_ascii=True, separators=(",", ":")))
PY
)
echo "Trigger Dokploy for ${service_name}"
RESPONSE=$(curl -fsS \
--connect-timeout 5 \
--max-time 30 \
--retry 2 \
--retry-delay 2 \
-X POST \
-H "Content-Type: application/json" \
-H "X-Gitea-Event: push" \
-H "X-Gogs-Event: push" \
-H "X-GitHub-Event: push" \
"${AUTH_HEADER[@]}" \
--data "${PAYLOAD}" \
"${webhook_url}")
printf '%s\n' "${RESPONSE}"
if printf '%s' "${RESPONSE}" | grep -qi "Branch Not Match"; then
echo "Dokploy rejected ${service_name}: branch did not match" >&2
exit 1
fi
}
triggered=0
if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "web" ]; then
call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web"
triggered=1
fi
if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "worker" ] || [ "${TARGET}" = "celery" ]; then
call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker"
triggered=1
fi
if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "beat" ]; then
call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat"
triggered=1
fi
if [ "${triggered}" -ne 1 ]; then
echo "No Dokploy webhook was triggered" >&2
exit 1
fi
{
echo "Dokploy dev trigger completed."
echo "Target: ${TARGET}"
echo "Registry API: ${REGISTRY_API_URL}"
echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev"
echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev"
echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev"
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
cleanup_dev_database:
name: Drop and Recreate Dev Database
runs-on: ubuntu-latest
timeout-minutes: 15
if: |
github.event_name == 'workflow_dispatch' &&
github.ref == 'refs/heads/dev' &&
github.event.inputs.manual_action == 'cleanup_dev_database'
env:
POSTGRES_HOST: "10.10.0.114"
POSTGRES_PORT: "5432"
POSTGRES_DB: "mostovik"
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "postgres"
steps:
- name: Validate confirmation
env:
CONFIRM: ${{ github.event.inputs.cleanup_confirm }}
run: |
set -euo pipefail
if [ "${CONFIRM}" != "CLEAN_DEV_DB" ]; then
echo "Manual confirmation must be exactly CLEAN_DEV_DB" >&2
exit 1
fi
- name: Install PostgreSQL client
run: |
set -euo pipefail
APT_RUNNER=()
if [ "$(id -u)" -ne 0 ]; then
APT_RUNNER=(sudo)
fi
export DEBIAN_FRONTEND=noninteractive
"${APT_RUNNER[@]}" apt-get update
"${APT_RUNNER[@]}" apt-get install -y postgresql-client
- name: Drop and recreate UTF8 database
run: |
set -euo pipefail
export PGPASSWORD="${POSTGRES_PASSWORD}"
DB_EXISTS=$(psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--tuples-only \
--no-align \
--set=dbname="${POSTGRES_DB}" \
<<'SQL'
SELECT 1 FROM pg_database WHERE datname = :'dbname';
SQL
)
if [ "${DB_EXISTS:-}" = "1" ]; then
echo "Closing active connections to ${POSTGRES_DB}"
psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--set=dbname="${POSTGRES_DB}" \
<<'SQL'
ALTER DATABASE :"dbname" WITH ALLOW_CONNECTIONS false;
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = :'dbname'
AND pid <> pg_backend_pid();
SQL
fi
echo "Dropping and recreating ${POSTGRES_DB} with UTF8 encoding"
psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--set=dbname="${POSTGRES_DB}" \
--set=dbuser="${POSTGRES_USER}" \
<<'SQL'
DROP DATABASE IF EXISTS :"dbname";
CREATE DATABASE :"dbname" WITH OWNER :"dbuser" TEMPLATE template0 ENCODING 'UTF8';
SQL
DB_ENCODING=$(psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--tuples-only \
--no-align \
--set=dbname="${POSTGRES_DB}" \
<<'SQL'
SELECT pg_encoding_to_char(encoding)
FROM pg_database
WHERE datname = :'dbname';
SQL
)
printf '%s\n' "${DB_ENCODING}" | tee /tmp/mostovik-db-encoding
if [ "${DB_ENCODING}" != "UTF8" ]; then
echo "Database ${POSTGRES_DB} is not UTF8 after cleanup" >&2
exit 1
fi
- name: Trigger Dokploy after database recreate
env:
DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }}
run: |
set -euo pipefail
call_webhook() {
service_name="$1"
webhook_url="$2"
target="$3"
AUTH_HEADER=()
if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then
AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}")
fi
PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY'
import json
import os
repository = os.environ.get("GITHUB_REPOSITORY", "")
repository_name = repository.rsplit("/", 1)[-1]
branch = os.environ.get("GITHUB_REF_NAME") or "dev"
sha = os.environ.get("GITHUB_SHA") or ""
server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
repository_url = f"{server_url}/{repository}" if server_url and repository else ""
image_tag = (
f"{branch.replace('/', '-')}-{sha[:7]}"
if branch and sha
else branch or "dev"
)
celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:{image_tag}"
payload = {
"ref": f"refs/heads/{branch}",
"after": sha,
"checkout_sha": sha,
"repository": {
"name": repository_name,
"full_name": repository,
"html_url": repository_url,
"clone_url": f"{repository_url}.git" if repository_url else "",
},
"sender": {"login": os.environ.get("GITHUB_ACTOR")},
"pusher": {"name": os.environ.get("GITHUB_ACTOR")},
"head_commit": {
"id": sha,
"message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}",
"url": f"{repository_url}/commit/{sha}" if repository_url and sha else "",
},
"commits": [
{
"id": sha,
"message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}",
"url": f"{repository_url}/commit/{sha}" if repository_url and sha else "",
}
],
"project": repository,
"branch": branch,
"sha": sha,
"actor": os.environ.get("GITHUB_ACTOR"),
"target": os.environ.get("CURRENT_DOKPLOY_TARGET"),
"image_tag": image_tag,
"images": {
"web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:{image_tag}",
"worker": celery_image,
"beat": celery_image,
},
}
print(json.dumps(payload, ensure_ascii=True, separators=(",", ":")))
PY
)
echo "Trigger Dokploy for ${service_name}"
RESPONSE=$(curl -fsS \
--connect-timeout 5 \
--max-time 30 \
--retry 2 \
--retry-delay 2 \
-X POST \
-H "Content-Type: application/json" \
-H "X-Gitea-Event: push" \
-H "X-Gogs-Event: push" \
-H "X-GitHub-Event: push" \
"${AUTH_HEADER[@]}" \
--data "${PAYLOAD}" \
"${webhook_url}")
printf '%s\n' "${RESPONSE}"
if printf '%s' "${RESPONSE}" | grep -qi "Branch Not Match"; then
echo "Dokploy rejected ${service_name}: branch did not match" >&2
exit 1
fi
}
wait_for_migrations() {
export PGPASSWORD="${POSTGRES_PASSWORD}"
for attempt in $(seq 1 60); do
SCHEMA_STATE=$(psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname="${POSTGRES_DB}" \
--tuples-only \
--no-align \
<<'SQL'
SELECT CASE
WHEN to_regclass('public.django_migrations') IS NOT NULL
AND to_regclass('public.core_backgroundjob') IS NOT NULL
THEN 'ready'
ELSE 'waiting'
END;
SQL
)
if [ "${SCHEMA_STATE}" = "ready" ]; then
echo "Database schema is ready after web deploy"
return 0
fi
echo "Waiting for web migrations (${attempt}/60)"
sleep 5
done
echo "Database schema was not ready after web deploy" >&2
exit 1
}
call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web"
wait_for_migrations
call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker"
call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat"
- name: Summary
run: |
set -euo pipefail
{
echo "Dev database was dropped and recreated."
echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
echo "Encoding: UTF8"
echo "Dokploy web/worker/beat deploy was triggered."
} >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"

66
.gitea/workflows/manual-dev-actions.yml Normal file
View File

@@ -0,0 +1,66 @@
name: Manual Dev Actions
on:
workflow_dispatch:
inputs:
manual_action:
description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start"
required: true
default: "noop"
dokploy_target:
description: "Dokploy dev target: all, web, worker, or beat"
required: true
default: "all"
cleanup_confirm:
description: "Type CLEAN_DEV_DB to drop and recreate the dev database as UTF8"
required: false
default: ""
concurrency:
group: mostovik-backend-manual-dev-${{ github.ref }}
cancel-in-progress: false
env:
REGISTRY_API_URL: "https://registry.dev.nii-ecos.ru/v2/"
REGISTRY_HOST: "registry.dev.nii-ecos.ru"
REGISTRY_NAMESPACE: "${{ github.repository_owner }}"
WEB_IMAGE: "mostovik-backend-web"
CELERY_IMAGE: "mostovik-backend-celery"
CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden"
WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden"
CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden"
GOLDEN_TAG: "py311-uv0.7.2"
DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF"
DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA"
DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv"
jobs:
manual_dev_action:
name: Run Manual Dev Action
runs-on: ubuntu-latest
timeout-minutes: 60
env:
MANUAL_ACTION: ${{ github.event.inputs.manual_action }}
DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }}
CLEANUP_CONFIRM: ${{ github.event.inputs.cleanup_confirm }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }}
DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }}
POSTGRES_HOST: "10.10.0.114"
POSTGRES_PORT: "5432"
POSTGRES_DB: "mostovik"
POSTGRES_USER: "postgres"
POSTGRES_PASSWORD: "postgres"
steps:
- name: Checkout code
run: |
set -euo pipefail
REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Run selected action
run: bash scripts/ci/manual_dev_action.sh

410
scripts/ci/manual_dev_action.sh Executable file
View File

@@ -0,0 +1,410 @@
#!/usr/bin/env bash
set -euo pipefail
MANUAL_ACTION="${MANUAL_ACTION:-noop}"
DOKPLOY_TARGET="${DOKPLOY_TARGET:-all}"
CLEANUP_CONFIRM="${CLEANUP_CONFIRM:-}"
SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-/dev/stdout}"
require_dev_branch() {
if [ "${GITHUB_REF:-}" != "refs/heads/dev" ]; then
echo "Manual dev actions are allowed only from dev branch, got ${GITHUB_REF:-unknown}" >&2
exit 1
fi
}
write_usage() {
{
echo "No manual action selected."
echo "For dev DB cleanup run with:"
echo "- manual_action=cleanup_dev_database"
echo "- cleanup_confirm=CLEAN_DEV_DB"
echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat."
echo "For base image refresh run with manual_action=build_golden_images."
echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat."
} >>"${SUMMARY_FILE}"
}
registry_login() {
local registry_user registry_password
registry_user="${REGISTRY_USER:-${GITHUB_ACTOR}}"
registry_password="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}"
unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY
export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}"
export no_proxy="${no_proxy:-},${REGISTRY_HOST}"
if [ -z "${registry_password}" ]; then
echo "REGISTRY_TOKEN secret is not set and GITEA_TOKEN fallback is empty" >&2
exit 1
fi
echo "${registry_password}" \
| docker login "${REGISTRY_HOST}" \
-u "${registry_user}" \
--password-stdin
}
ensure_buildx() {
if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then
docker buildx create --name mostovik-builder --use
else
docker buildx use mostovik-builder
fi
docker buildx inspect --bootstrap
}
build_golden_images() {
local registry_path ci_golden_ref web_golden_ref celery_golden_ref
registry_path="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}"
ci_golden_ref="${registry_path}/${CI_GOLDEN_IMAGE}"
web_golden_ref="${registry_path}/${WEB_GOLDEN_IMAGE}"
celery_golden_ref="${registry_path}/${CELERY_GOLDEN_IMAGE}"
registry_login
ensure_buildx
docker buildx prune --all --force || true
docker builder prune --all --force || true
build_golden() {
local target="$1"
local ref="$2"
local build_args=()
if [ "${target}" = "celery-deps-base" ]; then
build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${web_golden_ref}:${GOLDEN_TAG}")
fi
docker buildx build \
-f ./docker/Dockerfile \
--target "${target}" \
"${build_args[@]}" \
--push \
-t "${ref}:${GOLDEN_TAG}" \
-t "${ref}:latest" \
.
}
build_golden "ci-deps-base" "${ci_golden_ref}"
build_golden "web-deps-base" "${web_golden_ref}"
build_golden "celery-deps-base" "${celery_golden_ref}"
{
echo "Golden images pushed:"
echo "- ${ci_golden_ref}:${GOLDEN_TAG}"
echo "- ${web_golden_ref}:${GOLDEN_TAG}"
echo "- ${celery_golden_ref}:${GOLDEN_TAG}"
} >>"${SUMMARY_FILE}"
}
dokploy_payload() {
CURRENT_DOKPLOY_TARGET="$1" python3 - <<'PY'
import json
import os
repository = os.environ.get("GITHUB_REPOSITORY", "")
repository_name = repository.rsplit("/", 1)[-1]
branch = os.environ.get("GITHUB_REF_NAME") or "dev"
sha = os.environ.get("GITHUB_SHA") or ""
server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/")
repository_url = f"{server_url}/{repository}" if server_url and repository else ""
image_tag = f"{branch.replace('/', '-')}-{sha[:7]}" if branch and sha else branch or "dev"
celery_image = (
f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/"
f"{os.environ['CELERY_IMAGE']}:{image_tag}"
)
payload = {
"ref": f"refs/heads/{branch}",
"after": sha,
"checkout_sha": sha,
"repository": {
"name": repository_name,
"full_name": repository,
"html_url": repository_url,
"clone_url": f"{repository_url}.git" if repository_url else "",
},
"sender": {"login": os.environ.get("GITHUB_ACTOR")},
"pusher": {"name": os.environ.get("GITHUB_ACTOR")},
"head_commit": {
"id": sha,
"message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}",
"url": f"{repository_url}/commit/{sha}" if repository_url and sha else "",
},
"commits": [
{
"id": sha,
"message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}",
"url": f"{repository_url}/commit/{sha}" if repository_url and sha else "",
}
],
"project": repository,
"branch": branch,
"sha": sha,
"actor": os.environ.get("GITHUB_ACTOR"),
"target": os.environ.get("CURRENT_DOKPLOY_TARGET"),
"image_tag": image_tag,
"images": {
"web": (
f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/"
f"{os.environ['WEB_IMAGE']}:{image_tag}"
),
"worker": celery_image,
"beat": celery_image,
},
}
print(json.dumps(payload, ensure_ascii=True, separators=(",", ":")))
PY
}
call_dokploy_webhook() {
local service_name="$1"
local webhook_url="$2"
local target="$3"
local auth_header=()
local payload response
if [ -z "${webhook_url}" ]; then
echo "Dokploy webhook for ${service_name} is not configured" >&2
exit 1
fi
if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then
auth_header=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}")
fi
payload="$(dokploy_payload "${target}")"
echo "Trigger Dokploy for ${service_name}"
response="$(curl -fsS \
--connect-timeout 5 \
--max-time 30 \
--retry 2 \
--retry-delay 2 \
-X POST \
-H "Content-Type: application/json" \
-H "X-Gitea-Event: push" \
-H "X-Gogs-Event: push" \
-H "X-GitHub-Event: push" \
"${auth_header[@]}" \
--data "${payload}" \
"${webhook_url}")"
printf '%s\n' "${response}"
if printf '%s' "${response}" | grep -qi "Branch Not Match"; then
echo "Dokploy rejected ${service_name}: branch did not match" >&2
exit 1
fi
}
trigger_dokploy() {
local target="$1"
local triggered=0
case "${target}" in
all | web | worker | celery | beat) ;;
*)
echo "dokploy_target must be one of: all, web, worker, beat" >&2
exit 1
;;
esac
if [ "${target}" = "all" ] || [ "${target}" = "web" ]; then
call_dokploy_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web"
triggered=1
fi
if [ "${target}" = "all" ] || [ "${target}" = "worker" ] || [ "${target}" = "celery" ]; then
call_dokploy_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker"
triggered=1
fi
if [ "${target}" = "all" ] || [ "${target}" = "beat" ]; then
call_dokploy_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat"
triggered=1
fi
if [ "${triggered}" -ne 1 ]; then
echo "No Dokploy webhook was triggered" >&2
exit 1
fi
{
echo "Dokploy dev trigger completed."
echo "Target: ${target}"
echo "Registry API: ${REGISTRY_API_URL}"
echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev"
echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev"
echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev"
} >>"${SUMMARY_FILE}"
}
install_postgres_client() {
local apt_runner=()
if command -v psql >/dev/null 2>&1; then
return 0
fi
if [ "$(id -u)" -ne 0 ]; then
apt_runner=(sudo)
fi
export DEBIAN_FRONTEND=noninteractive
"${apt_runner[@]}" apt-get update
"${apt_runner[@]}" apt-get install -y postgresql-client
}
drop_and_recreate_database() {
local db_exists db_encoding
export PGPASSWORD="${POSTGRES_PASSWORD}"
db_exists="$(psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--tuples-only \
--no-align \
--set=dbname="${POSTGRES_DB}" \
<<'SQL'
SELECT 1 FROM pg_database WHERE datname = :'dbname';
SQL
)"
if [ "${db_exists:-}" = "1" ]; then
echo "Closing active connections to ${POSTGRES_DB}"
psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--set=dbname="${POSTGRES_DB}" \
<<'SQL'
ALTER DATABASE :"dbname" WITH ALLOW_CONNECTIONS false;
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname = :'dbname'
AND pid <> pg_backend_pid();
SQL
fi
echo "Dropping and recreating ${POSTGRES_DB} with UTF8 encoding"
psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--set=dbname="${POSTGRES_DB}" \
--set=dbuser="${POSTGRES_USER}" \
<<'SQL'
DROP DATABASE IF EXISTS :"dbname";
CREATE DATABASE :"dbname" WITH OWNER :"dbuser" TEMPLATE template0 ENCODING 'UTF8';
SQL
db_encoding="$(psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname=postgres \
--tuples-only \
--no-align \
--set=dbname="${POSTGRES_DB}" \
<<'SQL'
SELECT pg_encoding_to_char(encoding)
FROM pg_database
WHERE datname = :'dbname';
SQL
)"
printf '%s\n' "${db_encoding}" | tee /tmp/mostovik-db-encoding
if [ "${db_encoding}" != "UTF8" ]; then
echo "Database ${POSTGRES_DB} is not UTF8 after cleanup" >&2
exit 1
fi
}
wait_for_migrations() {
local schema_state
export PGPASSWORD="${POSTGRES_PASSWORD}"
for attempt in $(seq 1 60); do
schema_state="$(psql \
--set ON_ERROR_STOP=1 \
--host="${POSTGRES_HOST}" \
--port="${POSTGRES_PORT}" \
--username="${POSTGRES_USER}" \
--dbname="${POSTGRES_DB}" \
--tuples-only \
--no-align \
<<'SQL'
SELECT CASE
WHEN to_regclass('public.django_migrations') IS NOT NULL
AND to_regclass('public.core_backgroundjob') IS NOT NULL
THEN 'ready'
ELSE 'waiting'
END;
SQL
)"
if [ "${schema_state}" = "ready" ]; then
echo "Database schema is ready after web deploy"
return 0
fi
echo "Waiting for web migrations (${attempt}/60)"
sleep 5
done
echo "Database schema was not ready after web deploy" >&2
exit 1
}
cleanup_dev_database() {
if [ "${CLEANUP_CONFIRM}" != "CLEAN_DEV_DB" ]; then
echo "Manual confirmation must be exactly CLEAN_DEV_DB" >&2
exit 1
fi
install_postgres_client
drop_and_recreate_database
call_dokploy_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web"
wait_for_migrations
call_dokploy_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker"
call_dokploy_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat"
{
echo "Dev database was dropped and recreated."
echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
echo "Encoding: UTF8"
echo "Dokploy web/worker/beat deploy was triggered."
} >>"${SUMMARY_FILE}"
}
require_dev_branch
case "${MANUAL_ACTION}" in
"" | noop)
write_usage
;;
build_golden_images)
build_golden_images
;;
dokploy_start)
trigger_dokploy "${DOKPLOY_TARGET}"
;;
cleanup_dev_database)
cleanup_dev_database
;;
*)
echo "Unknown manual_action: ${MANUAL_ACTION}" >&2
exit 1
;;
esac