diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 887327a..aaf6efe 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -30,6 +30,10 @@ env: DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" + DOKPLOY_API_URL: "https://deploy.dev.nii-ecos.ru/api" + DOKPLOY_DEV_WEB_APP_NAME: "service-backend-4mbxrs" + DOKPLOY_DEV_WORKER_APP_NAME: "service-backend-512y9c" + DOKPLOY_DEV_BEAT_APP_NAME: "service-backend-nvdyoq" UV_VERSION: "0.7.2" PIP_DISABLE_PIP_VERSION_CHECK: "1" @@ -380,9 +384,20 @@ jobs: if: needs.build_push.result == 'success' steps: - - name: Trigger dev Dokploy webhooks + - name: Checkout code + run: | + set -euo pipefail + REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" + BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . + git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" + + - name: Deploy prebuilt images in Dokploy env: DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | set -euo pipefail @@ -399,100 +414,4 @@ jobs: ;; esac - call_webhook() { - service_name="$1" - webhook_url="$2" - target="$3" - - AUTH_HEADER=() - if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then - AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") - fi - - PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY' - import json - import os - - repository = os.environ.get("GITHUB_REPOSITORY", "") - repository_name = repository.rsplit("/", 1)[-1] - branch = os.environ.get("GITHUB_REF_NAME") or "dev" - sha = os.environ.get("GITHUB_SHA") or "" - server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") - repository_url = f"{server_url}/{repository}" if server_url and repository else "" - image_tag = ( - f"{branch.replace('/', '-')}-{sha[:7]}" - if branch and sha - else branch or "dev" - ) - celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:{image_tag}" - payload = { - "ref": f"refs/heads/{branch}", - "after": sha, - "checkout_sha": sha, - "repository": { - "name": repository_name, - "full_name": repository, - "html_url": repository_url, - "clone_url": f"{repository_url}.git" if repository_url else "", - }, - "sender": {"login": os.environ.get("GITHUB_ACTOR")}, - "pusher": {"name": os.environ.get("GITHUB_ACTOR")}, - "head_commit": { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - }, - "commits": [ - { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - } - ], - "project": repository, - "branch": branch, - "sha": sha, - "actor": os.environ.get("GITHUB_ACTOR"), - "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), - "image_tag": image_tag, - "images": { - "web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:{image_tag}", - "worker": celery_image, - "beat": celery_image, - }, - } - print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) - PY - ) - - echo "Trigger Dokploy for ${service_name}" - RESPONSE=$(curl -fsS \ - --connect-timeout 5 \ - --max-time 30 \ - --retry 2 \ - --retry-delay 2 \ - -X POST \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gogs-Event: push" \ - -H "X-GitHub-Event: push" \ - "${AUTH_HEADER[@]}" \ - --data "${PAYLOAD}" \ - "${webhook_url}") - printf '%s\n' "${RESPONSE}" - if printf '%s' "${RESPONSE}" | grep -qi "Branch Not Match"; then - echo "Dokploy rejected ${service_name}: branch did not match" >&2 - exit 1 - fi - } - - call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" - call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" - call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" - - { - echo "Dokploy dev deploy triggered." - echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev" - echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" - echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" - } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + bash scripts/ci/dokploy_deploy_image.sh all diff --git a/.gitea/workflows/manual-dev-actions.yml b/.gitea/workflows/manual-dev-actions.yml index 69ba4fc..37927e1 100644 --- a/.gitea/workflows/manual-dev-actions.yml +++ b/.gitea/workflows/manual-dev-actions.yml @@ -33,6 +33,10 @@ env: DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" + DOKPLOY_API_URL: "https://deploy.dev.nii-ecos.ru/api" + DOKPLOY_DEV_WEB_APP_NAME: "service-backend-4mbxrs" + DOKPLOY_DEV_WORKER_APP_NAME: "service-backend-512y9c" + DOKPLOY_DEV_BEAT_APP_NAME: "service-backend-nvdyoq" jobs: manual_dev_action: diff --git a/scripts/ci/dokploy_deploy_image.sh b/scripts/ci/dokploy_deploy_image.sh new file mode 100755 index 0000000..407e2bb --- /dev/null +++ b/scripts/ci/dokploy_deploy_image.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET="${1:-${DOKPLOY_TARGET:-all}}" +SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-/dev/stdout}" +DOKPLOY_API_URL="${DOKPLOY_API_URL:-https://deploy.dev.nii-ecos.ru/api}" + +require_dokploy_api_token() { + if [ -z "${DOKPLOY_API_TOKEN:-}" ]; then + echo "DOKPLOY_API_TOKEN is required to deploy prebuilt images through Dokploy API" >&2 + exit 1 + fi +} + +branch_tag() { + local branch branch_tag sha_short + + branch="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-branch}}" + branch_tag="$(printf '%s' "${branch}" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's#[/[:space:]]+#-#g; s#[^a-z0-9_.-]+#-#g; s#^-+##; s#-+$##')" + branch_tag="${branch_tag:-branch}" + branch_tag="$(printf '%.120s' "${branch_tag}")" + sha_short="$(printf '%s' "${GITHUB_SHA:-}" | cut -c1-7)" + + if [ -n "${sha_short}" ]; then + printf '%s-%s' "${branch_tag}" "${sha_short}" + else + printf '%s' "${branch_tag}" + fi +} + +registry_username() { + printf '%s' "${REGISTRY_USER:-${GITHUB_ACTOR:-}}" +} + +registry_password() { + printf '%s' "${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" +} + +require_registry_credentials() { + if [ -z "$(registry_username)" ] || [ -z "$(registry_password)" ]; then + echo "REGISTRY_USER/REGISTRY_TOKEN are required so Dokploy can pull private images" >&2 + exit 1 + fi +} + +urlencode() { + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1" +} + +dokploy_get() { + local endpoint="$1" + + curl -fsS \ + --connect-timeout 5 \ + --max-time 30 \ + --retry 2 \ + --retry-delay 2 \ + -H "x-api-key: ${DOKPLOY_API_TOKEN}" \ + "${DOKPLOY_API_URL%/}/${endpoint}" +} + +dokploy_post() { + local endpoint="$1" + local payload="$2" + + curl -fsS \ + --connect-timeout 5 \ + --max-time 60 \ + --retry 2 \ + --retry-delay 2 \ + -X POST \ + -H "Content-Type: application/json" \ + -H "x-api-key: ${DOKPLOY_API_TOKEN}" \ + --data "${payload}" \ + "${DOKPLOY_API_URL%/}/${endpoint}" +} + +resolve_application_id() { + local explicit_id="$1" + local app_name="$2" + local encoded response application_id + + if [ -n "${explicit_id}" ]; then + printf '%s' "${explicit_id}" + return 0 + fi + + if [ -z "${app_name}" ]; then + echo "Dokploy application id or appName is required" >&2 + exit 1 + fi + + encoded="$(urlencode "${app_name}")" + response="$(dokploy_get "application.search?appName=${encoded}&limit=10")" + application_id="$(printf '%s' "${response}" | APP_NAME="${app_name}" python3 -c ' +import json +import os +import sys + +app_name = os.environ["APP_NAME"] +payload = json.load(sys.stdin) + +def walk(node): + if isinstance(node, list): + for item in node: + yield from walk(item) + return + if not isinstance(node, dict): + return + if any(key in node for key in ("applicationId", "id", "appName", "name")): + yield node + for key in ("applications", "data", "items", "results"): + if key in node: + yield from walk(node[key]) + +candidates = list(walk(payload)) +matches = [ + item + for item in candidates + if item.get("appName") == app_name or item.get("name") == app_name +] +if not matches and len(candidates) == 1: + matches = candidates +if not matches: + raise SystemExit(f"Application not found by appName={app_name!r}") +identifier = matches[0].get("applicationId") or matches[0].get("id") +if not identifier: + raise SystemExit(f"Application id is missing for appName={app_name!r}") +print(identifier) +')" + + printf '%s' "${application_id}" +} + +image_for_target() { + local target="$1" + local registry_path image_tag + + registry_path="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + image_tag="$(branch_tag)" + + case "${target}" in + web) + printf '%s/%s:%s' "${registry_path}" "${WEB_IMAGE}" "${image_tag}" + ;; + worker | beat) + printf '%s/%s:%s' "${registry_path}" "${CELERY_IMAGE}" "${image_tag}" + ;; + *) + echo "Unknown Dokploy target: ${target}" >&2 + exit 1 + ;; + esac +} + +target_app_id_env() { + case "$1" in + web) printf '%s' "${DOKPLOY_DEV_WEB_APPLICATION_ID:-}" ;; + worker) printf '%s' "${DOKPLOY_DEV_WORKER_APPLICATION_ID:-}" ;; + beat) printf '%s' "${DOKPLOY_DEV_BEAT_APPLICATION_ID:-}" ;; + *) return 1 ;; + esac +} + +target_app_name_env() { + case "$1" in + web) printf '%s' "${DOKPLOY_DEV_WEB_APP_NAME:-service-backend-4mbxrs}" ;; + worker) printf '%s' "${DOKPLOY_DEV_WORKER_APP_NAME:-service-backend-512y9c}" ;; + beat) printf '%s' "${DOKPLOY_DEV_BEAT_APP_NAME:-service-backend-nvdyoq}" ;; + *) return 1 ;; + esac +} + +save_docker_provider() { + local application_id="$1" + local image="$2" + local payload + + payload="$(APPLICATION_ID="${application_id}" \ + DOCKER_IMAGE="${image}" \ + REGISTRY_USERNAME="$(registry_username)" \ + REGISTRY_PASSWORD_VALUE="$(registry_password)" \ + python3 - <<'PY' +import json +import os + +print(json.dumps({ + "applicationId": os.environ["APPLICATION_ID"], + "dockerImage": os.environ["DOCKER_IMAGE"], + "username": os.environ["REGISTRY_USERNAME"], + "password": os.environ["REGISTRY_PASSWORD_VALUE"], + "registryUrl": os.environ["REGISTRY_HOST"], +}, ensure_ascii=True, separators=(",", ":"))) +PY + )" + + dokploy_post "application.saveDockerProvider" "${payload}" >/dev/null +} + +deploy_application() { + local target="$1" + local application_id="$2" + local image="$3" + local payload + + payload="$(APPLICATION_ID="${application_id}" \ + DEPLOY_TARGET="${target}" \ + DOCKER_IMAGE="${image}" \ + python3 - <<'PY' +import json +import os + +target = os.environ["DEPLOY_TARGET"] +image = os.environ["DOCKER_IMAGE"] +sha = os.environ.get("GITHUB_SHA", "") + +print(json.dumps({ + "applicationId": os.environ["APPLICATION_ID"], + "title": f"CI deploy {target}", + "description": f"{image} ({sha[:7]})" if sha else image, +}, ensure_ascii=True, separators=(",", ":"))) +PY + )" + + dokploy_post "application.deploy" "${payload}" >/dev/null +} + +deploy_target() { + local target="$1" + local application_id image + + application_id="$(resolve_application_id \ + "$(target_app_id_env "${target}")" \ + "$(target_app_name_env "${target}")")" + image="$(image_for_target "${target}")" + + echo "Dokploy ${target}: set Docker image ${image}" + save_docker_provider "${application_id}" "${image}" + deploy_application "${target}" "${application_id}" "${image}" + + { + echo "- ${target}: ${image}" + } >>"${SUMMARY_FILE}" +} + +main() { + require_dokploy_api_token + require_registry_credentials + + case "${TARGET}" in + all | web | worker | celery | beat) ;; + *) + echo "dokploy target must be one of: all, web, worker, beat" >&2 + exit 1 + ;; + esac + + { + echo "Dokploy Docker-image deploy:" + echo "Registry API: ${REGISTRY_API_URL:-${REGISTRY_HOST}}" + } >>"${SUMMARY_FILE}" + + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "web" ]; then + deploy_target "web" + fi + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "worker" ] || [ "${TARGET}" = "celery" ]; then + deploy_target "worker" + fi + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "beat" ]; then + deploy_target "beat" + fi +} + +main diff --git a/scripts/ci/manual_dev_action.sh b/scripts/ci/manual_dev_action.sh index cd6fed5..ca9ee4f 100755 --- a/scripts/ci/manual_dev_action.sh +++ b/scripts/ci/manual_dev_action.sh @@ -21,7 +21,7 @@ write_usage() { echo "- cleanup_confirm=CLEAN_DEV_DB" echo "This drops and recreates the dev database, then triggers Dokploy web/worker/beat." echo "For base image refresh run with manual_action=build_golden_images." - echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat." + echo "For Dokploy image deploy run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat." } >>"${SUMMARY_FILE}" } @@ -100,109 +100,15 @@ build_golden_images() { } >>"${SUMMARY_FILE}" } -dokploy_payload() { - CURRENT_DOKPLOY_TARGET="$1" python3 - <<'PY' -import json -import os - -repository = os.environ.get("GITHUB_REPOSITORY", "") -repository_name = repository.rsplit("/", 1)[-1] -branch = os.environ.get("GITHUB_REF_NAME") or "dev" -sha = os.environ.get("GITHUB_SHA") or "" -server_url = os.environ.get("GITHUB_SERVER_URL", "").rstrip("/") -repository_url = f"{server_url}/{repository}" if server_url and repository else "" -image_tag = f"{branch.replace('/', '-')}-{sha[:7]}" if branch and sha else branch or "dev" -celery_image = ( - f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/" - f"{os.environ['CELERY_IMAGE']}:{image_tag}" -) -payload = { - "ref": f"refs/heads/{branch}", - "after": sha, - "checkout_sha": sha, - "repository": { - "name": repository_name, - "full_name": repository, - "html_url": repository_url, - "clone_url": f"{repository_url}.git" if repository_url else "", - }, - "sender": {"login": os.environ.get("GITHUB_ACTOR")}, - "pusher": {"name": os.environ.get("GITHUB_ACTOR")}, - "head_commit": { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - }, - "commits": [ - { - "id": sha, - "message": f"CI deploy {os.environ.get('CURRENT_DOKPLOY_TARGET')}", - "url": f"{repository_url}/commit/{sha}" if repository_url and sha else "", - } - ], - "project": repository, - "branch": branch, - "sha": sha, - "actor": os.environ.get("GITHUB_ACTOR"), - "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), - "image_tag": image_tag, - "images": { - "web": ( - f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/" - f"{os.environ['WEB_IMAGE']}:{image_tag}" - ), - "worker": celery_image, - "beat": celery_image, - }, -} -print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) -PY -} - -call_dokploy_webhook() { - local service_name="$1" - local webhook_url="$2" - local target="$3" - local auth_header=() - local payload response - - if [ -z "${webhook_url}" ]; then - echo "Dokploy webhook for ${service_name} is not configured" >&2 - exit 1 - fi - - if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then - auth_header=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") - fi - - payload="$(dokploy_payload "${target}")" - - echo "Trigger Dokploy for ${service_name}" - response="$(curl -fsS \ - --connect-timeout 5 \ - --max-time 30 \ - --retry 2 \ - --retry-delay 2 \ - -X POST \ - -H "Content-Type: application/json" \ - -H "X-Gitea-Event: push" \ - -H "X-Gogs-Event: push" \ - -H "X-GitHub-Event: push" \ - "${auth_header[@]}" \ - --data "${payload}" \ - "${webhook_url}")" - printf '%s\n' "${response}" - - if printf '%s' "${response}" | grep -qi "Branch Not Match"; then - echo "Dokploy rejected ${service_name}: branch did not match" >&2 +require_dokploy_api_token() { + if [ -z "${DOKPLOY_API_TOKEN:-}" ]; then + echo "DOKPLOY_API_TOKEN is required for Dokploy image deploy" >&2 exit 1 fi } trigger_dokploy() { local target="$1" - local triggered=0 - case "${target}" in all | web | worker | celery | beat) ;; *) @@ -211,34 +117,7 @@ trigger_dokploy() { ;; esac - if [ "${target}" = "all" ] || [ "${target}" = "web" ]; then - call_dokploy_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" - triggered=1 - fi - - if [ "${target}" = "all" ] || [ "${target}" = "worker" ] || [ "${target}" = "celery" ]; then - call_dokploy_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" - triggered=1 - fi - - if [ "${target}" = "all" ] || [ "${target}" = "beat" ]; then - call_dokploy_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" - triggered=1 - fi - - if [ "${triggered}" -ne 1 ]; then - echo "No Dokploy webhook was triggered" >&2 - exit 1 - fi - - { - echo "Dokploy dev trigger completed." - echo "Target: ${target}" - echo "Registry API: ${REGISTRY_API_URL}" - echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev" - echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" - echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" - } >>"${SUMMARY_FILE}" + bash scripts/ci/dokploy_deploy_image.sh "${target}" } install_postgres_client() { @@ -373,18 +252,19 @@ cleanup_dev_database() { exit 1 fi + require_dokploy_api_token install_postgres_client drop_and_recreate_database - call_dokploy_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" + bash scripts/ci/dokploy_deploy_image.sh web wait_for_migrations - call_dokploy_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" - call_dokploy_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" + bash scripts/ci/dokploy_deploy_image.sh worker + bash scripts/ci/dokploy_deploy_image.sh beat { echo "Dev database was dropped and recreated." echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" echo "Encoding: UTF8" - echo "Dokploy web/worker/beat deploy was triggered." + echo "Dokploy web/worker/beat image deploy was triggered." } >>"${SUMMARY_FILE}" }