From 4aa552341f4e6bc61719cadae29932948d2df416 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 28 Apr 2026 13:18:06 +0200 Subject: [PATCH] ci: wire dokploy deploy triggers #no_deploy --- .gitea/workflows/ci-cd.yml | 209 ++++++++++++++++++++----- src/apps/core/startup_checks.py | 18 ++- tests/apps/core/test_startup_checks.py | 19 ++- 3 files changed, 207 insertions(+), 39 deletions(-) diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 0a5d65a..2972ad5 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -18,11 +18,11 @@ on: required: true default: "noop" dokploy_target: - description: "Dokploy dev target: all, web, or celery" + description: "Dokploy dev target: all, web, worker, or beat" required: true default: "all" cleanup_confirm: - description: "Type CLEAN_DEV_DB to drop and recreate the dev public schema" + description: "Type CLEAN_DEV_DB to recreate/reset the dev database as UTF8" required: false default: "" @@ -37,6 +37,9 @@ env: REGISTRY_NAMESPACE: "${{ github.repository_owner }}" WEB_IMAGE: "mostovik-backend-web" CELERY_IMAGE: "mostovik-backend-celery" + DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" + DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" + DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" UV_VERSION: "0.7.2" PIP_DISABLE_PIP_VERSION_CHECK: "1" @@ -59,7 +62,7 @@ jobs: echo "For dev DB cleanup run with:" echo "- manual_action=cleanup_dev_database" echo "- cleanup_confirm=CLEAN_DEV_DB" - echo "For Dokploy start run with manual_action=dokploy_start." + echo "For Dokploy start run with manual_action=dokploy_start and dokploy_target=all|web|worker|beat." } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" quality: @@ -308,6 +311,79 @@ jobs: --data "${PAYLOAD}" \ "${CI_NOTIFY_WEBHOOK_URL}" + deploy_dev: + name: Deploy Dev in Dokploy + runs-on: ubuntu-latest + timeout-minutes: 5 + needs: [build_push] + if: | + github.event_name == 'push' && + github.ref == 'refs/heads/dev' && + needs.build_push.result == 'success' && + !contains(github.event.head_commit.message, '#no_deploy') + + steps: + - name: Trigger dev Dokploy webhooks + env: + DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} + run: | + set -euo pipefail + + call_webhook() { + service_name="$1" + webhook_url="$2" + target="$3" + + AUTH_HEADER=() + if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then + AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") + fi + + PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY' + import json + import os + + celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:dev" + payload = { + "project": os.environ.get("GITHUB_REPOSITORY"), + "branch": os.environ.get("GITHUB_REF_NAME"), + "sha": os.environ.get("GITHUB_SHA"), + "actor": os.environ.get("GITHUB_ACTOR"), + "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), + "images": { + "web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:dev", + "worker": celery_image, + "beat": celery_image, + }, + } + print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) + PY + ) + + echo "Trigger Dokploy for ${service_name}" + curl -fsS \ + --connect-timeout 5 \ + --max-time 30 \ + --retry 2 \ + --retry-delay 2 \ + -X POST \ + -H "Content-Type: application/json" \ + "${AUTH_HEADER[@]}" \ + --data "${PAYLOAD}" \ + "${webhook_url}" + } + + call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" + call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" + call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" + + { + echo "Dokploy dev deploy triggered." + echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev" + echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" + echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" + } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + dokploy_dev_start: name: Start Dev Containers in Dokploy runs-on: ubuntu-latest @@ -321,18 +397,15 @@ jobs: - name: Trigger Dokploy webhooks env: DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }} - DOKPLOY_DEV_WEBHOOK_URL: ${{ secrets.DOKPLOY_DEV_WEBHOOK_URL }} - DOKPLOY_DEV_WEB_WEBHOOK_URL: ${{ secrets.DOKPLOY_DEV_WEB_WEBHOOK_URL }} - DOKPLOY_DEV_CELERY_WEBHOOK_URL: ${{ secrets.DOKPLOY_DEV_CELERY_WEBHOOK_URL }} DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} run: | set -euo pipefail TARGET="${DOKPLOY_TARGET:-all}" case "${TARGET}" in - all|web|celery) ;; + all|web|worker|celery|beat) ;; *) - echo "dokploy_target must be one of: all, web, celery" >&2 + echo "dokploy_target must be one of: all, web, worker, beat" >&2 exit 1 ;; esac @@ -340,6 +413,7 @@ jobs: call_webhook() { service_name="$1" webhook_url="$2" + target="$3" if [ -z "${webhook_url}" ]; then echo "Dokploy webhook for ${service_name} is not configured" >&2 @@ -351,10 +425,11 @@ jobs: AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") fi - PAYLOAD=$(python3 - <<'PY' + PAYLOAD=$(CURRENT_DOKPLOY_TARGET="${target}" python3 - <<'PY' import json import os + celery_image = f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['CELERY_IMAGE']}:dev" payload = { "project": os.environ.get("GITHUB_REPOSITORY"), "branch": os.environ.get("GITHUB_REF_NAME"), @@ -362,8 +437,9 @@ jobs: "actor": os.environ.get("GITHUB_ACTOR"), "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), "images": { - "web": "registry.dev.nii-ecos.ru/avm/mostovik-backend-web:dev", - "celery": "registry.dev.nii-ecos.ru/avm/mostovik-backend-celery:dev", + "web": f"{os.environ['REGISTRY_HOST']}/{os.environ['REGISTRY_NAMESPACE']}/{os.environ['WEB_IMAGE']}:dev", + "worker": celery_image, + "beat": celery_image, }, } print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) @@ -385,19 +461,19 @@ jobs: triggered=0 - if [ "${TARGET}" = "all" ] && [ -n "${DOKPLOY_DEV_WEBHOOK_URL:-}" ]; then - CURRENT_DOKPLOY_TARGET="all" call_webhook "dev stack" "${DOKPLOY_DEV_WEBHOOK_URL}" + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "web" ]; then + call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL}" "web" triggered=1 - else - if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "web" ]; then - CURRENT_DOKPLOY_TARGET="web" call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL:-${DOKPLOY_DEV_WEBHOOK_URL:-}}" - triggered=1 - fi + fi - if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "celery" ]; then - CURRENT_DOKPLOY_TARGET="celery" call_webhook "dev celery" "${DOKPLOY_DEV_CELERY_WEBHOOK_URL:-${DOKPLOY_DEV_WEBHOOK_URL:-}}" - triggered=1 - fi + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "worker" ] || [ "${TARGET}" = "celery" ]; then + call_webhook "dev worker" "${DOKPLOY_DEV_WORKER_WEBHOOK_URL}" "worker" + triggered=1 + fi + + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "beat" ]; then + call_webhook "dev beat" "${DOKPLOY_DEV_BEAT_WEBHOOK_URL}" "beat" + triggered=1 fi if [ "${triggered}" -ne 1 ]; then @@ -409,8 +485,9 @@ jobs: echo "Dokploy dev trigger completed." echo "Target: ${TARGET}" echo "Registry API: ${REGISTRY_API_URL}" - echo "Web image: registry.dev.nii-ecos.ru/avm/mostovik-backend-web:dev" - echo "Celery image: registry.dev.nii-ecos.ru/avm/mostovik-backend-celery:dev" + echo "Web image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${WEB_IMAGE}:dev" + echo "Worker image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" + echo "Beat image: ${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${CELERY_IMAGE}:dev" } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" cleanup_dev_database: @@ -451,28 +528,87 @@ jobs: "${APT_RUNNER[@]}" apt-get update "${APT_RUNNER[@]}" apt-get install -y postgresql-client - - name: Drop and recreate public schema + - name: Recreate UTF8 database or reset public schema run: | set -euo pipefail export PGPASSWORD="${POSTGRES_PASSWORD}" + terminate_connections() { + psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --set=dbname="${POSTGRES_DB}" \ + <<'SQL' + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = :'dbname' + AND pid <> pg_backend_pid(); + SQL + } + + ENCODING=$(psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=dbname="${POSTGRES_DB}" \ + --command="SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = :'dbname';") + + if [ "${ENCODING:-}" != "UTF8" ]; then + echo "Database ${POSTGRES_DB} encoding is ${ENCODING:-missing}; recreating as UTF8" + terminate_connections + psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname=postgres \ + --set=dbname="${POSTGRES_DB}" \ + --set=dbuser="${POSTGRES_USER}" \ + <<'SQL' + DROP DATABASE IF EXISTS :"dbname"; + CREATE DATABASE :"dbname" WITH OWNER :"dbuser" TEMPLATE template0 ENCODING 'UTF8'; + SQL + else + echo "Database ${POSTGRES_DB} is already UTF8; resetting public schema" + terminate_connections + psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname="${POSTGRES_DB}" \ + --set=dbuser="${POSTGRES_USER}" \ + <<'SQL' + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public AUTHORIZATION :"dbuser"; + GRANT ALL ON SCHEMA public TO :"dbuser"; + GRANT ALL ON SCHEMA public TO public; + SQL + fi + psql \ --set ON_ERROR_STOP=1 \ --host="${POSTGRES_HOST}" \ --port="${POSTGRES_PORT}" \ --username="${POSTGRES_USER}" \ - --dbname="${POSTGRES_DB}" \ - <<'SQL' - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = current_database() - AND pid <> pg_backend_pid(); + --dbname=postgres \ + --tuples-only \ + --no-align \ + --set=dbname="${POSTGRES_DB}" \ + --command="SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = :'dbname';" \ + | tee /tmp/mostovik-db-encoding - DROP SCHEMA IF EXISTS public CASCADE; - CREATE SCHEMA public; - GRANT ALL ON SCHEMA public TO postgres; - GRANT ALL ON SCHEMA public TO public; - SQL + if [ "$(cat /tmp/mostovik-db-encoding)" != "UTF8" ]; then + echo "Database ${POSTGRES_DB} is not UTF8 after cleanup" >&2 + exit 1 + fi - name: Summary run: | @@ -480,4 +616,5 @@ jobs: { echo "Dev database cleanup completed." echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" + echo "Encoding: UTF8" } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" diff --git a/src/apps/core/startup_checks.py b/src/apps/core/startup_checks.py index 4053990..3812a05 100644 --- a/src/apps/core/startup_checks.py +++ b/src/apps/core/startup_checks.py @@ -39,8 +39,22 @@ def _check_db(timeout_seconds: int) -> tuple[bool, str]: try: conn = psycopg2.connect(**params) with conn.cursor() as cursor: - cursor.execute("SELECT 1") - cursor.fetchone() + cursor.execute( + """ + SELECT pg_encoding_to_char(encoding) + FROM pg_database + WHERE datname = current_database() + """ + ) + row = cursor.fetchone() + encoding = row[0] if row else None + if encoding != "UTF8": + return ( + False, + f"{params['host']}:{params['port']}/{params['dbname']} " + f"(database encoding is {encoding or 'unknown'}, expected UTF8; " + "recreate the database with ENCODING 'UTF8')", + ) return True, "OK" except Exception as exc: # noqa: BLE001 target = f"{params['host']}:{params['port']}/{params['dbname']}" diff --git a/tests/apps/core/test_startup_checks.py b/tests/apps/core/test_startup_checks.py index e8f0d36..739a0d8 100644 --- a/tests/apps/core/test_startup_checks.py +++ b/tests/apps/core/test_startup_checks.py @@ -38,6 +38,7 @@ class StartupChecksTest(SimpleTestCase): @patch("apps.core.startup_checks.psycopg2.connect") def test_check_db_success(self, connect_mock): cursor = MagicMock() + cursor.fetchone.return_value = ("UTF8",) connection = MagicMock() connection.cursor.return_value.__enter__.return_value = cursor connect_mock.return_value = connection @@ -55,10 +56,26 @@ class StartupChecksTest(SimpleTestCase): connect_timeout=7, sslmode="require", ) - cursor.execute.assert_called_once_with("SELECT 1") + self.assertIn("pg_encoding_to_char", cursor.execute.call_args.args[0]) cursor.fetchone.assert_called_once_with() connection.close.assert_called_once_with() + @override_settings(DATABASES=TEST_DATABASES) + @patch("apps.core.startup_checks.psycopg2.connect") + def test_check_db_fails_on_non_utf8_database(self, connect_mock): + cursor = MagicMock() + cursor.fetchone.return_value = ("SQL_ASCII",) + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + connect_mock.return_value = connection + + success, message = startup_checks._check_db(7) + + self.assertFalse(success) + self.assertIn("database encoding is SQL_ASCII", message) + self.assertIn("expected UTF8", message) + connection.close.assert_called_once_with() + @override_settings(DATABASES=TEST_DATABASES) @patch( "apps.core.startup_checks.psycopg2.connect",