diff --git a/.env.prod.example b/.env.prod.example index 0243de8..3ce7abb 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -22,6 +22,7 @@ GUNICORN_WORKERS=4 GUNICORN_TIMEOUT=60 CELERY_LOG_LEVEL=INFO CELERY_WORKER_CONCURRENCY=2 +CELERY_STARTUP_REFRESH_ENABLED=true # Parsers API keys CHECKO_API_KEY= diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index b4a74f3..1f330fd 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -1,446 +1,337 @@ -name: CI/CD Pipeline +name: Mostovik Backend CI/CD on: push: branches: - - main - dev - - "feature/**" - - "codex/**" + # Add main only after the first manual main restore/rollback drill passes. pull_request: branches: - - main - dev + - main + workflow_dispatch: concurrency: - group: mostovik-backend-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} - cancel-in-progress: true + group: mostovik-backend-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: - PYTHON_VERSION: "3.11" - REGISTRY_API_URL: "https://registry.dev.nii-ecos.ru/v2/" - REGISTRY_HOST: "registry.dev.nii-ecos.ru" - REGISTRY_NAMESPACE: "${{ github.repository_owner }}" - WEB_IMAGE: "mostovik-backend-web" - CELERY_IMAGE: "mostovik-backend-celery" - CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden" - WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden" - CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden" - GOLDEN_TAG: "py311-uv0.7.2" - UV_VERSION: "0.7.2" - PIP_DISABLE_PIP_VERSION_CHECK: "1" + REGISTRY_HOST: registry.dev.nii-ecos.ru + REGISTRY_NAMESPACE: ${{ github.repository_owner }} + WEB_IMAGE: mostovik-backend-web + CELERY_IMAGE: mostovik-backend-celery + CUSTOMER_DEPLOY_HOST: 10.0.10.174 + CUSTOMER_DEPLOY_USER: ecos + CUSTOMER_SSH_PROXY_HOST: ecos-proxy@10.10.0.121 jobs: quality: - name: Quality Gate + name: Tests and lint runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 30 steps: - - name: Checkout code + - name: Checkout exact event revision + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false + + - name: Run quality gate run: | set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Free Docker space - run: | - set -euo pipefail - docker system df || true - docker buildx prune --all --force || true - docker builder prune --all --force || true - docker system prune --all --force --volumes || true - docker system df || true - - - name: Run quality in golden image - env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - REGISTRY_USER: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - SKIP_LINT: ${{ contains(github.event.head_commit.message, '#no_lint') }} - SKIP_TEST: ${{ contains(github.event.head_commit.message, '#no_test') }} - run: | - set -euo pipefail - - REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" - CI_GOLDEN_REF="${REGISTRY_PATH}/${CI_GOLDEN_IMAGE}" - REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" - REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" - - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY - export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" - export no_proxy="${no_proxy:-},${REGISTRY_HOST}" - - if [ -z "${REGISTRY_PASSWORD}" ]; then - echo "REGISTRY_PASSWORD secret is not set and GITEA_TOKEN fallback is empty" >&2 - exit 1 - fi - - echo "${REGISTRY_PASSWORD}" \ - | docker login "${REGISTRY_HOST}" \ - -u "${REGISTRY_USER}" \ - --password-stdin - - if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then - docker buildx create --name mostovik-builder --use - else - docker buildx use mostovik-builder - fi - docker buildx inspect --bootstrap - - lockfile_sha="$(sha256sum uv.lock | awk '{ print $1 }')" - - image_usable() { - docker run --rm "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ - bash -c " - set -euo pipefail - test -f /app/uv.lock - test \"\$(sha256sum /app/uv.lock | awk '{ print \$1 }')\" = '${lockfile_sha}' - /app/.venv/bin/ruff --version >/dev/null - /app/.venv/bin/python -m pytest --version >/dev/null - " + ci_image="mostovik-backend-ci:${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" + ci_builder="mostovik-backend-ci-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" + cleanup() { + docker image rm --force "${ci_image}" >/dev/null 2>&1 || true + docker buildx rm "${ci_builder}" >/dev/null 2>&1 || true } + trap cleanup EXIT - if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1 || ! image_usable; then - docker buildx prune --all --force || true - docker builder prune --all --force || true - docker buildx build \ - -f ./docker/Dockerfile \ - --target ci-deps-base \ - --push \ - -t "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ - -t "${CI_GOLDEN_REF}:latest" \ - . - docker pull "${CI_GOLDEN_REF}:${GOLDEN_TAG}" - fi + docker buildx create \ + --name "${ci_builder}" \ + --driver docker-container + docker buildx inspect "${ci_builder}" --bootstrap + docker buildx build \ + --builder "${ci_builder}" \ + --file docker/Dockerfile \ + --target ci-deps-base \ + --tag "${ci_image}" \ + --load \ + . - tar --exclude=.git -cf - . \ + tar --exclude=.git --exclude=.venv --exclude=.ruff_cache -cf - . \ | docker run --rm -i \ - -e DJANGO_SETTINGS_MODULE=settings.test \ - -e SECRET_KEY=test-secret-key-for-ci \ - -e SKIP_LINT="${SKIP_LINT}" \ - -e SKIP_TEST="${SKIP_TEST}" \ - "${CI_GOLDEN_REF}:${GOLDEN_TAG}" \ - bash -c ' - set -euo pipefail - mkdir -p /workspace - tar -xf - -C /workspace - cd /workspace - export PATH="/app/.venv/bin:${PATH}" - export PYTHONPATH="/workspace/src:${PYTHONPATH:-}" - if [ "${SKIP_LINT}" != "true" ]; then - /app/.venv/bin/ruff check src - /app/.venv/bin/ruff format src --check - fi - if [ "${SKIP_TEST}" != "true" ]; then - /app/.venv/bin/python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q - /app/.venv/bin/python -m pytest tests/test_api_inventory_e2e.py -q - fi - ' + --env DJANGO_SETTINGS_MODULE=settings.test \ + --env SECRET_KEY=test-secret-key-for-ci \ + "${ci_image}" \ + bash -c ' + set -euo pipefail + mkdir -p /workspace + tar -xf - -C /workspace + cd /workspace + export PATH="/app/.venv/bin:${PATH}" + export PYTHONPATH="/workspace/src:${PYTHONPATH:-}" + ruff check src + ruff format src --check + python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q + python -m pytest tests/test_api_inventory_e2e.py -q + ' - build_push: - name: Build and Push Images - runs-on: ubuntu-latest - timeout-minutes: 45 + build: + name: Build linux/amd64 release images + runs-on: [backend-docker] + timeout-minutes: 60 needs: [quality] - if: needs.quality.result == 'success' + if: ${{ needs.quality.result == 'success' && github.event_name != 'pull_request' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }} + outputs: + web_ref: ${{ steps.release.outputs.web_ref }} + celery_ref: ${{ steps.release.outputs.celery_ref }} + web_tag_ref: ${{ steps.release.outputs.web_tag_ref }} + celery_tag_ref: ${{ steps.release.outputs.celery_tag_ref }} steps: - - name: Check whether image build is required + - name: Checkout exact event revision + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false + + - name: Build once and publish immutable digests + id: release env: - HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: | - set -euo pipefail - - if [ "${GITHUB_EVENT_NAME}" != "push" ]; then - echo "Skip image build for ${GITHUB_EVENT_NAME}" - exit 0 - fi - - if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then - echo "Skip image build for ${GITHUB_REF}" - exit 0 - fi - - case "${HEAD_COMMIT_MESSAGE:-}" in - *"#no_image"*) - echo "Skip image build because commit message contains #no_image" - exit 0 - ;; - esac - - echo "Image build is required for ${GITHUB_REF}" - - - name: Checkout code - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }} - run: | - set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Free Docker build space - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }} - run: | - set -euo pipefail - docker system df || true - docker buildx prune --all --force || true - docker builder prune --all --force || true - docker system prune --all --force --volumes || true - docker system df || true - - - name: Build and push dev images - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }} - env: - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - REGISTRY_USER: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} run: | set -euo pipefail - - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-branch}}" - BRANCH_TAG=$(printf '%s' "${BRANCH}" \ - | tr '[:upper:]' '[:lower:]' \ - | sed -E 's#[/[:space:]]+#-#g; s#[^a-z0-9_.-]+#-#g; s#^-+##; s#-+$##') - BRANCH_TAG="${BRANCH_TAG:-branch}" - BRANCH_TAG=$(printf '%.120s' "${BRANCH_TAG}") - SHA_SHORT=$(printf '%s' "${GITHUB_SHA}" | cut -c1-7) - - REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" - WEB_REF="${REGISTRY_PATH}/${WEB_IMAGE}" - CELERY_REF="${REGISTRY_PATH}/${CELERY_IMAGE}" - WEB_GOLDEN_REF="${REGISTRY_PATH}/${WEB_GOLDEN_IMAGE}" - CELERY_GOLDEN_REF="${REGISTRY_PATH}/${CELERY_GOLDEN_IMAGE}" - REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" - REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" - BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY - export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" - export no_proxy="${no_proxy:-},${REGISTRY_HOST}" - - if [ -z "${REGISTRY_PASSWORD}" ]; then - echo "REGISTRY_PASSWORD secret is not set and GITEA_TOKEN fallback is empty" >&2 + if [ -z "${REGISTRY_USERNAME:-}" ] || [ -z "${REGISTRY_PASSWORD:-}" ]; then + echo "REGISTRY_USERNAME and REGISTRY_PASSWORD secrets are required" >&2 exit 1 fi - echo "${REGISTRY_PASSWORD}" \ - | docker login "${REGISTRY_HOST}" \ - -u "${REGISTRY_USER}" \ - --password-stdin + printf '%s' "${REGISTRY_PASSWORD}" \ + | docker login "${REGISTRY_HOST}" --username "${REGISTRY_USERNAME}" --password-stdin - WEB_TAGS=( - -t "${WEB_REF}:dev-${SHA_SHORT}" - -t "${WEB_REF}:dev" - ) - CELERY_TAGS=( - -t "${CELERY_REF}:dev-${SHA_SHORT}" - -t "${CELERY_REF}:dev" - ) - - if ! docker buildx inspect mostovik-builder >/dev/null 2>&1; then - docker buildx create --name mostovik-builder --use - else - docker buildx use mostovik-builder - fi + builder="mostovik-backend-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" + docker buildx create --name "${builder}" --driver docker-container --use + cleanup() { + docker buildx rm "${builder}" >/dev/null 2>&1 || true + rm -f web-metadata.json celery-metadata.json + } + trap cleanup EXIT docker buildx inspect --bootstrap - lockfile_sha="$(sha256sum uv.lock | awk '{ print $1 }')" - - golden_usable() { - local ref="$1" - - docker run --rm "${ref}:${GOLDEN_TAG}" \ - bash -c " - set -euo pipefail - test -f /app/uv.lock - test \"\$(sha256sum /app/uv.lock | awk '{ print \$1 }')\" = '${lockfile_sha}' - /app/.venv/bin/python -c 'import polymorphic' - " - } - - ensure_golden() { - local target="$1" - local ref="$2" - local build_args=() - if [ "${target}" = "celery-deps-base" ]; then - build_args+=(--build-arg "GOLDEN_WEB_IMAGE=${WEB_GOLDEN_REF}:${GOLDEN_TAG}") - fi - if docker buildx imagetools inspect "${ref}:${GOLDEN_TAG}" >/dev/null 2>&1 \ - && golden_usable "${ref}"; then - return 0 - fi - - docker buildx prune --all --force || true - docker builder prune --all --force || true - docker buildx build \ - -f ./docker/Dockerfile \ - --target "${target}" \ - "${build_args[@]}" \ - --push \ - -t "${ref}:${GOLDEN_TAG}" \ - -t "${ref}:latest" \ - . - } - - ensure_golden "web-deps-base" "${WEB_GOLDEN_REF}" - ensure_golden "celery-deps-base" "${CELERY_GOLDEN_REF}" + sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)" + image_tag="${GITHUB_REF_NAME}-${sha_short}" + registry_path="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + web_repository="${registry_path}/${WEB_IMAGE}" + celery_repository="${registry_path}/${CELERY_IMAGE}" + build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" docker buildx build \ - -f ./docker/Dockerfile \ + --platform linux/amd64 \ + --file docker/Dockerfile \ --target runtime-web \ - --build-arg INSTALL_DEV=false \ - --build-arg GOLDEN_WEB_IMAGE="${WEB_GOLDEN_REF}:${GOLDEN_TAG}" \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.created=${BUILD_TIME}" \ + --label "org.opencontainers.image.created=${build_time}" \ + --metadata-file web-metadata.json \ --push \ - "${WEB_TAGS[@]}" \ + --tag "${web_repository}:${image_tag}" \ . docker buildx build \ - -f ./docker/Dockerfile \ + --platform linux/amd64 \ + --file docker/Dockerfile \ --target runtime-celery \ - --build-arg INSTALL_DEV=false \ - --build-arg GOLDEN_CELERY_IMAGE="${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.created=${BUILD_TIME}" \ + --label "org.opencontainers.image.created=${build_time}" \ + --metadata-file celery-metadata.json \ --push \ - "${CELERY_TAGS[@]}" \ + --tag "${celery_repository}:${image_tag}" \ . + web_digest="$(jq -r '."containerimage.digest" // empty' web-metadata.json)" + celery_digest="$(jq -r '."containerimage.digest" // empty' celery-metadata.json)" + printf '%s\n' "${web_digest}" | grep -Eq '^sha256:[0-9a-f]{64}$' + printf '%s\n' "${celery_digest}" | grep -Eq '^sha256:[0-9a-f]{64}$' + { - echo "Registry API: ${REGISTRY_API_URL}" - echo "Golden images:" - echo "- ${WEB_GOLDEN_REF}:${GOLDEN_TAG}" - echo "- ${CELERY_GOLDEN_REF}:${GOLDEN_TAG}" - echo "Pushed images:" - echo "- ${WEB_REF}:dev-${SHA_SHORT}" - echo "- ${WEB_REF}:dev" - echo "- ${CELERY_REF}:dev-${SHA_SHORT}" - echo "- ${CELERY_REF}:dev" - } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" - - notify: - name: Internal Notify - runs-on: ubuntu-latest - timeout-minutes: 1 - needs: [quality, build_push] - if: ${{ always() && github.event_name != 'workflow_dispatch' }} - - steps: - - name: Send CI status webhook - continue-on-error: true - env: - CI_NOTIFY_WEBHOOK_URL: ${{ secrets.CI_NOTIFY_WEBHOOK_URL }} - CI_NOTIFY_TOKEN: ${{ secrets.CI_NOTIFY_TOKEN }} - QUALITY_RESULT: ${{ needs.quality.result }} - BUILD_PUSH_RESULT: ${{ needs.build_push.result }} - run: | - set -euo pipefail - if [ -z "${CI_NOTIFY_WEBHOOK_URL:-}" ]; then - echo "CI_NOTIFY_WEBHOOK_URL is not set; skip internal notification" - exit 0 - fi - - STATUS="success" - if [ "${QUALITY_RESULT}" != "success" ]; then - STATUS="${QUALITY_RESULT}" - elif [ "${BUILD_PUSH_RESULT}" = "failure" ] || [ "${BUILD_PUSH_RESULT}" = "cancelled" ]; then - STATUS="${BUILD_PUSH_RESULT}" - fi - export STATUS - - PAYLOAD=$(python3 - <<'PY' - import json - import os - - payload = { - "project": os.environ.get("GITHUB_REPOSITORY"), - "workflow": os.environ.get("GITHUB_WORKFLOW"), - "status": os.environ.get("STATUS"), - "branch": os.environ.get("GITHUB_HEAD_REF") or os.environ.get("GITHUB_REF_NAME"), - "sha": os.environ.get("GITHUB_SHA"), - "actor": os.environ.get("GITHUB_ACTOR"), - "server_url": os.environ.get("GITHUB_SERVER_URL"), - "run_id": os.environ.get("GITHUB_RUN_ID"), - "results": { - "quality": os.environ.get("QUALITY_RESULT"), - "build_push": os.environ.get("BUILD_PUSH_RESULT"), - }, - } - print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) - PY - ) - - AUTH_HEADER=() - if [ -n "${CI_NOTIFY_TOKEN:-}" ]; then - AUTH_HEADER=(-H "Authorization: Bearer ${CI_NOTIFY_TOKEN}") - fi - - curl -fsS \ - --connect-timeout 3 \ - --max-time 8 \ - --retry 1 \ - -H "Content-Type: application/json" \ - "${AUTH_HEADER[@]}" \ - --data "${PAYLOAD}" \ - "${CI_NOTIFY_WEBHOOK_URL}" + echo "web_ref=${web_repository}@${web_digest}" + echo "celery_ref=${celery_repository}@${celery_digest}" + echo "web_tag_ref=${web_repository}:${image_tag}" + echo "celery_tag_ref=${celery_repository}:${image_tag}" + } >> "${GITHUB_OUTPUT}" deploy_dev: - name: Deploy Dev via Compose + name: Deploy dev runs-on: ubuntu-latest - timeout-minutes: 5 - needs: [build_push] - if: needs.build_push.result == 'success' + timeout-minutes: 15 + needs: [build] + if: ${{ github.ref == 'refs/heads/dev' && needs.build.result == 'success' }} steps: - - name: Checkout code - run: | - set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Deploy prebuilt images via SSH + - name: Invoke dev release wrapper env: DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} DEPLOY_USER: ${{ secrets.DEPLOY_USER }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - REGISTRY_HOST: ${{ secrets.REGISTRY_HOST }} - HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + INTERNAL_DEPLOY_SSH_KEY_B64: ${{ secrets.INTERNAL_DEPLOY_SSH_KEY_B64 }} + INTERNAL_KNOWN_HOSTS_B64: ${{ secrets.INTERNAL_KNOWN_HOSTS_B64 }} + WEB_REF: ${{ needs.build.outputs.web_ref }} + CELERY_REF: ${{ needs.build.outputs.celery_ref }} run: | set -euo pipefail - - if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then - echo "Skip dev deploy for ${GITHUB_REF}" - exit 0 + if [ -z "${DEPLOY_HOST:-}" ] || [ -z "${DEPLOY_USER:-}" ] || [ -z "${INTERNAL_DEPLOY_SSH_KEY_B64:-}" ] || [ -z "${INTERNAL_KNOWN_HOSTS_B64:-}" ]; then + echo "Internal deploy SSH secrets are required" >&2 + exit 1 fi + case "${DEPLOY_USER}" in *[!A-Za-z0-9._-]*) echo "DEPLOY_USER has unsupported characters" >&2; exit 1;; esac + case "${DEPLOY_HOST}" in *[!A-Za-z0-9.:-]*) echo "DEPLOY_HOST has unsupported characters" >&2; exit 1;; esac + printf '%s\n' "${WEB_REF}" | grep -Eq '@sha256:[0-9a-f]{64}$' + printf '%s\n' "${CELERY_REF}" | grep -Eq '@sha256:[0-9a-f]{64}$' - case "${HEAD_COMMIT_MESSAGE:-}" in - *"#no_deploy"* | *"#no_image"*) - echo "Skip dev deploy because commit message disables deploy or image build" + ssh_dir="$(mktemp -d)" + trap 'rm -rf "${ssh_dir}"' EXIT + key_path="${ssh_dir}/deploy_key" + known_hosts_path="${ssh_dir}/known_hosts" + printf '%s' "${INTERNAL_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" + printf '%s' "${INTERNAL_KNOWN_HOSTS_B64}" | base64 -d > "${known_hosts_path}" + chmod 600 "${key_path}" "${known_hosts_path}" + ssh-keygen -y -f "${key_path}" >/dev/null + + ssh \ + -i "${key_path}" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=yes \ + -o "UserKnownHostsFile=${known_hosts_path}" \ + -o GlobalKnownHostsFile=/dev/null \ + -o ConnectTimeout=15 \ + "${DEPLOY_USER}@${DEPLOY_HOST}" \ + /opt/ecos-dev/release-service \ + mostovik-backend \ + "${GITHUB_RUN_ID:-${GITHUB_SHA}}-${GITHUB_RUN_ATTEMPT:-1}" \ + "${WEB_REF}" \ + "${CELERY_REF}" + + deploy_internal_main: + name: Deploy and verify internal main + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [build] + if: ${{ github.ref == 'refs/heads/main' && needs.build.result == 'success' }} + + steps: + - name: Refresh clone and invoke internal-main release wrapper + env: + MAIN_DEPLOY_HOST: ${{ secrets.MAIN_DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + INTERNAL_DEPLOY_SSH_KEY_B64: ${{ secrets.INTERNAL_DEPLOY_SSH_KEY_B64 }} + INTERNAL_KNOWN_HOSTS_B64: ${{ secrets.INTERNAL_KNOWN_HOSTS_B64 }} + INTERNAL_MAIN_MOSTOVIK_BACKEND_URL: ${{ secrets.INTERNAL_MAIN_MOSTOVIK_BACKEND_URL }} + WEB_REF: ${{ needs.build.outputs.web_ref }} + CELERY_REF: ${{ needs.build.outputs.celery_ref }} + run: | + set -euo pipefail + if [ -z "${MAIN_DEPLOY_HOST:-}" ] || [ -z "${DEPLOY_USER:-}" ] || [ -z "${INTERNAL_DEPLOY_SSH_KEY_B64:-}" ] || [ -z "${INTERNAL_KNOWN_HOSTS_B64:-}" ] || [ -z "${INTERNAL_MAIN_MOSTOVIK_BACKEND_URL:-}" ]; then + echo "Internal-main deploy and smoke secrets are required" >&2 + exit 1 + fi + case "${DEPLOY_USER}" in *[!A-Za-z0-9._-]*) echo "DEPLOY_USER has unsupported characters" >&2; exit 1;; esac + case "${MAIN_DEPLOY_HOST}" in *[!A-Za-z0-9.:-]*) echo "MAIN_DEPLOY_HOST has unsupported characters" >&2; exit 1;; esac + printf '%s\n' "${WEB_REF}" | grep -Eq '@sha256:[0-9a-f]{64}$' + printf '%s\n' "${CELERY_REF}" | grep -Eq '@sha256:[0-9a-f]{64}$' + + ssh_dir="$(mktemp -d)" + trap 'rm -rf "${ssh_dir}"' EXIT + key_path="${ssh_dir}/deploy_key" + known_hosts_path="${ssh_dir}/known_hosts" + printf '%s' "${INTERNAL_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" + printf '%s' "${INTERNAL_KNOWN_HOSTS_B64}" | base64 -d > "${known_hosts_path}" + chmod 600 "${key_path}" "${known_hosts_path}" + ssh-keygen -y -f "${key_path}" >/dev/null + + ssh \ + -i "${key_path}" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=yes \ + -o "UserKnownHostsFile=${known_hosts_path}" \ + -o GlobalKnownHostsFile=/dev/null \ + -o ConnectTimeout=15 \ + "${DEPLOY_USER}@${MAIN_DEPLOY_HOST}" \ + /opt/ecos-main/release-service \ + mostovik-backend \ + "${GITHUB_RUN_ID:-${GITHUB_SHA}}-${GITHUB_RUN_ATTEMPT:-1}" \ + "${WEB_REF}" \ + "${CELERY_REF}" \ + --refresh-data + + smoke_url="${INTERNAL_MAIN_MOSTOVIK_BACKEND_URL%/}/health/ready/" + for attempt in $(seq 1 30); do + if curl --fail --silent --output /dev/null --max-time 10 "${smoke_url}"; then + echo "Internal-main backend readiness passed" exit 0 - ;; - esac + fi + echo "Internal-main backend readiness attempt ${attempt}/30 failed" + sleep 5 + done + echo "Internal-main backend readiness failed" >&2 + exit 1 - short_sha="$(printf '%s' "${GITHUB_SHA}" | cut -c1-7)" - image_tag="dev-${short_sha}" - mkdir -p ~/.ssh - printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > ~/.ssh/ecos_deploy_key - chmod 0600 ~/.ssh/ecos_deploy_key - ssh-keyscan -H "${DEPLOY_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - tmp_current="$(mktemp)" - ssh -i ~/.ssh/ecos_deploy_key "${DEPLOY_USER}@${DEPLOY_HOST}" 'cat /opt/ecos-dev/releases/current.env' > "${tmp_current}" - grep -v '^MOSTOVIK_BACKEND_' "${tmp_current}" > "${tmp_current}.new" - cat >> "${tmp_current}.new" < /opt/ecos-dev/releases/current.env && rm -f /tmp/current.env && /opt/ecos-dev/deploy.sh mostovik-backend' + deploy_customer_main: + name: Deploy customer main + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [build, deploy_internal_main] + if: ${{ github.ref == 'refs/heads/main' && needs.build.result == 'success' && needs.deploy_internal_main.result == 'success' }} + + steps: + - name: Invoke customer release wrapper with the verified digests + env: + CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} + CUSTOMER_KNOWN_HOSTS_B64: ${{ secrets.CUSTOMER_KNOWN_HOSTS_B64 }} + WEB_REF: ${{ needs.build.outputs.web_ref }} + CELERY_REF: ${{ needs.build.outputs.celery_ref }} + run: | + set -euo pipefail + for name in CUSTOMER_DEPLOY_HOST CUSTOMER_DEPLOY_USER CUSTOMER_SSH_PROXY_HOST CUSTOMER_DEPLOY_SSH_KEY_B64 CUSTOMER_KNOWN_HOSTS_B64; do + if [ -z "${!name:-}" ]; then + echo "Missing required customer SSH secret: ${name}" >&2 + exit 1 + fi + done + case "${CUSTOMER_DEPLOY_USER}" in *[!A-Za-z0-9._-]*) echo "Customer SSH user has unsupported characters" >&2; exit 1;; esac + case "${CUSTOMER_DEPLOY_HOST}" in *[!A-Za-z0-9.:-]*) echo "Customer SSH host has unsupported characters" >&2; exit 1;; esac + case "${CUSTOMER_SSH_PROXY_HOST}" in *[!A-Za-z0-9._@:-]*) echo "Customer SSH proxy has unsupported characters" >&2; exit 1;; esac + printf '%s\n' "${WEB_REF}" | grep -Eq '@sha256:[0-9a-f]{64}$' + printf '%s\n' "${CELERY_REF}" | grep -Eq '@sha256:[0-9a-f]{64}$' + + ssh_dir="$(mktemp -d)" + trap 'rm -rf "${ssh_dir}"' EXIT + key_path="${ssh_dir}/deploy_key" + known_hosts_path="${ssh_dir}/known_hosts" + printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" + printf '%s' "${CUSTOMER_KNOWN_HOSTS_B64}" | base64 -d > "${known_hosts_path}" + chmod 600 "${key_path}" "${known_hosts_path}" + ssh-keygen -y -f "${key_path}" >/dev/null + + ssh_common=( + -i "${key_path}" + -o BatchMode=yes + -o IdentitiesOnly=yes + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=${known_hosts_path}" + -o GlobalKnownHostsFile=/dev/null + -o ConnectTimeout=15 + ) + proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${known_hosts_path} -o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}" + + ssh \ + "${ssh_common[@]}" \ + -o "ProxyCommand=${proxy_command}" \ + "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \ + /ecos/release-service \ + mostovik-backend \ + "${GITHUB_RUN_ID:-${GITHUB_SHA}}-${GITHUB_RUN_ATTEMPT:-1}" \ + "${WEB_REF}" \ + "${CELERY_REF}" diff --git a/.gitea/workflows/deploy-customer-main.yml b/.gitea/workflows/deploy-customer-main.yml deleted file mode 100644 index d0b88c1..0000000 --- a/.gitea/workflows/deploy-customer-main.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: Deploy Customer Main - -on: - push: - branches: - - main - workflow_dispatch: - -concurrency: - group: mostovik-backend-customer-main-${{ github.ref }} - cancel-in-progress: true - -env: - CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru - CUSTOMER_REGISTRY_NAMESPACE: avm - CUSTOMER_WEB_IMAGE: mostovik-backend-web - CUSTOMER_CELERY_IMAGE: mostovik-backend-celery - CUSTOMER_DEPLOY_USER: ecos - CUSTOMER_DEPLOY_HOST: 10.0.10.174 - CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121 - CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml - CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh - -jobs: - deploy: - name: Build, Push, Deploy - runs-on: [backend-docker] - timeout-minutes: 90 - - steps: - - name: Ensure main branch - run: | - set -euo pipefail - if [ "${GITHUB_REF_NAME}" != "main" ]; then - echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2 - exit 1 - fi - - - name: Checkout code - run: | - set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Validate deploy inputs - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} - CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - home_dir="${HOME:-/root}" - - if [ -z "${registry_user}" ]; then - echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2 - exit 1 - fi - if [ -z "${registry_password}" ]; then - echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2 - exit 1 - fi - if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \ - && [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \ - && [ -z "${DEPLOY_SSH_KEY:-}" ] \ - && [ ! -f "${home_dir}/.ssh/ci-key" ] \ - && [ ! -f "/root/.ssh/ci-key" ]; then - echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2 - exit 1 - fi - - - name: Setup Docker Buildx - run: | - set -euo pipefail - if ! docker buildx inspect customer-builder >/dev/null 2>&1; then - docker buildx create --name customer-builder --driver docker-container --use - else - docker buildx use customer-builder - fi - docker buildx inspect --bootstrap - - - name: Build and push customer images - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)" - registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}" - web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}" - celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}" - - printf '%s' "${registry_password}" \ - | docker login "${CUSTOMER_REGISTRY_HOST}" \ - -u "${registry_user}" \ - --password-stdin - - docker buildx build \ - --platform linux/amd64 \ - -f ./docker/Dockerfile \ - --target runtime-web \ - --build-arg INSTALL_DEV=false \ - --push \ - -t "${web_ref}:latest" \ - -t "${web_ref}:${sha_short}" \ - . - - docker buildx build \ - --platform linux/amd64 \ - -f ./docker/Dockerfile \ - --target runtime-celery \ - --build-arg INSTALL_DEV=false \ - --push \ - -t "${celery_ref}:latest" \ - -t "${celery_ref}:${sha_short}" \ - . - - - name: Deploy customer stack - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} - CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - case "${registry_user}" in - *[!A-Za-z0-9._@-]*) - echo "Registry user contains unsupported characters" >&2 - exit 1 - ;; - esac - - home_dir="${HOME:-/root}" - mkdir -p "${home_dir}/.ssh" - key_path="${home_dir}/.ssh/customer_deploy_key" - if [ -f "/root/.ssh/ci-key" ]; then - cp "/root/.ssh/ci-key" "${key_path}" - elif [ -f "${home_dir}/.ssh/ci-key" ]; then - cp "${home_dir}/.ssh/ci-key" "${key_path}" - elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then - printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" - elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then - printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}" - elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then - printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}" - else - echo "Customer deploy SSH key is unavailable" >&2 - exit 1 - fi - chmod 600 "${key_path}" - - ssh_common=( - -i "${key_path}" - -o BatchMode=yes - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o GlobalKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o ConnectTimeout=15 - ) - proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}" - remote_command="set -euo pipefail - flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'" - - ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true" - printf '%s' "${registry_password}" \ - | ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \ - "docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin" - ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}" diff --git a/.gitea/workflows/manual-dev-actions.yml b/.gitea/workflows/manual-dev-actions.yml deleted file mode 100644 index 502c43f..0000000 --- a/.gitea/workflows/manual-dev-actions.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Manual Dev Actions - -on: - workflow_dispatch: - inputs: - manual_action: - description: "Manual action: noop, build_golden_images, cleanup_dev_database, or dokploy_start" - required: true - default: "noop" - dokploy_target: - description: "Dokploy dev target: all, web, worker, or beat" - required: true - default: "all" - cleanup_confirm: - description: "Type CLEAN_DEV_DB to drop and recreate the dev database as UTF8" - required: false - default: "" - -concurrency: - group: mostovik-backend-manual-dev-${{ github.ref }} - cancel-in-progress: false - -env: - REGISTRY_API_URL: "https://registry.dev.nii-ecos.ru/v2/" - REGISTRY_HOST: "registry.dev.nii-ecos.ru" - REGISTRY_NAMESPACE: "${{ github.repository_owner }}" - WEB_IMAGE: "mostovik-backend-web" - CELERY_IMAGE: "mostovik-backend-celery" - CI_GOLDEN_IMAGE: "mostovik-backend-ci-golden" - WEB_GOLDEN_IMAGE: "mostovik-backend-web-golden" - CELERY_GOLDEN_IMAGE: "mostovik-backend-celery-golden" - GOLDEN_TAG: "py311-uv0.7.2" - DOKPLOY_DEV_WEB_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/_EjfuYBpzGJ18uPwBZ3iF" - DOKPLOY_DEV_WORKER_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/hltL7K2HmG1a8EIzr-mVA" - DOKPLOY_DEV_BEAT_WEBHOOK_URL: "https://deploy.dev.nii-ecos.ru/api/deploy/RkdykbqU6faErrZBAN9Rv" - DOKPLOY_API_URL: "https://deploy.dev.nii-ecos.ru/api" - DOKPLOY_DEV_WEB_APPLICATION_ID: "x2l_Twc2z2A4lJhMVqlNg" - DOKPLOY_DEV_WORKER_APPLICATION_ID: "m8ECastEeQKhDZVFonUTS" - DOKPLOY_DEV_BEAT_APPLICATION_ID: "Ut5e5mcMMslxG9Zrpbp0_" - DOKPLOY_DEV_WEB_APP_NAME: "service-backend-4mbxrs" - DOKPLOY_DEV_WORKER_APP_NAME: "service-backend-512y9c" - DOKPLOY_DEV_BEAT_APP_NAME: "service-backend-nvdyoq" - -jobs: - manual_dev_action: - name: Run Manual Dev Action - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - MANUAL_ACTION: ${{ github.event.inputs.manual_action }} - DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }} - CLEANUP_CONFIRM: ${{ github.event.inputs.cleanup_confirm }} - GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} - DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} - DOKPLOY_API_TOKEN_FALLBACK: "cmhRpAPDlWPCbwkCdteTgpHuHzhPHCNtZrUcRddsfiHdijmyXKsIIojiBmcVpfpo" - POSTGRES_HOST: "10.10.0.114" - POSTGRES_PORT: "5432" - POSTGRES_DB: "mostovik" - POSTGRES_USER: "postgres" - POSTGRES_PASSWORD: "postgres" - - steps: - - name: Checkout code - run: | - set -euo pipefail - REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Run selected action - run: bash scripts/ci/manual_dev_action.sh diff --git a/scripts/ci/dokploy_deploy_image.sh b/scripts/ci/dokploy_deploy_image.sh index bf94b37..4f35496 100755 --- a/scripts/ci/dokploy_deploy_image.sh +++ b/scripts/ci/dokploy_deploy_image.sh @@ -4,7 +4,7 @@ set -euo pipefail TARGET="${1:-${DOKPLOY_TARGET:-all}}" SUMMARY_FILE="${GITHUB_STEP_SUMMARY:-/dev/stdout}" DOKPLOY_API_URL="${DOKPLOY_API_URL:-https://deploy.dev.nii-ecos.ru/api}" -DOKPLOY_API_TOKEN="${DOKPLOY_API_TOKEN:-${DOKPLOY_API_TOKEN_FALLBACK:-}}" +DOKPLOY_API_TOKEN="${DOKPLOY_API_TOKEN:-}" if [ -n "${DOKPLOY_API_TOKEN}" ]; then echo "::add-mask::${DOKPLOY_API_TOKEN}" || true diff --git a/src/apps/core/management/commands/sanitize_cloned_environment.py b/src/apps/core/management/commands/sanitize_cloned_environment.py new file mode 100644 index 0000000..b7c49af --- /dev/null +++ b/src/apps/core/management/commands/sanitize_cloned_environment.py @@ -0,0 +1,97 @@ +"""Deactivate environment-bound state after restoring a database clone.""" + +import json + +from apps.core.management.commands.base import BaseAppCommand +from apps.core.models import BackgroundJob, JobStatus +from apps.exchange.models import ExchangeConnection +from apps.parsers.models import ( + CheckoCollectionAttempt, + FinancialReport, + ParserLoadLog, +) +from django.db.models import Q +from django.utils import timezone +from django_celery_beat.models import PeriodicTask, PeriodicTasks + +CLONE_TERMINATION_MESSAGE = "Closed after database clone" + + +class Command(BaseAppCommand): + """Make a restored database safe to start in another environment.""" + + help = "Disable schedules and close environment-bound state after a database clone" + use_transaction = True + + def execute_command(self, *args, **options) -> str: + now = timezone.now() + + periodic_tasks = PeriodicTask.objects.filter(enabled=True).update( + enabled=False, + date_changed=now, + ) + if periodic_tasks: + PeriodicTasks.update_changed() + + exchange_connections = ExchangeConnection.objects.filter( + Q(is_active=True) + | ~Q(password="") + | Q(last_checked_at__isnull=False) + | ~Q(last_error="") + ).update( + is_active=False, + password="", + last_checked_at=None, + last_error="", + updated_at=now, + ) + + background_jobs = BackgroundJob.objects.filter( + status__in=(JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY) + ).update( + status=JobStatus.REVOKED, + completed_at=now, + updated_at=now, + ) + + parser_load_logs = ParserLoadLog.objects.filter( + status__in=( + ParserLoadLog.Status.PENDING, + ParserLoadLog.Status.IN_PROGRESS, + ) + ).update( + status=ParserLoadLog.Status.FAILED, + error_message=CLONE_TERMINATION_MESSAGE, + updated_at=now, + ) + + checko_attempts = CheckoCollectionAttempt.objects.filter( + status=CheckoCollectionAttempt.Status.IN_PROGRESS + ).update( + status=CheckoCollectionAttempt.Status.FAILED, + error_message=CLONE_TERMINATION_MESSAGE, + updated_at=now, + ) + + financial_reports = FinancialReport.objects.filter( + status__in=( + FinancialReport.Status.PENDING, + FinancialReport.Status.PROCESSING, + ) + ).update( + status=FinancialReport.Status.FAILED, + error_message=CLONE_TERMINATION_MESSAGE, + updated_at=now, + ) + + return json.dumps( + { + "background_jobs": background_jobs, + "checko_attempts": checko_attempts, + "exchange_connections": exchange_connections, + "financial_reports": financial_reports, + "parser_load_logs": parser_load_logs, + "periodic_tasks": periodic_tasks, + }, + sort_keys=True, + ) diff --git a/src/apps/core/management/commands/validate_cloned_migrations.py b/src/apps/core/management/commands/validate_cloned_migrations.py new file mode 100644 index 0000000..b2e618a --- /dev/null +++ b/src/apps/core/management/commands/validate_cloned_migrations.py @@ -0,0 +1,40 @@ +"""Validate that a cloned database contains only migrations known to this image.""" + +from django.core.management.base import BaseCommand, CommandError +from django.db import DEFAULT_DB_ALIAS, connections +from django.db.migrations.loader import MigrationLoader +from django.db.migrations.recorder import MigrationRecorder + + +class Command(BaseCommand): + """Reject a clone produced by code newer than the candidate image.""" + + help = "Fail if the database has applied migrations absent from this image" + requires_migrations_checks = False + + def add_arguments(self, parser) -> None: + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, + choices=tuple(connections), + help="Database to validate", + ) + + def handle(self, *args, **options) -> str: + database = options["database"] + connection = connections[database] + disk_migrations = set(MigrationLoader(connection).disk_migrations) + applied_migrations = set(MigrationRecorder(connection).applied_migrations()) + unknown_migrations = sorted(applied_migrations - disk_migrations) + + if unknown_migrations: + formatted = ", ".join( + f"{app_label}.{migration_name}" + for app_label, migration_name in unknown_migrations + ) + raise CommandError( + "Cloned database has applied migrations absent from this image: " + f"{formatted}" + ) + + return "Cloned database migrations are compatible with this image" diff --git a/src/apps/core/views.py b/src/apps/core/views.py index 5f8bbe7..8e88940 100644 --- a/src/apps/core/views.py +++ b/src/apps/core/views.py @@ -191,6 +191,7 @@ class ReadinessView(APIView): permission_classes = [AllowAny] authentication_classes = [] + throttle_classes = [] @swagger_auto_schema( tags=[HEALTH_TAG], diff --git a/src/settings/base.py b/src/settings/base.py index a59e6d0..094cc56 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -7,6 +7,8 @@ import warnings from datetime import timedelta from pathlib import Path +from django.core.exceptions import ImproperlyConfigured + BASE_DIR = Path(__file__).resolve().parent.parent # Repository root (one level above src/) PROJECT_ROOT = BASE_DIR.parent @@ -17,6 +19,23 @@ STARTUP_CHECKS_ENABLED = True STARTUP_DB_TIMEOUT_SECONDS = 3 STARTUP_REDIS_TIMEOUT_SECONDS = 3 + +def _env_bool(name: str, *, default: bool) -> bool: + """Read a strict boolean environment flag with an explicit default.""" + raw_value = os.getenv(name) + if raw_value is None: + return default + + value = raw_value.strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + if value in {"0", "false", "no", "off"}: + return False + raise ImproperlyConfigured( + f"{name} must be one of: 1, true, yes, on, 0, false, no, off" + ) + + # Suppress noisy warning from legacy coreapi used by drf-yasg. warnings.filterwarnings( "ignore", @@ -271,7 +290,10 @@ CELERY_BROKER_CONNECTION_RETRY = True CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True # Celery: автообновление источников при старте worker. -CELERY_STARTUP_REFRESH_ENABLED = True +CELERY_STARTUP_REFRESH_ENABLED = _env_bool( + "CELERY_STARTUP_REFRESH_ENABLED", + default=True, +) CELERY_STARTUP_REFRESH_DELAY_SECONDS = 30 CELERY_STARTUP_REFRESH_LOCK_KEY = "celery:startup:parse_all_sources:lock" CELERY_STARTUP_REFRESH_LOCK_TTL_SECONDS = 3600 diff --git a/tests/apps/core/test_celery_module.py b/tests/apps/core/test_celery_module.py index 2d6b274..fc0f646 100644 --- a/tests/apps/core/test_celery_module.py +++ b/tests/apps/core/test_celery_module.py @@ -111,6 +111,21 @@ class CeleryModuleTest(SimpleTestCase): apply_async_mock.assert_not_called() + def test_startup_refresh_skips_when_disabled(self): + with patch.dict( + os.environ, {"DJANGO_SETTINGS_MODULE": "settings.test"}, clear=True + ), patch.object(sys, "argv", ["python", "manage.py", "shell"]): + module = _load_module("isolated_core_celery_startup_disabled") + + with patch.object( + module, + "settings", + SimpleNamespace(CELERY_STARTUP_REFRESH_ENABLED=False), + ), patch.object(module.cache, "add") as add_mock: + module._queue_startup_sources_refresh() + + add_mock.assert_not_called() + def test_debug_task_prints_request(self): with patch.dict( os.environ, {"DJANGO_SETTINGS_MODULE": "settings.test"}, clear=True diff --git a/tests/apps/core/test_cloned_environment_commands.py b/tests/apps/core/test_cloned_environment_commands.py new file mode 100644 index 0000000..9927018 --- /dev/null +++ b/tests/apps/core/test_cloned_environment_commands.py @@ -0,0 +1,264 @@ +"""Regression tests for commands used by the internal-main restore wrapper.""" + +import json +from datetime import date, timedelta +from io import StringIO + +from apps.core.management.commands.sanitize_cloned_environment import ( + CLONE_TERMINATION_MESSAGE, +) +from apps.core.models import BackgroundJob, JobStatus +from apps.exchange.models import ExchangeConnection +from apps.parsers.models import ( + CheckoCollectionAttempt, + FinancialReport, + FinancialReportLine, + ParserLoadLog, +) +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db.migrations.recorder import MigrationRecorder +from django.test import TestCase +from django.utils import timezone +from django_celery_beat.models import IntervalSchedule, PeriodicTask +from organizations.models import Organization + + +class SanitizeClonedEnvironmentCommandTest(TestCase): + def _run_command(self) -> dict[str, int]: + stdout = StringIO() + call_command( + "sanitize_cloned_environment", + silent=True, + stdout=stdout, + ) + return json.loads(stdout.getvalue()) + + def test_sanitizes_only_environment_bound_and_in_flight_state_idempotently(self): + interval = IntervalSchedule.objects.create( + every=1, + period=IntervalSchedule.HOURS, + ) + enabled_schedule = PeriodicTask.objects.create( + name="clone-enabled-schedule", + task="tests.clone.enabled", + interval=interval, + enabled=True, + ) + disabled_schedule = PeriodicTask.objects.create( + name="clone-disabled-schedule", + task="tests.clone.disabled", + interval=interval, + enabled=False, + ) + + checked_at = timezone.now() - timedelta(hours=1) + connection = ExchangeConnection.objects.create( + server="database.internal", + username="clone-user", + password="environment-secret", # noqa: S106 - synthetic test value + database_name="environment-db", + is_active=True, + last_checked_at=checked_at, + last_error="old check error", + ) + clean_connection = ExchangeConnection.objects.create( + server="disabled.internal", + username="disabled-user", + password="", + database_name="disabled-db", + is_active=False, + ) + + active_jobs = [ + BackgroundJob.objects.create( + task_id=f"clone-{status}", + task_name="tests.clone", + status=status, + result={"partial": status}, + ) + for status in (JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY) + ] + completed_jobs = [ + BackgroundJob.objects.create( + task_id=f"clone-finished-{status}", + task_name="tests.clone", + status=status, + result={"preserved": status}, + completed_at=checked_at, + ) + for status in (JobStatus.SUCCESS, JobStatus.FAILURE, JobStatus.REVOKED) + ] + + active_load_logs = [ + ParserLoadLog.objects.create( + batch_id=index, + source=ParserLoadLog.Source.INDUSTRIAL, + status=status, + records_count=index, + ) + for index, status in enumerate( + (ParserLoadLog.Status.PENDING, ParserLoadLog.Status.IN_PROGRESS), + start=1, + ) + ] + completed_load_log = ParserLoadLog.objects.create( + batch_id=3, + source=ParserLoadLog.Source.INDUSTRIAL, + status=ParserLoadLog.Status.SUCCESS, + records_count=37, + ) + + organization = Organization.objects.create(name="Clone command test") + active_attempt = CheckoCollectionAttempt.objects.create( + organization=organization, + source=CheckoCollectionAttempt.Source.ARBITRATION, + period_month=date(2026, 8, 1), + status=CheckoCollectionAttempt.Status.IN_PROGRESS, + records_count=4, + ) + completed_attempt = CheckoCollectionAttempt.objects.create( + organization=organization, + source=CheckoCollectionAttempt.Source.BANKRUPTCY, + period_month=date(2026, 8, 1), + status=CheckoCollectionAttempt.Status.SUCCESS, + records_count=11, + ) + + active_reports = [ + FinancialReport.objects.create( + external_id=f"clone-active-{index}", + ogrn=f"100000000000{index}", + file_name=f"active-{index}.xlsx", + file_hash=str(index) * 64, + load_batch=index, + status=status, + source=FinancialReport.SourceType.API, + ) + for index, status in enumerate( + (FinancialReport.Status.PENDING, FinancialReport.Status.PROCESSING), + start=1, + ) + ] + completed_report = FinancialReport.objects.create( + external_id="clone-completed", + ogrn="1000000000003", + file_name="completed.xlsx", + file_hash="3" * 64, + load_batch=3, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + completed_line = FinancialReportLine.objects.create( + report=completed_report, + form_code="1", + line_code="1100", + line_name="Preserved business data", + year=2025, + period_start=10, + period_end=20, + ) + + first_result = self._run_command() + + self.assertEqual( + first_result, + { + "background_jobs": 3, + "checko_attempts": 1, + "exchange_connections": 1, + "financial_reports": 2, + "parser_load_logs": 2, + "periodic_tasks": 1, + }, + ) + + enabled_schedule.refresh_from_db() + disabled_schedule.refresh_from_db() + self.assertFalse(enabled_schedule.enabled) + self.assertFalse(disabled_schedule.enabled) + + connection.refresh_from_db() + self.assertFalse(connection.is_active) + self.assertEqual(connection.password, "") + self.assertIsNone(connection.last_checked_at) + self.assertEqual(connection.last_error, "") + clean_connection.refresh_from_db() + self.assertEqual(clean_connection.password, "") + + for job in active_jobs: + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.REVOKED) + self.assertIsNotNone(job.completed_at) + self.assertEqual( + job.result, {"partial": job.task_id.removeprefix("clone-")} + ) + for job in completed_jobs: + job.refresh_from_db() + self.assertEqual(job.result, {"preserved": job.status}) + self.assertEqual(job.completed_at, checked_at) + + for load_log in active_load_logs: + load_log.refresh_from_db() + self.assertEqual(load_log.status, ParserLoadLog.Status.FAILED) + self.assertEqual(load_log.error_message, CLONE_TERMINATION_MESSAGE) + completed_load_log.refresh_from_db() + self.assertEqual(completed_load_log.status, ParserLoadLog.Status.SUCCESS) + self.assertEqual(completed_load_log.records_count, 37) + + active_attempt.refresh_from_db() + self.assertEqual(active_attempt.status, CheckoCollectionAttempt.Status.FAILED) + self.assertEqual(active_attempt.error_message, CLONE_TERMINATION_MESSAGE) + completed_attempt.refresh_from_db() + self.assertEqual( + completed_attempt.status, CheckoCollectionAttempt.Status.SUCCESS + ) + self.assertEqual(completed_attempt.records_count, 11) + + for report in active_reports: + report.refresh_from_db() + self.assertEqual(report.status, FinancialReport.Status.FAILED) + self.assertEqual(report.error_message, CLONE_TERMINATION_MESSAGE) + completed_report.refresh_from_db() + completed_line.refresh_from_db() + self.assertEqual(completed_report.status, FinancialReport.Status.SUCCESS) + self.assertEqual(completed_line.period_start, 10) + self.assertEqual(completed_line.period_end, 20) + + first_timestamps = { + "connection": connection.updated_at, + "job": active_jobs[0].updated_at, + "load_log": active_load_logs[0].updated_at, + "attempt": active_attempt.updated_at, + "report": active_reports[0].updated_at, + } + second_result = self._run_command() + + self.assertEqual(second_result, {key: 0 for key in first_result}) + connection.refresh_from_db() + active_jobs[0].refresh_from_db() + active_load_logs[0].refresh_from_db() + active_attempt.refresh_from_db() + active_reports[0].refresh_from_db() + self.assertEqual(connection.updated_at, first_timestamps["connection"]) + self.assertEqual(active_jobs[0].updated_at, first_timestamps["job"]) + self.assertEqual(active_load_logs[0].updated_at, first_timestamps["load_log"]) + self.assertEqual(active_attempt.updated_at, first_timestamps["attempt"]) + self.assertEqual(active_reports[0].updated_at, first_timestamps["report"]) + + +class ValidateClonedMigrationsCommandTest(TestCase): + def test_accepts_database_when_all_applied_migrations_exist_in_image(self): + call_command("validate_cloned_migrations", stdout=StringIO()) + + def test_rejects_applied_migration_absent_from_image(self): + MigrationRecorder.Migration.objects.create( + app="removed_clone_app", + name="9999_unknown_migration", + ) + + with self.assertRaisesMessage( + CommandError, + "removed_clone_app.9999_unknown_migration", + ): + call_command("validate_cloned_migrations", stdout=StringIO()) diff --git a/tests/apps/core/test_views.py b/tests/apps/core/test_views.py index 2d688a0..e4202bf 100644 --- a/tests/apps/core/test_views.py +++ b/tests/apps/core/test_views.py @@ -3,6 +3,7 @@ import sys import types from datetime import timedelta +from unittest.mock import patch from apps.core import views as core_views from apps.core.views import HealthCheckView @@ -10,6 +11,7 @@ from django.urls import reverse from django.utils import timezone from rest_framework import status from rest_framework.test import APIRequestFactory, APITestCase +from rest_framework.views import APIView from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake @@ -245,6 +247,20 @@ class ReadinessViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["status"], "ready") + def test_readiness_bypasses_global_throttling(self): + class _RejectAllThrottle: + def allow_request(self, request, view): + return False + + def wait(self): + return None + + with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]): + response = self.client.get(reverse("core:readiness")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["status"], "ready") + def test_readiness_returns_not_ready_on_db_error(self): original_connection = core_views.connection diff --git a/tests/settings/test_base_settings.py b/tests/settings/test_base_settings.py new file mode 100644 index 0000000..6533ba0 --- /dev/null +++ b/tests/settings/test_base_settings.py @@ -0,0 +1,29 @@ +import pytest +from django.core.exceptions import ImproperlyConfigured +from settings.base import _env_bool + + +def test_env_bool_uses_explicit_default(monkeypatch): + monkeypatch.delenv("TEST_BOOLEAN_FLAG", raising=False) + + assert _env_bool("TEST_BOOLEAN_FLAG", default=True) is True + assert _env_bool("TEST_BOOLEAN_FLAG", default=False) is False + + +def test_env_bool_accepts_supported_false_values(monkeypatch): + for value in ("0", "false", "NO", " off "): + monkeypatch.setenv("TEST_BOOLEAN_FLAG", value) + assert _env_bool("TEST_BOOLEAN_FLAG", default=True) is False + + +def test_env_bool_accepts_supported_true_values(monkeypatch): + for value in ("1", "true", "YES", " on "): + monkeypatch.setenv("TEST_BOOLEAN_FLAG", value) + assert _env_bool("TEST_BOOLEAN_FLAG", default=False) is True + + +def test_env_bool_rejects_ambiguous_values(monkeypatch): + monkeypatch.setenv("TEST_BOOLEAN_FLAG", "enabled") + + with pytest.raises(ImproperlyConfigured, match="TEST_BOOLEAN_FLAG must be one of"): + _env_bool("TEST_BOOLEAN_FLAG", default=True)