From 9194ff3d0da7c2855442d5798f13226cacf598fd Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Thu, 27 Aug 2026 12:49:43 +0300 Subject: [PATCH] feat: add sanitized internal main deployment --- .gitea/workflows/ci-cd.yml | 370 ++++++++++-------- .gitea/workflows/deploy-customer-main.yml | 192 --------- scripts/ci/release-service.sh | 141 +++++++ scripts/ci/smoke.sh | 26 ++ .../commands/sanitize_cloned_environment.py | 61 +++ .../commands/validate_cloned_migrations.py | 46 +++ ...est_sanitize_cloned_environment_command.py | 212 ++++++++++ ...test_validate_cloned_migrations_command.py | 41 ++ 8 files changed, 742 insertions(+), 347 deletions(-) delete mode 100644 .gitea/workflows/deploy-customer-main.yml create mode 100755 scripts/ci/release-service.sh create mode 100755 scripts/ci/smoke.sh create mode 100644 src/apps/core/management/commands/sanitize_cloned_environment.py create mode 100644 src/apps/core/management/commands/validate_cloned_migrations.py create mode 100644 tests/apps/core/test_sanitize_cloned_environment_command.py create mode 100644 tests/apps/core/test_validate_cloned_migrations_command.py diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index b611f67..5a1783a 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -1,47 +1,47 @@ -name: CI/CD Pipeline +name: State Corp Backend CI/CD on: push: branches: - - main - - develop - dev + # Add main only after the first manual main restore/rollback drill passes. pull_request: branches: - - main - - develop - dev + - main + workflow_dispatch: + +concurrency: + group: state-corp-backend-${{ github.ref }} + cancel-in-progress: false env: PYTHON_VERSION: "3.11" - REGISTRY_HOST: "registry.dev.nii-ecos.ru" - REGISTRY_NAMESPACE: "${{ github.repository_owner }}" - WEB_IMAGE: "state-corp-backend-web" - CELERY_IMAGE: "state-corp-backend-celery" + REGISTRY_HOST: registry.dev.nii-ecos.ru + REGISTRY_NAMESPACE: avm + WEB_IMAGE: state-corp-backend-web + CELERY_IMAGE: state-corp-backend-celery jobs: - lint: - name: Code Quality Checks + quality: + name: Quality gate runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 steps: - - name: Checkout code - run: | - set -euo pipefail - REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . - git checkout "${GITHUB_SHA}" + - name: Checkout + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false - name: Install Python and uv run: | set -euo pipefail - PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" - PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")" - printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env + project_python_version="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" + python_bin="$(./scripts/ensure-ci-python.sh "${project_python_version}")" + printf 'PYTHON_BIN=%s\n' "${python_bin}" > .ci-python-env - - name: Create virtual environment and install dependencies + - name: Install dependencies run: | set -euo pipefail . ./.ci-python-env @@ -56,55 +56,14 @@ jobs: --no-managed-python \ --no-python-downloads - - name: Run Ruff linting + - name: Lint and formatting run: | set -euo pipefail . .venv/bin/activate ruff check src tests scripts - - - name: Run Ruff formatting check - run: | - set -euo pipefail - . .venv/bin/activate ruff format src tests scripts --check - test: - name: Run Tests - runs-on: ubuntu-latest - timeout-minutes: 20 - - steps: - - name: Checkout code - run: | - set -euo pipefail - REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . - git checkout "${GITHUB_SHA}" - - - name: Install Python and uv - run: | - set -euo pipefail - PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" - PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")" - printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env - - - name: Create virtual environment and install dependencies - run: | - set -euo pipefail - . ./.ci-python-env - "${PYTHON_BIN}" -m venv .venv - . .venv/bin/activate - python -m pip install --upgrade pip uv - uv sync \ - --dev \ - --frozen \ - --active \ - --python "${PYTHON_BIN}" \ - --no-managed-python \ - --no-python-downloads - - - name: Run pytest suite + - name: Tests env: DJANGO_SETTINGS_MODULE: settings.test SECRET_KEY: test-secret-key-for-ci @@ -113,126 +72,227 @@ jobs: export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" .venv/bin/python -m pytest tests -q - build_push: - name: Build and Push Dev Images - runs-on: ubuntu-latest - timeout-minutes: 45 - needs: [lint, test] - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') + build: + name: Build linux/amd64 images once + runs-on: [backend-docker] + timeout-minutes: 60 + needs: [quality] + if: >- + github.event_name != 'pull_request' && + (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') + outputs: + celery_ref: ${{ steps.image_refs.outputs.celery_ref }} + celery_tag_ref: ${{ steps.image_refs.outputs.celery_tag_ref }} + run_id: ${{ steps.image_refs.outputs.run_id }} + web_ref: ${{ steps.image_refs.outputs.web_ref }} + web_tag_ref: ${{ steps.image_refs.outputs.web_tag_ref }} steps: - - name: Checkout code - run: | - set -euo pipefail - REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . - git checkout "${GITHUB_SHA}" + - name: Checkout + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false - - name: Build and push images + - name: Validate registry credentials env: - REGISTRY_USER: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} run: | set -euo pipefail - if [ -z "${REGISTRY_USER:-}" ] || [ -z "${REGISTRY_PASSWORD:-}" ]; then - echo "Registry credentials are not configured" >&2 + [ -n "${REGISTRY_USERNAME:-}" ] || { + echo 'Registry username is not configured.' >&2 exit 1 - fi + } + [ -n "${REGISTRY_PASSWORD:-}" ] || { + echo 'Registry password/token is not configured.' >&2 + exit 1 + } - SHA_SHORT="$(printf '%s' "${GITHUB_SHA}" | cut -c1-7)" - BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - REGISTRY="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" - WEB_REF="${REGISTRY}/${WEB_IMAGE}" - CELERY_REF="${REGISTRY}/${CELERY_IMAGE}" + - name: Prepare Buildx + run: | + set -euo pipefail + builder_name="state-corp-backend-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" + docker buildx create \ + --driver docker-container \ + --name "${builder_name}" \ + --use + docker buildx inspect --bootstrap + printf 'BUILDER_NAME=%s\n' "${builder_name}" >> "${GITHUB_ENV}" - echo "${REGISTRY_PASSWORD}" \ + - name: Build and push immutable images + id: image_refs + env: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + run: | + set -euo pipefail + + registry_path="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + web_image="${registry_path}/${WEB_IMAGE}" + celery_image="${registry_path}/${CELERY_IMAGE}" + short_sha="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)" + branch="${GITHUB_REF_NAME}" + immutable_tag="${branch}-${short_sha}" + build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + printf '%s' "${REGISTRY_PASSWORD}" \ | docker login "${REGISTRY_HOST}" \ - -u "${REGISTRY_USER}" \ + --username "${REGISTRY_USERNAME}" \ --password-stdin - if ! docker buildx inspect state-corp-builder >/dev/null 2>&1; then - docker buildx create --name state-corp-builder --use - else - docker buildx use state-corp-builder - fi - docker buildx inspect --bootstrap - docker buildx build \ - -f ./docker/Dockerfile \ + --platform linux/amd64 \ + --file ./docker/Dockerfile \ --target runtime-web \ --build-arg INSTALL_DEV=false \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.created=${BUILD_TIME}" \ - --tag "${WEB_REF}:dev-${SHA_SHORT}" \ - --tag "${WEB_REF}:dev" \ + --label "org.opencontainers.image.created=${build_time}" \ + --tag "${web_image}:${immutable_tag}" \ + --metadata-file web-metadata.json \ --push \ . docker buildx build \ - -f ./docker/Dockerfile \ + --platform linux/amd64 \ + --file ./docker/Dockerfile \ --target runtime-celery \ --build-arg INSTALL_DEV=false \ --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.created=${BUILD_TIME}" \ - --tag "${CELERY_REF}:dev-${SHA_SHORT}" \ - --tag "${CELERY_REF}:dev" \ + --label "org.opencontainers.image.created=${build_time}" \ + --tag "${celery_image}:${immutable_tag}" \ + --metadata-file celery-metadata.json \ --push \ . - deploy_dev: - name: Deploy Dev via Compose - runs-on: ubuntu-latest - timeout-minutes: 5 - needs: [build_push] - if: needs.build_push.result == 'success' + metadata_digest() { + sed -n \ + 's/^[[:space:]]*"containerimage.digest":[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$1" \ + | head -n 1 + } - steps: - - name: Checkout code + web_digest="$(metadata_digest web-metadata.json)" + celery_digest="$(metadata_digest celery-metadata.json)" + digest_pattern='^sha256:[a-f0-9]{64}$' + [[ "${web_digest}" =~ ${digest_pattern} ]] || { + echo 'Buildx did not return a valid web image digest.' >&2 + exit 1 + } + [[ "${celery_digest}" =~ ${digest_pattern} ]] || { + echo 'Buildx did not return a valid Celery image digest.' >&2 + exit 1 + } + + release_run_id="${GITHUB_RUN_ID:-${GITHUB_SHA}}-${GITHUB_RUN_ATTEMPT:-1}" + { + printf 'web_ref=%s@%s\n' "${web_image}" "${web_digest}" + printf 'celery_ref=%s@%s\n' "${celery_image}" "${celery_digest}" + printf 'web_tag_ref=%s:%s\n' "${web_image}" "${immutable_tag}" + printf 'celery_tag_ref=%s:%s\n' "${celery_image}" "${immutable_tag}" + printf 'run_id=%s\n' "${release_run_id}" + } >> "${GITHUB_OUTPUT}" + + - name: Remove Buildx builder + if: always() run: | - set -euo pipefail - REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . - git checkout "${GITHUB_SHA}" - - - name: Deploy prebuilt images via SSH - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - DEPLOY_USER: ${{ secrets.DEPLOY_USER }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - REGISTRY_HOST: ${{ secrets.REGISTRY_HOST }} - HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: | - set -euo pipefail - - if [ "${GITHUB_REF}" != "refs/heads/dev" ]; then - echo "Skip dev deploy for ${GITHUB_REF}" - exit 0 + if [ -n "${BUILDER_NAME:-}" ]; then + docker buildx rm "${BUILDER_NAME}" || true fi - case "${HEAD_COMMIT_MESSAGE:-}" in - *"#no_deploy"* | *"#no_image"*) - echo "Skip dev deploy because commit message disables deploy or image build" - exit 0 - ;; - esac + deploy_dev: + name: Release dev + runs-on: [backend-docker] + timeout-minutes: 20 + needs: [build] + if: needs.build.result == 'success' && github.ref == 'refs/heads/dev' - short_sha="$(printf '%s' "${GITHUB_SHA}" | cut -c1-7)" - image_tag="dev-${short_sha}" - mkdir -p ~/.ssh - printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > ~/.ssh/ecos_deploy_key - chmod 0600 ~/.ssh/ecos_deploy_key - ssh-keyscan -H "${DEPLOY_HOST}" >> ~/.ssh/known_hosts 2>/dev/null - tmp_current="$(mktemp)" - ssh -i ~/.ssh/ecos_deploy_key "${DEPLOY_USER}@${DEPLOY_HOST}" 'cat /opt/ecos-dev/releases/current.env' > "${tmp_current}" - grep -v '^STATE_CORP_BACKEND_' "${tmp_current}" > "${tmp_current}.new" - cat >> "${tmp_current}.new" < /opt/ecos-dev/releases/current.env && rm -f /tmp/current.env && /opt/ecos-dev/deploy.sh state-corp-backend' - ssh -i ~/.ssh/ecos_deploy_key "${DEPLOY_USER}@${DEPLOY_HOST}" 'cd /opt/ecos-dev && docker compose --env-file runtime.env --env-file releases/current.env -f compose.yml run --rm --no-deps state-corp-web python src/manage.py migrate --noinput' + steps: + - name: Checkout + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false + + - name: Release exact images + env: + RELEASE_HOST: ${{ secrets.DEPLOY_HOST }} + RELEASE_KNOWN_HOSTS_B64: ${{ secrets.INTERNAL_KNOWN_HOSTS_B64 }} + RELEASE_SERVICE_PATH: /opt/ecos-dev/release-service + RELEASE_SSH_KEY_B64: ${{ secrets.INTERNAL_DEPLOY_SSH_KEY_B64 }} + RELEASE_USER: ${{ secrets.DEPLOY_USER }} + run: | + ./scripts/ci/release-service.sh \ + state-corp-backend \ + "${{ needs.build.outputs.run_id }}" \ + "${{ needs.build.outputs.web_ref }}" \ + "${{ needs.build.outputs.celery_ref }}" + + - name: Smoke dev backend + env: + APP_BASE_URL: ${{ secrets.APP_BASE_URL }} + run: ./scripts/ci/smoke.sh "${APP_BASE_URL%/}/health/ready/" + + deploy_main_internal: + name: Refresh and release internal main + runs-on: [backend-docker] + timeout-minutes: 60 + needs: [build] + if: needs.build.result == 'success' && github.ref == 'refs/heads/main' + + steps: + - name: Checkout + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false + + - name: Refresh cloned data and release exact images + env: + RELEASE_HOST: ${{ secrets.MAIN_DEPLOY_HOST }} + RELEASE_KNOWN_HOSTS_B64: ${{ secrets.INTERNAL_KNOWN_HOSTS_B64 }} + RELEASE_SERVICE_PATH: /opt/ecos-main/release-service + RELEASE_SSH_KEY_B64: ${{ secrets.INTERNAL_DEPLOY_SSH_KEY_B64 }} + RELEASE_USER: ${{ secrets.DEPLOY_USER }} + run: | + ./scripts/ci/release-service.sh \ + state-corp-backend \ + "${{ needs.build.outputs.run_id }}" \ + "${{ needs.build.outputs.web_ref }}" \ + "${{ needs.build.outputs.celery_ref }}" \ + --refresh-data + + - name: Smoke internal main backend + env: + INTERNAL_MAIN_STATE_CORP_BACKEND_URL: ${{ secrets.INTERNAL_MAIN_STATE_CORP_BACKEND_URL }} + run: ./scripts/ci/smoke.sh "${INTERNAL_MAIN_STATE_CORP_BACKEND_URL%/}/health/ready/" + + deploy_main_customer: + name: Release customer main + runs-on: [backend-docker] + timeout-minutes: 30 + needs: [build, deploy_main_internal] + if: >- + needs.build.result == 'success' && + needs.deploy_main_internal.result == 'success' && + github.ref == 'refs/heads/main' + + steps: + - name: Checkout + uses: actions/checkout@v4.2.2 + with: + persist-credentials: false + + - name: Release the same exact images without refresh + env: + RELEASE_HOST: 10.0.10.174 + RELEASE_KNOWN_HOSTS_B64: ${{ secrets.CUSTOMER_KNOWN_HOSTS_B64 }} + RELEASE_PROXY: ecos-proxy@10.10.0.121 + RELEASE_SERVICE_PATH: /ecos/release-service + RELEASE_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} + RELEASE_USER: ecos + run: | + ./scripts/ci/release-service.sh \ + state-corp-backend \ + "${{ needs.build.outputs.run_id }}" \ + "${{ needs.build.outputs.web_ref }}" \ + "${{ needs.build.outputs.celery_ref }}" diff --git a/.gitea/workflows/deploy-customer-main.yml b/.gitea/workflows/deploy-customer-main.yml deleted file mode 100644 index cdf4e3b..0000000 --- a/.gitea/workflows/deploy-customer-main.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: Deploy Customer Main - -on: - push: - branches: - - main - workflow_dispatch: - -concurrency: - group: state-corp-backend-customer-main-${{ github.ref }} - cancel-in-progress: true - -env: - CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru - CUSTOMER_REGISTRY_NAMESPACE: avm - CUSTOMER_WEB_IMAGE: state-corp-backend-web - CUSTOMER_CELERY_IMAGE: state-corp-backend-celery - CUSTOMER_DEPLOY_USER: ecos - CUSTOMER_DEPLOY_HOST: 10.0.10.174 - CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121 - CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml - CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh - -jobs: - deploy: - name: Build, Push, Deploy - runs-on: [backend-docker] - timeout-minutes: 90 - - steps: - - name: Ensure main branch - run: | - set -euo pipefail - if [ "${GITHUB_REF_NAME}" != "main" ]; then - echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2 - exit 1 - fi - - - name: Checkout code - run: | - set -euo pipefail - REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . - git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}" - - - name: Validate deploy inputs - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} - CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - home_dir="${HOME:-/root}" - - if [ -z "${registry_user}" ]; then - echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2 - exit 1 - fi - if [ -z "${registry_password}" ]; then - echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2 - exit 1 - fi - if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \ - && [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \ - && [ -z "${DEPLOY_SSH_KEY:-}" ] \ - && [ ! -f "${home_dir}/.ssh/ci-key" ] \ - && [ ! -f "/root/.ssh/ci-key" ]; then - echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2 - exit 1 - fi - - - name: Setup Docker Buildx - run: | - set -euo pipefail - if ! docker buildx inspect customer-builder >/dev/null 2>&1; then - docker buildx create --name customer-builder --driver docker-container --use - else - docker buildx use customer-builder - fi - docker buildx inspect --bootstrap - - - name: Build and push customer images - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)" - registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}" - web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}" - celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}" - - printf '%s' "${registry_password}" \ - | docker login "${CUSTOMER_REGISTRY_HOST}" \ - -u "${registry_user}" \ - --password-stdin - - docker buildx build \ - --platform linux/amd64 \ - -f ./docker/Dockerfile \ - --target runtime-web \ - --build-arg INSTALL_DEV=false \ - --push \ - -t "${web_ref}:latest" \ - -t "${web_ref}:${sha_short}" \ - . - - docker buildx build \ - --platform linux/amd64 \ - -f ./docker/Dockerfile \ - --target runtime-celery \ - --build-arg INSTALL_DEV=false \ - --push \ - -t "${celery_ref}:latest" \ - -t "${celery_ref}:${sha_short}" \ - . - - - name: Deploy customer stack - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - GITEA_TOKEN: ${{ gitea.token }} - CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} - CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} - DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - run: | - set -euo pipefail - - registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" - case "${registry_user}" in - *[!A-Za-z0-9._@-]*) - echo "Registry user contains unsupported characters" >&2 - exit 1 - ;; - esac - - home_dir="${HOME:-/root}" - mkdir -p "${home_dir}/.ssh" - key_path="${home_dir}/.ssh/customer_deploy_key" - if [ -f "/root/.ssh/ci-key" ]; then - cp "/root/.ssh/ci-key" "${key_path}" - elif [ -f "${home_dir}/.ssh/ci-key" ]; then - cp "${home_dir}/.ssh/ci-key" "${key_path}" - elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then - printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" - elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then - printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}" - elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then - printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}" - else - echo "Customer deploy SSH key is unavailable" >&2 - exit 1 - fi - chmod 600 "${key_path}" - - ssh_common=( - -i "${key_path}" - -o BatchMode=yes - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o GlobalKnownHostsFile=/dev/null - -o LogLevel=ERROR - -o ConnectTimeout=15 - ) - proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}" - remote_command="set -euo pipefail - flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'" - - ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true" - printf '%s' "${registry_password}" \ - | ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \ - "docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin" - ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}" diff --git a/scripts/ci/release-service.sh b/scripts/ci/release-service.sh new file mode 100755 index 0000000..d7d740f --- /dev/null +++ b/scripts/ci/release-service.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + printf 'release-service client: %s\n' "$*" >&2 + exit 1 +} + +require_safe_value() { + local name="$1" + local value="$2" + local pattern="$3" + + [[ -n "${value}" ]] || fail "${name} is required" + [[ "${value}" =~ ${pattern} ]] || fail "${name} contains unsupported characters" +} + +decode_required_file() { + local output_path="$1" + local encoded_value="$2" + local description="$3" + + [[ -n "${encoded_value}" ]] || fail "${description} base64 secret is required" + printf '%s' "${encoded_value}" | base64 -d >"${output_path}" || fail \ + "${description} base64 secret is invalid" + [[ -s "${output_path}" ]] || fail "${description} decoded to an empty file" +} + +[[ $# -ge 3 ]] || fail \ + "usage: release-service.sh [--refresh-data]" + +service_group="$1" +run_id="$2" +shift 2 + +refresh_data=false +if [[ "${!#}" == "--refresh-data" ]]; then + refresh_data=true + set -- "${@:1:$(($# - 1))}" +fi + +case "${service_group}" in + state-corp-backend) + expected_refs=2 + ;; + state-corp-frontend) + expected_refs=1 + ;; + *) + fail "unsupported service group: ${service_group}" + ;; +esac + +[[ $# -eq ${expected_refs} ]] || fail \ + "${service_group} requires exactly ${expected_refs} image reference(s)" +require_safe_value "run id" "${run_id}" '^[A-Za-z0-9._:-]+$' + +release_host="${RELEASE_HOST:-}" +release_user="${RELEASE_USER:-}" +release_proxy="${RELEASE_PROXY:-}" +release_service_path="${RELEASE_SERVICE_PATH:-}" + +require_safe_value "RELEASE_HOST" "${release_host}" '^[A-Za-z0-9._:-]+$' +require_safe_value "RELEASE_USER" "${release_user}" '^[A-Za-z0-9._-]+$' +require_safe_value \ + "RELEASE_SERVICE_PATH" "${release_service_path}" '^/[A-Za-z0-9._/-]+$' +case "${release_service_path}" in + /opt/ecos-dev/release-service) + image_ref_pattern='^[A-Za-z0-9._:/-]+:dev-[a-f0-9]{7,40}$' + [[ "${refresh_data}" == false ]] || fail "dev releases cannot refresh data" + ;; + /opt/ecos-main/release-service) + image_ref_pattern='^[A-Za-z0-9._:/-]+@sha256:[a-f0-9]{64}$' + ;; + /ecos/release-service) + image_ref_pattern='^[A-Za-z0-9._:/-]+@sha256:[a-f0-9]{64}$' + [[ "${refresh_data}" == false ]] || fail "customer releases cannot refresh data" + ;; + *) + fail "unsupported RELEASE_SERVICE_PATH" + ;; +esac +if [[ "${refresh_data}" == true && "${service_group}" != state-corp-backend ]]; then + fail "only an internal-main backend release can refresh data" +fi +image_refs=("$@") +for image_ref in "${image_refs[@]}"; do + [[ "${image_ref}" =~ ${image_ref_pattern} ]] || fail \ + "image reference does not match the selected release target contract" +done +if [[ -n "${release_proxy}" ]]; then + require_safe_value "RELEASE_PROXY" "${release_proxy}" '^[A-Za-z0-9._@:-]+$' +fi + +ssh_tmp_dir="$(mktemp -d)" +trap 'rm -rf "${ssh_tmp_dir}"' EXIT +key_path="${ssh_tmp_dir}/deploy-key" +known_hosts_path="${ssh_tmp_dir}/known-hosts" + +decode_required_file \ + "${key_path}" \ + "${RELEASE_SSH_KEY_B64:-}" \ + "deploy SSH key" +decode_required_file \ + "${known_hosts_path}" \ + "${RELEASE_KNOWN_HOSTS_B64:-}" \ + "pinned SSH known_hosts" +chmod 0600 "${key_path}" "${known_hosts_path}" + +ssh_options=( + -i "${key_path}" + -o BatchMode=yes + -o ConnectTimeout=15 + -o GlobalKnownHostsFile=/dev/null + -o IdentitiesOnly=yes + -o LogLevel=ERROR + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=${known_hosts_path}" +) + +if [[ -n "${release_proxy}" ]]; then + proxy_command="ssh -i ${key_path} -o BatchMode=yes -o ConnectTimeout=15" + proxy_command+=" -o GlobalKnownHostsFile=/dev/null -o IdentitiesOnly=yes" + proxy_command+=" -o LogLevel=ERROR -o StrictHostKeyChecking=yes" + proxy_command+=" -o UserKnownHostsFile=${known_hosts_path}" + proxy_command+=" -W %h:%p ${release_proxy}" + ssh_options+=(-o "ProxyCommand=${proxy_command}") +fi + +remote_args=("${service_group}" "${run_id}" "${image_refs[@]}") +if [[ "${refresh_data}" == true ]]; then + remote_args+=(--refresh-data) +fi +printf -v remote_command '%q ' "${release_service_path}" "${remote_args[@]}" + +# The command is deliberately assembled and validated on the client. +# shellcheck disable=SC2029 +ssh "${ssh_options[@]}" \ + "${release_user}@${release_host}" \ + "${remote_command}" diff --git a/scripts/ci/smoke.sh b/scripts/ci/smoke.sh new file mode 100755 index 0000000..e3b30c0 --- /dev/null +++ b/scripts/ci/smoke.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +smoke_url="${1:-}" +[[ "${smoke_url}" =~ ^https?:// ]] || { + printf 'a complete http(s) smoke URL is required\n' >&2 + exit 1 +} + +for attempt in $(seq 1 24); do + if curl --fail --silent \ + --connect-timeout 5 \ + --max-time 15 \ + --output /dev/null \ + "${smoke_url}"; then + printf 'Smoke check passed.\n' + exit 0 + fi + + printf 'Smoke attempt %s/24 failed; retrying in 5 seconds.\n' "${attempt}" >&2 + sleep 5 +done + +printf 'Smoke check failed after 24 attempts.\n' >&2 +exit 1 diff --git a/src/apps/core/management/commands/sanitize_cloned_environment.py b/src/apps/core/management/commands/sanitize_cloned_environment.py new file mode 100644 index 0000000..937420e --- /dev/null +++ b/src/apps/core/management/commands/sanitize_cloned_environment.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import json + +from apps.core.management.commands.base import BaseAppCommand +from apps.core.models import ( + BackgroundJob, + JobStatus, + ReportUpload, + ReportUploadStatus, +) +from django.utils import timezone +from django_celery_beat.models import PeriodicTask, PeriodicTasks + +CLONED_JOB_MESSAGE = "Задача остановлена при подготовке клонированного окружения" +CLONED_UPLOAD_MESSAGE = "Обработка остановлена при подготовке клонированного окружения" + + +class Command(BaseAppCommand): + help = ( + "Disable Celery Beat schedules and close in-flight job records after " + "cloning an environment." + ) + use_transaction = True + + def execute_command(self, *args, **options) -> str: + completed_at = timezone.now() + + disabled_periodic_tasks = PeriodicTask.objects.filter(enabled=True).update( + enabled=False + ) + if disabled_periodic_tasks: + PeriodicTasks.update_changed() + + revoked_background_jobs = BackgroundJob.objects.filter( + status__in=(JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY) + ).update( + status=JobStatus.REVOKED, + completed_at=completed_at, + progress_message=CLONED_JOB_MESSAGE, + ) + + failed_report_uploads = ReportUpload.objects.filter( + status__in=(ReportUploadStatus.QUEUED, ReportUploadStatus.PROCESSING) + ).update( + status=ReportUploadStatus.FAILED, + status_message=CLONED_UPLOAD_MESSAGE, + completed_at=completed_at, + ) + + result = json.dumps( + { + "disabled_periodic_tasks": disabled_periodic_tasks, + "failed_report_uploads": failed_report_uploads, + "revoked_background_jobs": revoked_background_jobs, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_info(result) + return result diff --git a/src/apps/core/management/commands/validate_cloned_migrations.py b/src/apps/core/management/commands/validate_cloned_migrations.py new file mode 100644 index 0000000..aae0091 --- /dev/null +++ b/src/apps/core/management/commands/validate_cloned_migrations.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import json + +from apps.core.management.commands.base import BaseAppCommand +from django.core.management.base import CommandError +from django.db import connection +from django.db.migrations.loader import MigrationLoader +from django.db.migrations.recorder import MigrationRecorder + + +class Command(BaseAppCommand): + help = ( + "Reject a cloned database when it records an applied migration that is " + "absent from the current image." + ) + requires_migrations_checks = False + requires_system_checks: list[str] = [] + + def execute_command(self, *args, **options) -> str: + applied_migrations = set(MigrationRecorder(connection).applied_migrations()) + disk_migrations = set( + MigrationLoader(connection, ignore_no_migrations=True).disk_migrations + ) + unknown_migrations = sorted(applied_migrations - disk_migrations) + + if unknown_migrations: + formatted = ", ".join( + f"{app_label}.{migration_name}" + for app_label, migration_name in unknown_migrations + ) + raise CommandError( + "Cloned database contains applied migrations absent from this " + f"image: {formatted}" + ) + + result = json.dumps( + { + "applied_migrations": len(applied_migrations), + "disk_migrations": len(disk_migrations), + "unknown_applied_migrations": 0, + }, + sort_keys=True, + ) + self.log_info(result) + return result diff --git a/tests/apps/core/test_sanitize_cloned_environment_command.py b/tests/apps/core/test_sanitize_cloned_environment_command.py new file mode 100644 index 0000000..b82b745 --- /dev/null +++ b/tests/apps/core/test_sanitize_cloned_environment_command.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import json +from io import StringIO + +from apps.core.models import ( + BackgroundJob, + JobStatus, + ReportUpload, + ReportUploadStatus, +) +from django.core.management import call_command +from django.test import TestCase +from django.utils import timezone +from django_celery_beat.models import IntervalSchedule, PeriodicTask + +from tests.apps.user.factories import UserFactory + + +class SanitizeClonedEnvironmentCommandTest(TestCase): + def setUp(self) -> None: + PeriodicTask.objects.all().delete() + interval = IntervalSchedule.objects.create( + every=1, + period=IntervalSchedule.HOURS, + ) + self.enabled_schedule = PeriodicTask.objects.create( + name="enabled-schedule", + task="apps.core.tasks.enabled", + interval=interval, + enabled=True, + ) + self.disabled_schedule = PeriodicTask.objects.create( + name="disabled-schedule", + task="apps.core.tasks.disabled", + interval=interval, + enabled=False, + ) + + self.user = UserFactory.create_user() + self.active_jobs = [ + BackgroundJob.objects.create( + task_id=f"active-{status}", + task_name="apps.core.tasks.active", + status=status, + user_id=self.user.id, + ) + for status in (JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY) + ] + self.finished_job = BackgroundJob.objects.create( + task_id="finished-success", + task_name="apps.core.tasks.finished", + status=JobStatus.SUCCESS, + progress=100, + result={"records": 42}, + user_id=self.user.id, + completed_at=timezone.now(), + ) + + self.active_uploads = [ + self._create_upload(status=status, suffix=str(index)) + for index, status in enumerate( + (ReportUploadStatus.QUEUED, ReportUploadStatus.PROCESSING), + start=1, + ) + ] + self.finished_upload = self._create_upload( + status=ReportUploadStatus.SUCCESS, + suffix="finished", + load_batch=42, + ) + + def _create_upload( + self, + *, + status: str, + suffix: str, + load_batch: int | None = None, + ) -> ReportUpload: + return ReportUpload.objects.create( + form="f1", + load_batch=load_batch, + original_file=f"report_uploads/f1/{suffix}/report.xlsx", + file_name=f"report-{suffix}.xlsx", + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + file_size=128, + file_hash=suffix.ljust(64, "0")[:64], + status=status, + uploaded_by=self.user, + completed_at=( + timezone.now() + if status + in { + ReportUploadStatus.SUCCESS, + ReportUploadStatus.PARTIAL, + ReportUploadStatus.FAILED, + } + else None + ), + ) + + def test_disables_schedules_and_closes_only_in_flight_records(self) -> None: + result = json.loads( + call_command( + "sanitize_cloned_environment", + stdout=StringIO(), + ) + ) + + self.assertEqual( + result, + { + "disabled_periodic_tasks": 1, + "failed_report_uploads": 2, + "revoked_background_jobs": 3, + }, + ) + self.enabled_schedule.refresh_from_db() + self.disabled_schedule.refresh_from_db() + self.assertFalse(self.enabled_schedule.enabled) + self.assertFalse(self.disabled_schedule.enabled) + + for job in self.active_jobs: + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.REVOKED) + self.assertIsNotNone(job.completed_at) + + for upload in self.active_uploads: + upload.refresh_from_db() + self.assertEqual(upload.status, ReportUploadStatus.FAILED) + self.assertIsNotNone(upload.completed_at) + + self.finished_job.refresh_from_db() + self.assertEqual(self.finished_job.status, JobStatus.SUCCESS) + self.assertEqual(self.finished_job.result, {"records": 42}) + self.finished_upload.refresh_from_db() + self.assertEqual(self.finished_upload.status, ReportUploadStatus.SUCCESS) + self.assertEqual(self.finished_upload.load_batch, 42) + self.assertEqual(self.finished_upload.uploaded_by, self.user) + self.assertTrue(type(self.user).objects.filter(pk=self.user.pk).exists()) + + def test_is_idempotent(self) -> None: + call_command("sanitize_cloned_environment", stdout=StringIO()) + first_completed_at = { + job.pk: BackgroundJob.objects.get(pk=job.pk).completed_at + for job in self.active_jobs + } + first_upload_completed_at = { + upload.pk: ReportUpload.objects.get(pk=upload.pk).completed_at + for upload in self.active_uploads + } + + result = json.loads( + call_command( + "sanitize_cloned_environment", + stdout=StringIO(), + ) + ) + + self.assertEqual( + result, + { + "disabled_periodic_tasks": 0, + "failed_report_uploads": 0, + "revoked_background_jobs": 0, + }, + ) + self.assertEqual( + first_completed_at, + { + job.pk: BackgroundJob.objects.get(pk=job.pk).completed_at + for job in self.active_jobs + }, + ) + self.assertEqual( + first_upload_completed_at, + { + upload.pk: ReportUpload.objects.get(pk=upload.pk).completed_at + for upload in self.active_uploads + }, + ) + + def test_dry_run_rolls_back_all_changes(self) -> None: + result = json.loads( + call_command( + "sanitize_cloned_environment", + dry_run=True, + stdout=StringIO(), + ) + ) + + self.assertEqual(result["disabled_periodic_tasks"], 1) + self.enabled_schedule.refresh_from_db() + self.assertTrue(self.enabled_schedule.enabled) + self.assertEqual( + set( + BackgroundJob.objects.filter( + pk__in=[job.pk for job in self.active_jobs] + ).values_list("status", flat=True) + ), + {JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY}, + ) + self.assertEqual( + set( + ReportUpload.objects.filter( + pk__in=[upload.pk for upload in self.active_uploads] + ).values_list("status", flat=True) + ), + {ReportUploadStatus.QUEUED, ReportUploadStatus.PROCESSING}, + ) diff --git a/tests/apps/core/test_validate_cloned_migrations_command.py b/tests/apps/core/test_validate_cloned_migrations_command.py new file mode 100644 index 0000000..3c49995 --- /dev/null +++ b/tests/apps/core/test_validate_cloned_migrations_command.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import json +from io import StringIO + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.db import connection +from django.db.migrations.recorder import MigrationRecorder +from django.test import TestCase + + +class ValidateClonedMigrationsCommandTest(TestCase): + def test_accepts_database_when_all_applied_migrations_exist_in_image(self) -> None: + result = json.loads( + call_command( + "validate_cloned_migrations", + stdout=StringIO(), + ) + ) + + self.assertEqual(result["unknown_applied_migrations"], 0) + self.assertGreaterEqual( + result["disk_migrations"], + result["applied_migrations"], + ) + + def test_rejects_applied_migration_missing_from_image(self) -> None: + MigrationRecorder(connection).record_applied( + "removed_legacy_app", + "0001_removed", + ) + + with self.assertRaisesMessage( + CommandError, + "removed_legacy_app.0001_removed", + ): + call_command( + "validate_cloned_migrations", + stdout=StringIO(), + )