diff --git a/.env.prod.example b/.env.prod.example index fd06fa3..4650201 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -1,36 +1,46 @@ -# Docker Compose production example. -# Copy to .env.prod and replace CHANGE_ME values. -DJANGO_SETTINGS_MODULE=config.settings.production -DEBUG=False -SECRET_KEY=CHANGE_ME_PROD_SECRET_KEY -ALLOWED_HOSTS=example.com,api.example.com +# Docker Compose local network defaults. +# These values match docker/Dockerfile runtime defaults. +DJANGO_SETTINGS_MODULE=settings.dev +DEBUG=True +SECRET_KEY=django-insecure-development-key-mostovik-2024 +ALLOWED_HOSTS=* -POSTGRES_HOST=CHANGE_ME_POSTGRES_HOST +POSTGRES_HOST=10.10.0.114 POSTGRES_PORT=5432 POSTGRES_DB=mostovik POSTGRES_USER=postgres -POSTGRES_PASSWORD=CHANGE_ME_POSTGRES_PASSWORD -POSTGRES_SSLMODE=require +POSTGRES_PASSWORD=postgres +POSTGRES_SSLMODE=disable -REDIS_CACHE_URL=redis://CHANGE_ME_REDIS_HOST:6379/1 -CELERY_BROKER_URL=redis://CHANGE_ME_REDIS_HOST:6379/0 -CELERY_RESULT_BACKEND=redis://CHANGE_ME_REDIS_HOST:6379/0 +REDIS_HOST=10.10.0.110 +REDIS_CACHE_URL=redis://10.10.0.110:6379/1 +CELERY_BROKER_URL=redis://10.10.0.110:6379/0 +CELERY_RESULT_BACKEND=redis://10.10.0.110:6379/0 PORT=8000 -GUNICORN_WORKERS=3 +GUNICORN_WORKERS=4 GUNICORN_TIMEOUT=60 CELERY_LOG_LEVEL=INFO -CELERY_WORKER_CONCURRENCY=4 +CELERY_WORKER_CONCURRENCY=2 # Parsers API keys -CHECKO_API_KEY=CHANGE_ME_CHECKO_API_KEY -ZAKUPKI_TOKEN=CHANGE_ME_ZAKUPKI_TOKEN +CHECKO_API_KEY=pRiEnJuD1tclsLCb +ZAKUPKI_TOKEN=019c03d7-e1f6-7091-b296-8c88b4c585dd # Optional: comma-separated HTTP(S) proxies for parser tasks # Example: PARSER_PROXIES=http://user:pass@proxy1:8080,http://user:pass@proxy2:8080 PARSER_PROXIES= # 1 to collect static files during migrate service, 0 to skip -COLLECTSTATIC_ON_MIGRATE=1 +COLLECTSTATIC_ON_MIGRATE=0 -WEB_IMAGE=registry.example.com/mostovik/web:latest -CELERY_IMAGE=registry.example.com/mostovik/celery:latest +BACKUP_ENCRYPTION_KEY=a2tra2tra2tra2tra2tra2tra2tra2tra2tra2s +BACKUP_KEY_ID=default +BACKUP_EXPORT_DIRECTORY=/app/media/backups + +STATE_CORP_EXCHANGE_URL= +STATE_CORP_EXCHANGE_TOKEN= +STATE_CORP_EXCHANGE_KEY_ID=state-corp-shared-token +STATE_CORP_EXCHANGE_TIMEOUT_SECONDS=60 + +WEB_IMAGE=10.10.0.50/avm/mostovik-backend-web:dev +CELERY_IMAGE=10.10.0.50/avm/mostovik-backend-celery:dev diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 0e8ee0b..2de4369 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -5,27 +5,44 @@ on: branches: - main - dev + - "feature/**" + - "codex/**" pull_request: branches: - main - dev + workflow_dispatch: + inputs: + dokploy_target: + description: "Dokploy dev target: all, web, or celery" + required: true + default: "all" + +concurrency: + group: mostovik-backend-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true env: PYTHON_VERSION: "3.11" + REGISTRY_HOST: "10.10.0.50" + REGISTRY_NAMESPACE: "${{ github.repository_owner }}" + WEB_IMAGE: "mostovik-backend-web" + CELERY_IMAGE: "mostovik-backend-celery" + CRANE_VERSION: "v0.19.0" + UV_VERSION: "0.7.2" + PIP_DISABLE_PIP_VERSION_CHECK: "1" jobs: - lint: - name: Code Quality Checks + quality: + name: Quality Gate runs-on: ubuntu-latest - timeout-minutes: 15 - if: ${{ !contains(github.event.head_commit.message, '#no_lint') }} - env: - TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} - TG_CHANNEL: ${{ secrets.TG_CHANNEL }} + timeout-minutes: 25 + if: ${{ github.event_name != 'workflow_dispatch' }} steps: - name: Checkout code run: | + set -euo pipefail REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . @@ -45,7 +62,7 @@ jobs: . ./.ci-python-env "${PYTHON_BIN}" -m venv .venv . .venv/bin/activate - python -m pip install --upgrade pip uv + python -m pip install "uv==${UV_VERSION}" uv sync \ --dev \ --frozen \ @@ -55,87 +72,21 @@ jobs: --no-python-downloads - name: Run Ruff linting + if: ${{ !contains(github.event.head_commit.message, '#no_lint') }} run: | set -euo pipefail . .venv/bin/activate ruff check src - name: Run Ruff formatting check + if: ${{ !contains(github.event.head_commit.message, '#no_lint') }} run: | set -euo pipefail . .venv/bin/activate ruff format src --check - - name: Telegram notify (lint failed) - if: failure() - continue-on-error: true - run: | - set -euo pipefail - if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then - echo "TG_BOT_KEY or TG_CHANNEL is not set; skip telegram notification" - exit 0 - fi - - COMMIT_MESSAGE=$(git log -1 --pretty=%s 2>/dev/null || echo "n/a") - - MSG="❌ [mostovik-backend] lint failed - branch=${GITHUB_REF_NAME} - sha=${GITHUB_SHA} - actor=${GITHUB_ACTOR} - commit=${COMMIT_MESSAGE}" - - curl -fsS \ - --connect-timeout 5 \ - --max-time 15 \ - --retry 2 \ - --retry-delay 2 \ - --retry-all-errors \ - -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ - -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" \ - || echo "Telegram notification failed; continue pipeline" - - test: - name: Run Tests - runs-on: ubuntu-latest - timeout-minutes: 20 - if: ${{ !contains(github.event.head_commit.message, '#no_test') }} - env: - TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} - TG_CHANNEL: ${{ secrets.TG_CHANNEL }} - - steps: - - name: Checkout code - run: | - REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") - BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" - git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . - git checkout "${GITHUB_SHA}" - - - name: Install Python and uv - run: | - set -euo pipefail - PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" - PYTHON_BIN="$(./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}")" - - printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env - - - name: Create virtual environment and install dependencies - run: | - set -euo pipefail - . ./.ci-python-env - "${PYTHON_BIN}" -m venv .venv - . .venv/bin/activate - python -m pip install --upgrade pip uv - uv sync \ - --dev \ - --frozen \ - --active \ - --python "${PYTHON_BIN}" \ - --no-managed-python \ - --no-python-downloads - - name: Run regular pytest suite + if: ${{ !contains(github.event.head_commit.message, '#no_test') }} env: DJANGO_SETTINGS_MODULE: settings.test SECRET_KEY: test-secret-key-for-ci @@ -144,90 +95,8 @@ jobs: export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" .venv/bin/python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q - - name: Pack prepared test workspace - if: success() - run: | - set -euo pipefail - WORKSPACE_ARCHIVE="/tmp/ci-test-workspace.tar.gz" - tar \ - --exclude='.git' \ - --exclude='.pytest_cache' \ - --exclude='htmlcov' \ - --exclude='__pycache__' \ - -czf "${WORKSPACE_ARCHIVE}" \ - . - - - name: Upload prepared test workspace - if: success() - uses: actions/upload-artifact@v3 - with: - name: ci-test-workspace - path: /tmp/ci-test-workspace.tar.gz - if-no-files-found: error - retention-days: 1 - - - name: Telegram notify (test failed) - if: failure() - continue-on-error: true - run: | - set -euo pipefail - if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then - echo "TG_BOT_KEY or TG_CHANNEL is not set; skip telegram notification" - exit 0 - fi - - COMMIT_MESSAGE=$(git log -1 --pretty=%s 2>/dev/null || echo "n/a") - - MSG="❌ [mostovik-backend] test failed - branch=${GITHUB_REF_NAME} - sha=${GITHUB_SHA} - actor=${GITHUB_ACTOR} - commit=${COMMIT_MESSAGE}" - - curl -fsS \ - --connect-timeout 5 \ - --max-time 15 \ - --retry 2 \ - --retry-delay 2 \ - --retry-all-errors \ - -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ - -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" \ - || echo "Telegram notification failed; continue pipeline" - - test_api_inventory_e2e: - name: Run API Inventory E2E Tests - runs-on: ubuntu-latest - timeout-minutes: 10 - needs: [test] - if: ${{ needs.test.result == 'success' }} - env: - TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} - TG_CHANNEL: ${{ secrets.TG_CHANNEL }} - - steps: - - name: Download prepared test workspace - uses: actions/download-artifact@v3 - with: - name: ci-test-workspace - - - name: Extract prepared test workspace - run: | - set -euo pipefail - ARCHIVE_PATH="$(find . -maxdepth 2 -name 'ci-test-workspace.tar.gz' -print -quit)" - if [ -z "${ARCHIVE_PATH}" ]; then - echo "ci-test-workspace.tar.gz not found after artifact download" >&2 - exit 1 - fi - tar -xzf "${ARCHIVE_PATH}" - - - name: Install Python for artifact environment - run: | - set -euo pipefail - PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" - ./scripts/ensure-ci-python.sh "${PROJECT_PYTHON_VERSION}" >/dev/null - - name: Run API inventory pytest suite + if: ${{ !contains(github.event.head_commit.message, '#no_test') }} env: DJANGO_SETTINGS_MODULE: settings.test SECRET_KEY: test-secret-key-for-ci @@ -236,70 +105,268 @@ jobs: export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" .venv/bin/python -m pytest tests/test_api_inventory_e2e.py -q - - name: Telegram notify (api inventory e2e failed) - if: failure() - continue-on-error: true + build_push: + name: Build and Push Images + runs-on: ubuntu-latest + timeout-minutes: 45 + needs: [quality] + if: | + github.event_name == 'push' && + (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') && + needs.quality.result == 'success' && + !contains(github.event.head_commit.message, '#no_image') + + steps: + - name: Checkout code run: | set -euo pipefail - if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then - echo "TG_BOT_KEY or TG_CHANNEL is not set; skip telegram notification" - exit 0 + REPO_URL=$(echo "${GITHUB_SERVER_URL}" | sed "s|://|://oauth2:${{ gitea.token }}@|") + BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + git clone --depth=1 --branch="${BRANCH}" "${REPO_URL}/${GITHUB_REPOSITORY}.git" . + git checkout "${GITHUB_SHA}" + + - name: Build and push branch images + env: + GITEA_TOKEN: ${{ gitea.token }} + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -euo pipefail + + curl -fsSL \ + "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" \ + | tar xz crane + chmod +x crane + + BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-branch}}" + BRANCH_TAG=$(printf '%s' "${BRANCH}" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's#[/[:space:]]+#-#g; s#[^a-z0-9_.-]+#-#g; s#^-+##; s#-+$##') + BRANCH_TAG="${BRANCH_TAG:-branch}" + BRANCH_TAG=$(printf '%.120s' "${BRANCH_TAG}") + SHA_SHORT=$(printf '%s' "${GITHUB_SHA}" | cut -c1-7) + + REGISTRY_PATH="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}" + WEB_REF="${REGISTRY_PATH}/${WEB_IMAGE}" + CELERY_REF="${REGISTRY_PATH}/${CELERY_IMAGE}" + REGISTRY_USER="${REGISTRY_USER:-${GITHUB_ACTOR}}" + REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}" + + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY + export NO_PROXY="${NO_PROXY:-},${REGISTRY_HOST}" + export no_proxy="${no_proxy:-},${REGISTRY_HOST}" + + if [ -z "${REGISTRY_PASSWORD}" ]; then + echo "REGISTRY_TOKEN secret is not set and gitea.token fallback is empty" >&2 + exit 1 fi - MSG="❌ [mostovik-backend] api inventory e2e failed - branch=${GITHUB_REF_NAME} - sha=${GITHUB_SHA} - actor=${GITHUB_ACTOR}" + echo "${REGISTRY_PASSWORD}" \ + | ./crane auth login --insecure "${REGISTRY_HOST}" \ + -u "${REGISTRY_USER}" \ + --password-stdin - curl -fsS \ - --connect-timeout 5 \ - --max-time 15 \ - --retry 2 \ - --retry-delay 2 \ - --retry-all-errors \ - -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ - -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" \ - || echo "Telegram notification failed; continue pipeline" + docker build \ + -f ./docker/Dockerfile \ + --target runtime-web \ + --build-arg INSTALL_DEV=false \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + -t "${WEB_IMAGE}:local" . + docker save "${WEB_IMAGE}:local" -o /tmp/web.tar - notify_success: - name: Telegram Notify Success + ./crane push --insecure /tmp/web.tar "${WEB_REF}:${BRANCH_TAG}" + ./crane push --insecure /tmp/web.tar "${WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" + if [ "${GITHUB_REF_NAME}" = "main" ]; then + ./crane push --insecure /tmp/web.tar "${WEB_REF}:latest" + fi + + docker build \ + -f ./docker/Dockerfile \ + --target runtime-celery \ + --build-arg INSTALL_DEV=false \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + -t "${CELERY_IMAGE}:local" . + docker save "${CELERY_IMAGE}:local" -o /tmp/celery.tar + + ./crane push --insecure /tmp/celery.tar "${CELERY_REF}:${BRANCH_TAG}" + ./crane push --insecure /tmp/celery.tar "${CELERY_REF}:${BRANCH_TAG}-${SHA_SHORT}" + if [ "${GITHUB_REF_NAME}" = "main" ]; then + ./crane push --insecure /tmp/celery.tar "${CELERY_REF}:latest" + fi + + { + echo "Pushed images:" + echo "- ${WEB_REF}:${BRANCH_TAG}" + echo "- ${WEB_REF}:${BRANCH_TAG}-${SHA_SHORT}" + echo "- ${CELERY_REF}:${BRANCH_TAG}" + echo "- ${CELERY_REF}:${BRANCH_TAG}-${SHA_SHORT}" + } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + + notify: + name: Internal Notify runs-on: ubuntu-latest timeout-minutes: 1 - needs: [lint, test, test_api_inventory_e2e] - if: | - always() && - needs.lint.result == 'success' && - needs.test.result == 'success' && - needs.test_api_inventory_e2e.result == 'success' - env: - TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} - TG_CHANNEL: ${{ secrets.TG_CHANNEL }} + needs: [quality, build_push] + if: ${{ always() && github.event_name != 'workflow_dispatch' }} + steps: - - name: Telegram notify (lint+tests+e2e success) + - name: Send CI status webhook continue-on-error: true env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + CI_NOTIFY_WEBHOOK_URL: ${{ secrets.CI_NOTIFY_WEBHOOK_URL }} + CI_NOTIFY_TOKEN: ${{ secrets.CI_NOTIFY_TOKEN }} + QUALITY_RESULT: ${{ needs.quality.result }} + BUILD_PUSH_RESULT: ${{ needs.build_push.result }} run: | set -euo pipefail - if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then - echo "TG_BOT_KEY or TG_CHANNEL is not set; skip telegram notification" + if [ -z "${CI_NOTIFY_WEBHOOK_URL:-}" ]; then + echo "CI_NOTIFY_WEBHOOK_URL is not set; skip internal notification" exit 0 fi - MSG="✅ [mostovik-backend] lint + tests + api inventory e2e passed - branch=${GITHUB_REF_NAME} - sha=${GITHUB_SHA} - actor=${GITHUB_ACTOR} - commit=${COMMIT_MESSAGE:-n/a}" + STATUS="success" + if [ "${QUALITY_RESULT}" != "success" ]; then + STATUS="${QUALITY_RESULT}" + elif [ "${BUILD_PUSH_RESULT}" = "failure" ] || [ "${BUILD_PUSH_RESULT}" = "cancelled" ]; then + STATUS="${BUILD_PUSH_RESULT}" + fi + export STATUS + + PAYLOAD=$(python3 - <<'PY' + import json + import os + + payload = { + "project": os.environ.get("GITHUB_REPOSITORY"), + "workflow": os.environ.get("GITHUB_WORKFLOW"), + "status": os.environ.get("STATUS"), + "branch": os.environ.get("GITHUB_HEAD_REF") or os.environ.get("GITHUB_REF_NAME"), + "sha": os.environ.get("GITHUB_SHA"), + "actor": os.environ.get("GITHUB_ACTOR"), + "server_url": os.environ.get("GITHUB_SERVER_URL"), + "run_id": os.environ.get("GITHUB_RUN_ID"), + "results": { + "quality": os.environ.get("QUALITY_RESULT"), + "build_push": os.environ.get("BUILD_PUSH_RESULT"), + }, + } + print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) + PY + ) + + AUTH_HEADER=() + if [ -n "${CI_NOTIFY_TOKEN:-}" ]; then + AUTH_HEADER=(-H "Authorization: Bearer ${CI_NOTIFY_TOKEN}") + fi curl -fsS \ - --connect-timeout 5 \ - --max-time 15 \ - --retry 2 \ - --retry-delay 2 \ - --retry-all-errors \ - -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ - -d "chat_id=${TG_CHANNEL}" \ - --data-urlencode "text=${MSG}" \ - || echo "Telegram notification failed; continue pipeline" + --connect-timeout 3 \ + --max-time 8 \ + --retry 1 \ + -H "Content-Type: application/json" \ + "${AUTH_HEADER[@]}" \ + --data "${PAYLOAD}" \ + "${CI_NOTIFY_WEBHOOK_URL}" + + dokploy_dev_start: + name: Start Dev Containers in Dokploy + runs-on: ubuntu-latest + timeout-minutes: 5 + if: ${{ github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/dev' }} + + steps: + - name: Trigger Dokploy webhooks + env: + DOKPLOY_TARGET: ${{ github.event.inputs.dokploy_target }} + DOKPLOY_DEV_WEBHOOK_URL: ${{ secrets.DOKPLOY_DEV_WEBHOOK_URL }} + DOKPLOY_DEV_WEB_WEBHOOK_URL: ${{ secrets.DOKPLOY_DEV_WEB_WEBHOOK_URL }} + DOKPLOY_DEV_CELERY_WEBHOOK_URL: ${{ secrets.DOKPLOY_DEV_CELERY_WEBHOOK_URL }} + DOKPLOY_API_TOKEN: ${{ secrets.DOKPLOY_API_TOKEN }} + run: | + set -euo pipefail + + TARGET="${DOKPLOY_TARGET:-all}" + case "${TARGET}" in + all|web|celery) ;; + *) + echo "dokploy_target must be one of: all, web, celery" >&2 + exit 1 + ;; + esac + + call_webhook() { + service_name="$1" + webhook_url="$2" + + if [ -z "${webhook_url}" ]; then + echo "Dokploy webhook for ${service_name} is not configured" >&2 + return 1 + fi + + AUTH_HEADER=() + if [ -n "${DOKPLOY_API_TOKEN:-}" ]; then + AUTH_HEADER=(-H "Authorization: Bearer ${DOKPLOY_API_TOKEN}") + fi + + PAYLOAD=$(python3 - <<'PY' + import json + import os + + payload = { + "project": os.environ.get("GITHUB_REPOSITORY"), + "branch": os.environ.get("GITHUB_REF_NAME"), + "sha": os.environ.get("GITHUB_SHA"), + "actor": os.environ.get("GITHUB_ACTOR"), + "target": os.environ.get("CURRENT_DOKPLOY_TARGET"), + "images": { + "web": "10.10.0.50/avm/mostovik-backend-web:dev", + "celery": "10.10.0.50/avm/mostovik-backend-celery:dev", + }, + } + print(json.dumps(payload, ensure_ascii=True, separators=(",", ":"))) + PY + ) + + echo "Trigger Dokploy for ${service_name}" + curl -fsS \ + --connect-timeout 5 \ + --max-time 30 \ + --retry 2 \ + --retry-delay 2 \ + -X POST \ + -H "Content-Type: application/json" \ + "${AUTH_HEADER[@]}" \ + --data "${PAYLOAD}" \ + "${webhook_url}" + } + + triggered=0 + + if [ "${TARGET}" = "all" ] && [ -n "${DOKPLOY_DEV_WEBHOOK_URL:-}" ]; then + CURRENT_DOKPLOY_TARGET="all" call_webhook "dev stack" "${DOKPLOY_DEV_WEBHOOK_URL}" + triggered=1 + else + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "web" ]; then + CURRENT_DOKPLOY_TARGET="web" call_webhook "dev web" "${DOKPLOY_DEV_WEB_WEBHOOK_URL:-${DOKPLOY_DEV_WEBHOOK_URL:-}}" + triggered=1 + fi + + if [ "${TARGET}" = "all" ] || [ "${TARGET}" = "celery" ]; then + CURRENT_DOKPLOY_TARGET="celery" call_webhook "dev celery" "${DOKPLOY_DEV_CELERY_WEBHOOK_URL:-${DOKPLOY_DEV_WEBHOOK_URL:-}}" + triggered=1 + fi + fi + + if [ "${triggered}" -ne 1 ]; then + echo "No Dokploy webhook was triggered" >&2 + exit 1 + fi + + { + echo "Dokploy dev trigger completed." + echo "Target: ${TARGET}" + echo "Web image: 10.10.0.50/avm/mostovik-backend-web:dev" + echo "Celery image: 10.10.0.50/avm/mostovik-backend-celery:dev" + } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" diff --git a/.gitea/workflows/dev-db-maintenance.yml b/.gitea/workflows/dev-db-maintenance.yml new file mode 100644 index 0000000..4426962 --- /dev/null +++ b/.gitea/workflows/dev-db-maintenance.yml @@ -0,0 +1,77 @@ +name: Dev Database Maintenance + +on: + workflow_dispatch: + inputs: + confirm: + description: "Type CLEAN_DEV_DB to drop and recreate the dev public schema" + required: true + default: "" + +env: + POSTGRES_HOST: "10.10.0.114" + POSTGRES_PORT: "5432" + POSTGRES_DB: "mostovik" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "postgres" + +jobs: + cleanup_dev_database: + name: Cleanup Dev Database + runs-on: ubuntu-latest + timeout-minutes: 10 + if: ${{ github.ref == 'refs/heads/dev' }} + + steps: + - name: Validate confirmation + env: + CONFIRM: ${{ github.event.inputs.confirm }} + run: | + set -euo pipefail + if [ "${CONFIRM}" != "CLEAN_DEV_DB" ]; then + echo "Manual confirmation must be exactly CLEAN_DEV_DB" >&2 + exit 1 + fi + + - name: Install PostgreSQL client + run: | + set -euo pipefail + APT_RUNNER=() + if [ "$(id -u)" -ne 0 ]; then + APT_RUNNER=(sudo) + fi + + export DEBIAN_FRONTEND=noninteractive + "${APT_RUNNER[@]}" apt-get update + "${APT_RUNNER[@]}" apt-get install -y postgresql-client + + - name: Drop and recreate public schema + run: | + set -euo pipefail + export PGPASSWORD="${POSTGRES_PASSWORD}" + + psql \ + --set ON_ERROR_STOP=1 \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --username="${POSTGRES_USER}" \ + --dbname="${POSTGRES_DB}" \ + <<'SQL' + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = current_database() + AND pid <> pg_backend_pid(); + + DROP SCHEMA IF EXISTS public CASCADE; + CREATE SCHEMA public; + GRANT ALL ON SCHEMA public TO postgres; + GRANT ALL ON SCHEMA public TO public; + SQL + + - name: Summary + run: | + set -euo pipefail + { + echo "Dev database cleanup completed." + echo "Database: ${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" + } >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 02df776..1fda102 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,7 +26,7 @@ services: web: build: *web-build - image: http://10.10.0.10:3000/v2/avm/mostovik-backend:dev + image: ${WEB_IMAGE:-mostovik/web:latest} container_name: mostovik_web restart: unless-stopped env_file: @@ -49,7 +49,7 @@ services: container_name: mostovik_celery_worker restart: unless-stopped environment: - CELERY_WORKER_CONCURRENCY: "1" + CELERY_WORKER_CONCURRENCY: "2" CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "3145728" env_file: - .env.prod diff --git a/docker/Dockerfile b/docker/Dockerfile index 129f98e..6dd4b20 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -63,7 +63,33 @@ RUN mkdir -p logs media staticfiles input/fns input/fns/processed input/fns/fail && chown -R appuser:appgroup /app ENV PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH=/app/src + PYTHONPATH=/app/src \ + DJANGO_SETTINGS_MODULE=settings.dev \ + POSTGRES_HOST=10.10.0.114 \ + POSTGRES_PORT=5432 \ + POSTGRES_DB=mostovik \ + POSTGRES_USER=postgres \ + POSTGRES_PASSWORD=postgres \ + POSTGRES_SSLMODE=disable \ + REDIS_HOST=10.10.0.110 \ + REDIS_CACHE_URL=redis://10.10.0.110:6379/1 \ + CELERY_BROKER_URL=redis://10.10.0.110:6379/0 \ + CELERY_RESULT_BACKEND=redis://10.10.0.110:6379/0 \ + PORT=8000 \ + GUNICORN_WORKERS=4 \ + GUNICORN_TIMEOUT=60 \ + CELERY_LOG_LEVEL=INFO \ + CELERY_WORKER_CONCURRENCY=2 \ + CHECKO_API_KEY=pRiEnJuD1tclsLCb \ + ZAKUPKI_TOKEN=019c03d7-e1f6-7091-b296-8c88b4c585dd \ + COLLECTSTATIC_ON_MIGRATE=0 \ + BACKUP_ENCRYPTION_KEY=a2tra2tra2tra2tra2tra2tra2tra2tra2tra2s \ + BACKUP_KEY_ID=default \ + BACKUP_EXPORT_DIRECTORY=/app/media/backups \ + STATE_CORP_EXCHANGE_URL= \ + STATE_CORP_EXCHANGE_TOKEN= \ + STATE_CORP_EXCHANGE_KEY_ID=state-corp-shared-token \ + STATE_CORP_EXCHANGE_TIMEOUT_SECONDS=60 USER appuser diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 6e245f8..2519c6e 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -170,11 +170,12 @@ class ParsersViewSetTest(APITestCase): self.assertEqual(detail.status_code, status.HTTP_200_OK) def test_system_logs_support_search_and_organizations_count(self): + search_marker = "manufactures-unique-search-marker" first_log = ParserLoadLogFactory( source="manufactures", batch_id=101, status="success", - error_message="ok", + error_message=search_marker, ) ParserLoadLogFactory( source="inspections", @@ -188,7 +189,7 @@ class ParsersViewSetTest(APITestCase): self.client.force_authenticate(self.admin) response = self.client.get( reverse("api_v1:system:parser-logs-list"), - {"search": "101"}, + {"search": search_marker}, ) self.assertEqual(response.status_code, status.HTTP_200_OK)