Compare commits
4 Commits
dev
...
hotfix/jso
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d10e806eb | ||
|
|
fe1e5f79bd | ||
| 66fcacc45c | |||
| 23ef455122 |
@@ -45,14 +45,10 @@ jobs:
|
||||
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
|
||||
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
|
||||
|
||||
- name: Free Docker space
|
||||
- name: Inspect Docker space
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker system df || true
|
||||
docker buildx prune --all --force || true
|
||||
docker builder prune --all --force || true
|
||||
docker system prune --all --force --volumes || true
|
||||
docker system df || true
|
||||
|
||||
- name: Run quality in golden image
|
||||
env:
|
||||
@@ -104,8 +100,6 @@ jobs:
|
||||
}
|
||||
|
||||
if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1 || ! image_usable; then
|
||||
docker buildx prune --all --force || true
|
||||
docker builder prune --all --force || true
|
||||
docker buildx build \
|
||||
-f ./docker/Dockerfile \
|
||||
--target ci-deps-base \
|
||||
@@ -182,15 +176,11 @@ jobs:
|
||||
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
|
||||
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
|
||||
|
||||
- name: Free Docker build space
|
||||
- name: Inspect Docker build space
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker system df || true
|
||||
docker buildx prune --all --force || true
|
||||
docker builder prune --all --force || true
|
||||
docker system prune --all --force --volumes || true
|
||||
docker system df || true
|
||||
|
||||
- name: Build and push dev images
|
||||
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }}
|
||||
@@ -274,8 +264,6 @@ jobs:
|
||||
return 0
|
||||
fi
|
||||
|
||||
docker buildx prune --all --force || true
|
||||
docker builder prune --all --force || true
|
||||
docker buildx build \
|
||||
-f ./docker/Dockerfile \
|
||||
--target "${target}" \
|
||||
|
||||
@@ -1,192 +0,0 @@
|
||||
name: Deploy Customer Main
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: mostovik-backend-customer-main-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru
|
||||
CUSTOMER_REGISTRY_NAMESPACE: avm
|
||||
CUSTOMER_WEB_IMAGE: mostovik-backend-web
|
||||
CUSTOMER_CELERY_IMAGE: mostovik-backend-celery
|
||||
CUSTOMER_DEPLOY_USER: ecos
|
||||
CUSTOMER_DEPLOY_HOST: 10.0.10.174
|
||||
CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121
|
||||
CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml
|
||||
CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Build, Push, Deploy
|
||||
runs-on: [backend-docker]
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Ensure main branch
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${GITHUB_REF_NAME}" != "main" ]; then
|
||||
echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout code
|
||||
run: |
|
||||
set -euo pipefail
|
||||
REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
|
||||
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
|
||||
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
|
||||
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
|
||||
|
||||
- name: Validate deploy inputs
|
||||
env:
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
GITEA_TOKEN: ${{ gitea.token }}
|
||||
CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
|
||||
CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
|
||||
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
|
||||
home_dir="${HOME:-/root}"
|
||||
|
||||
if [ -z "${registry_user}" ]; then
|
||||
echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${registry_password}" ]; then
|
||||
echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \
|
||||
&& [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \
|
||||
&& [ -z "${DEPLOY_SSH_KEY:-}" ] \
|
||||
&& [ ! -f "${home_dir}/.ssh/ci-key" ] \
|
||||
&& [ ! -f "/root/.ssh/ci-key" ]; then
|
||||
echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! docker buildx inspect customer-builder >/dev/null 2>&1; then
|
||||
docker buildx create --name customer-builder --driver docker-container --use
|
||||
else
|
||||
docker buildx use customer-builder
|
||||
fi
|
||||
docker buildx inspect --bootstrap
|
||||
|
||||
- name: Build and push customer images
|
||||
env:
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
GITEA_TOKEN: ${{ gitea.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
|
||||
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
|
||||
sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)"
|
||||
registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}"
|
||||
web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}"
|
||||
celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}"
|
||||
|
||||
printf '%s' "${registry_password}" \
|
||||
| docker login "${CUSTOMER_REGISTRY_HOST}" \
|
||||
-u "${registry_user}" \
|
||||
--password-stdin
|
||||
|
||||
docker buildx build \
|
||||
--platform linux/amd64 \
|
||||
-f ./docker/Dockerfile \
|
||||
--target runtime-web \
|
||||
--build-arg INSTALL_DEV=false \
|
||||
--push \
|
||||
-t "${web_ref}:latest" \
|
||||
-t "${web_ref}:${sha_short}" \
|
||||
.
|
||||
|
||||
docker buildx build \
|
||||
--platform linux/amd64 \
|
||||
-f ./docker/Dockerfile \
|
||||
--target runtime-celery \
|
||||
--build-arg INSTALL_DEV=false \
|
||||
--push \
|
||||
-t "${celery_ref}:latest" \
|
||||
-t "${celery_ref}:${sha_short}" \
|
||||
.
|
||||
|
||||
- name: Deploy customer stack
|
||||
env:
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
GITEA_TOKEN: ${{ gitea.token }}
|
||||
CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
|
||||
CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
|
||||
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
|
||||
case "${registry_user}" in
|
||||
*[!A-Za-z0-9._@-]*)
|
||||
echo "Registry user contains unsupported characters" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
home_dir="${HOME:-/root}"
|
||||
mkdir -p "${home_dir}/.ssh"
|
||||
key_path="${home_dir}/.ssh/customer_deploy_key"
|
||||
if [ -f "/root/.ssh/ci-key" ]; then
|
||||
cp "/root/.ssh/ci-key" "${key_path}"
|
||||
elif [ -f "${home_dir}/.ssh/ci-key" ]; then
|
||||
cp "${home_dir}/.ssh/ci-key" "${key_path}"
|
||||
elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then
|
||||
printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}"
|
||||
elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then
|
||||
printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}"
|
||||
elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then
|
||||
printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}"
|
||||
else
|
||||
echo "Customer deploy SSH key is unavailable" >&2
|
||||
exit 1
|
||||
fi
|
||||
chmod 600 "${key_path}"
|
||||
|
||||
ssh_common=(
|
||||
-i "${key_path}"
|
||||
-o BatchMode=yes
|
||||
-o IdentitiesOnly=yes \
|
||||
-o StrictHostKeyChecking=no
|
||||
-o UserKnownHostsFile=/dev/null
|
||||
-o GlobalKnownHostsFile=/dev/null
|
||||
-o LogLevel=ERROR
|
||||
-o ConnectTimeout=15
|
||||
)
|
||||
proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}"
|
||||
remote_command="set -euo pipefail
|
||||
flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'"
|
||||
|
||||
ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true"
|
||||
printf '%s' "${registry_password}" \
|
||||
| ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \
|
||||
"docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin"
|
||||
ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}"
|
||||
5
docker/Dockerfile.health-hotfix
Normal file
5
docker/Dockerfile.health-hotfix
Normal file
@@ -0,0 +1,5 @@
|
||||
# Use only with the verified deployed image and a views-only application diff.
|
||||
ARG HEALTH_BASE_IMAGE
|
||||
FROM ${HEALTH_BASE_IMAGE}
|
||||
|
||||
COPY --chown=0:0 --chmod=0644 src/apps/core/views.py /app/src/apps/core/views.py
|
||||
36
docs/health-probes.md
Normal file
36
docs/health-probes.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Health probes
|
||||
|
||||
`GET /health/live/` and `GET /health/ready/` are unauthenticated infrastructure
|
||||
probes and do not consume API rate limits. Liveness reports the running process;
|
||||
readiness checks the database and still returns HTTP 503 when it is unavailable.
|
||||
The comprehensive `/health/` endpoint and ordinary API endpoints retain their
|
||||
configured throttling.
|
||||
|
||||
## Internal-main health hotfix
|
||||
|
||||
The internal-main Docker probe runs every 10 seconds. Applying the shared
|
||||
anonymous limit of 100 requests/hour to it causes false HTTP 429 failures.
|
||||
|
||||
Customer deployment workflows have been removed. Do not rerun historical
|
||||
customer workflows: they execute the configuration from their original commit.
|
||||
|
||||
For this views-only fix, `docker/Dockerfile.health-hotfix` can build a web image
|
||||
from the exact deployed `repository@sha256:digest`, preserving its dependencies,
|
||||
startup command and all other application files. Before using this recipe,
|
||||
verify that the base image matches the source baseline and that the application
|
||||
diff contains only the probe changes in `src/apps/core/views.py`. Label the new
|
||||
image with the committed source revision and publish an immutable digest.
|
||||
|
||||
The normal internal-main backend release requires `--refresh-data` and replaces
|
||||
the database/media clone. It must not be used for this health-only fix. Instead,
|
||||
under the existing release lock, verify that the candidate has no pending
|
||||
migrations, keep a private copy of the current manifest, and change only
|
||||
`MOSTOVIK_BACKEND_WEB_IMAGE`. Recreate only `mostovik-web` with `--no-deps` and the
|
||||
already-pulled image. Accept the manifest atomically after HTTP probes and Docker
|
||||
health succeed; restore the previous web image and manifest if they fail.
|
||||
Preserve database/media selections, Redis state and every other service image.
|
||||
|
||||
Acceptance includes repeated probes beyond the anonymous request allowance,
|
||||
database-failure regression coverage, and a control proving that ordinary
|
||||
endpoints are still throttled. Do not disable throttling globally or clear Redis
|
||||
to make the healthcheck pass.
|
||||
@@ -53,9 +53,22 @@ XLSX или JSON заново. Он упаковывает файлы после
|
||||
больше из-за разбиения крупных XLSX. Если финансовые показатели выбраны вместе
|
||||
с другим форматом, в ZIP для них всё равно включается JSON.
|
||||
|
||||
Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`,
|
||||
CSV и XLSX начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`,
|
||||
`КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого
|
||||
`payload`. Все записи текущего года включаются в публичные файлы, а техническое
|
||||
`payload`. JSON каждого источника — массив организаций с `okpo` и массивом
|
||||
записей под ключом группы API. Например, `bankruptcy-procedures.json`:
|
||||
|
||||
```json
|
||||
[{"okpo":"12345678","bankruptcy":[{"uid":"...","source_group":"bankruptcy","payload.case_number":"..."}]}]
|
||||
```
|
||||
|
||||
Записи одной организации собираются вместе независимо от порядка чтения из БД.
|
||||
`ОКПО` переносится из записи в `okpo` оболочки; остальные поля записи остаются
|
||||
в прежнем виде. При отсутствии ОКПО экспортируется `null`, а для группировки
|
||||
используются ИНН, ОГРН и наименование, чтобы не смешивать разные организации.
|
||||
Вложенные финансовые строки `financial_lines` сохраняются внутри финансовой
|
||||
записи. При выборе нескольких источников ZIP содержит отдельный JSON-файл для
|
||||
каждого. Все записи текущего года включаются в публичные файлы, а техническое
|
||||
наименование внешнего поставщика нейтрализуется. Исходные значения в БД
|
||||
сохраняются для работы интеграции и дедупликации.
|
||||
|
||||
@@ -71,8 +84,9 @@ Celery Beat запускает
|
||||
1. читает каждую группу из БД один раз без глобальной сортировки миллионов строк;
|
||||
выборка текущего года использует функциональный индекс по году строковой
|
||||
`record_date`, а записи без предметной даты — индекс `created_at`;
|
||||
2. пишет compact JSON-массив на диск и использует его как готовый JSON без второй копии;
|
||||
3. потоково создаёт CSV и XLSX без накопления всех строк в памяти;
|
||||
2. пишет плоский spool на диск, затем группирует JSON по ОКПО через временный
|
||||
дисковый индекс SQLite без накопления всех строк в памяти;
|
||||
3. потоково создаёт CSV и XLSX из того же spool;
|
||||
4. разбивает XLSX по умолчанию по 100 000 строк на отдельные файлы, ограничивая временный XML и не превышая лимит Excel;
|
||||
5. записывает размеры файлов, номера частей и календарный `export_year` в manifest;
|
||||
6. атомарно переключает `current.json` только после готовности всей матрицы;
|
||||
@@ -84,9 +98,11 @@ Celery Beat запускает
|
||||
### Расчёт диска
|
||||
|
||||
Атомарная публикация требует одновременно хранить уже опубликованные поколения
|
||||
и одно новое поколение в staging. Минимальный запас под артефакты рассчитывается
|
||||
как `(SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP + 1) * размер поколения`, плюс
|
||||
рабочий запас файловой системы. Исторический снимок dev от 2026-08-03 до
|
||||
и одно новое поколение в staging. Во время группировки JSON дополнительно
|
||||
создаётся временная база SQLite размером порядка исходного spool. Минимальный
|
||||
запас под артефакты рассчитывается как
|
||||
`(SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP + 1) * размер поколения`, плюс
|
||||
spool, временная база SQLite и рабочий запас файловой системы. Исторический снимок dev от 2026-08-03 до
|
||||
ограничения по году занимал 16,55 ГБ (54 физических файла); актуальный размер
|
||||
годового поколения нужно брать из `total_size` результата команды сборки.
|
||||
Временная копия целого ZIP при скачивании не создаётся.
|
||||
@@ -104,7 +120,10 @@ Celery Beat запускает
|
||||
Celery worker. В Docker Compose используется `./media:/app/media`.
|
||||
|
||||
После первого развёртывания готовое поколение можно создать сразу, не ожидая
|
||||
ночного расписания:
|
||||
ночного расписания. После обновления формата JSON поколения с manifest версии 2
|
||||
не раздаются: до первой успешной сборки версии 3 API возвращает
|
||||
`503 source_export_not_ready`. Запустите команду ниже после развёртывания,
|
||||
чтобы восстановить доступность выгрузок сразу:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=src uv run python src/manage.py build_source_record_exports
|
||||
|
||||
@@ -167,6 +167,7 @@ class LivenessView(APIView):
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes = []
|
||||
throttle_classes = []
|
||||
|
||||
@swagger_auto_schema(
|
||||
tags=[HEALTH_TAG],
|
||||
@@ -174,7 +175,7 @@ class LivenessView(APIView):
|
||||
operation_description="Возвращает 200 если приложение запущено.",
|
||||
responses={
|
||||
200: "Приложение запущено",
|
||||
**ErrorResponses.PUBLIC,
|
||||
500: CommonResponses.SERVER_ERROR,
|
||||
},
|
||||
)
|
||||
def get(self, request: Request) -> Response:
|
||||
@@ -191,6 +192,7 @@ class ReadinessView(APIView):
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes = []
|
||||
throttle_classes = []
|
||||
|
||||
@swagger_auto_schema(
|
||||
tags=[HEALTH_TAG],
|
||||
@@ -201,7 +203,7 @@ class ReadinessView(APIView):
|
||||
responses={
|
||||
200: "Приложение готово обрабатывать запросы",
|
||||
503: CommonResponses.SERVICE_UNAVAILABLE,
|
||||
**ErrorResponses.PUBLIC,
|
||||
500: CommonResponses.SERVER_ERROR,
|
||||
},
|
||||
)
|
||||
def get(self, request: Request) -> Response:
|
||||
|
||||
@@ -8,6 +8,7 @@ import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import sqlite3
|
||||
import zipfile
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
@@ -38,7 +39,7 @@ EXPORT_FORMAT_XLSX = "xlsx"
|
||||
EXPORT_FORMAT_JSON = "json"
|
||||
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
|
||||
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
|
||||
EXPORT_MANIFEST_VERSION = 2
|
||||
EXPORT_MANIFEST_VERSION = 3
|
||||
CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json"
|
||||
GENERATION_MANIFEST_FILE_NAME = "manifest.json"
|
||||
GENERATION_DIRECTORY_NAME = "generations"
|
||||
@@ -233,6 +234,7 @@ def build_source_record_export_artifacts(
|
||||
file_format=file_format,
|
||||
)
|
||||
artifact_paths = _render_source_group_artifact(
|
||||
source_group=source_group,
|
||||
row_spool_path=row_spool_path,
|
||||
output_path=staging_directory / file_name,
|
||||
headers=headers,
|
||||
@@ -645,6 +647,7 @@ def _spool_source_group_rows(
|
||||
|
||||
def _render_source_group_artifact(
|
||||
*,
|
||||
source_group: str,
|
||||
row_spool_path: Path,
|
||||
output_path: Path,
|
||||
headers: Sequence[str],
|
||||
@@ -665,7 +668,11 @@ def _render_source_group_artifact(
|
||||
headers=headers,
|
||||
records_count=records_count,
|
||||
)
|
||||
_render_json_file(row_spool_path=row_spool_path, output_path=output_path)
|
||||
_render_json_file(
|
||||
source_group=source_group,
|
||||
row_spool_path=row_spool_path,
|
||||
output_path=output_path,
|
||||
)
|
||||
return (output_path,)
|
||||
|
||||
|
||||
@@ -767,10 +774,75 @@ def _build_xlsx_part_path(
|
||||
|
||||
def _render_json_file(
|
||||
*,
|
||||
source_group: str,
|
||||
row_spool_path: Path,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
os.link(row_spool_path, output_path)
|
||||
grouping_path = output_path.with_name(f".{output_path.name}.grouping.sqlite3")
|
||||
grouping = sqlite3.connect(grouping_path)
|
||||
try:
|
||||
# The published spool can contain millions of unsorted records. Sort on
|
||||
# local disk instead of sorting the source queryset or loading it in RAM.
|
||||
grouping.execute("PRAGMA journal_mode=OFF")
|
||||
grouping.execute(
|
||||
"CREATE TABLE rows (group_key TEXT, position INTEGER, okpo TEXT, data TEXT)"
|
||||
)
|
||||
batch: list[tuple[str, int, str | None, str]] = []
|
||||
for position, row in enumerate(_iter_spooled_rows(row_spool_path)):
|
||||
raw_okpo = row.pop("ОКПО", None)
|
||||
okpo = str(raw_okpo) if raw_okpo else None
|
||||
fallback = [row.get("ИНН"), row.get("ОГРН"), row.get("Наименование")]
|
||||
if not any(fallback):
|
||||
fallback.append(row.get("uid"))
|
||||
group_key = json.dumps(
|
||||
[okpo] if okpo else [None, *fallback],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
batch.append(
|
||||
(
|
||||
group_key,
|
||||
position,
|
||||
okpo,
|
||||
json.dumps(row, ensure_ascii=False, separators=(",", ":")),
|
||||
)
|
||||
)
|
||||
if len(batch) >= SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE:
|
||||
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
|
||||
batch.clear()
|
||||
if batch:
|
||||
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
|
||||
grouping.commit()
|
||||
grouping.execute("CREATE INDEX rows_by_group ON rows (group_key, position)")
|
||||
|
||||
with output_path.open("w", encoding="utf-8") as output:
|
||||
output.write("[")
|
||||
current_group: str | None = None
|
||||
first_record = True
|
||||
for group_key, okpo, data in grouping.execute(
|
||||
"SELECT group_key, okpo, data FROM rows ORDER BY group_key, position"
|
||||
):
|
||||
if group_key != current_group:
|
||||
if current_group is not None:
|
||||
output.write("]},")
|
||||
output.write(
|
||||
'{"okpo":'
|
||||
+ json.dumps(okpo, ensure_ascii=False)
|
||||
+ ","
|
||||
+ json.dumps(source_group)
|
||||
+ ":["
|
||||
)
|
||||
current_group = group_key
|
||||
first_record = True
|
||||
if not first_record:
|
||||
output.write(",")
|
||||
output.write(data)
|
||||
first_record = False
|
||||
if current_group is not None:
|
||||
output.write("]}")
|
||||
output.write("]")
|
||||
finally:
|
||||
grouping.close()
|
||||
grouping_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _build_record_row(
|
||||
|
||||
@@ -3,18 +3,32 @@
|
||||
import sys
|
||||
import types
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.core import views as core_views
|
||||
from apps.core.views import HealthCheckView
|
||||
from django.urls import reverse
|
||||
from django.core.cache.backends.locmem import LocMemCache
|
||||
from django.urls import path, reverse
|
||||
from django.utils import timezone
|
||||
from drf_yasg import openapi
|
||||
from drf_yasg.generators import OpenAPISchemaGenerator
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIRequestFactory, APITestCase
|
||||
from rest_framework.throttling import AnonRateThrottle
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from tests.apps.user.factories import UserFactory
|
||||
from tests.utils.fixtures import fake
|
||||
|
||||
|
||||
class _RejectAllThrottle:
|
||||
def allow_request(self, request, view):
|
||||
return False
|
||||
|
||||
def wait(self):
|
||||
return None
|
||||
|
||||
|
||||
class HealthCheckViewTest(APITestCase):
|
||||
"""Tests for HealthCheckView"""
|
||||
|
||||
@@ -23,6 +37,12 @@ class HealthCheckViewTest(APITestCase):
|
||||
url = reverse("core:health")
|
||||
self.assertEqual(url, "/health/")
|
||||
|
||||
def test_detailed_health_check_keeps_global_throttling(self):
|
||||
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
|
||||
response = self.client.get(reverse("core:health"))
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
|
||||
|
||||
def test_health_check_success(self):
|
||||
"""Test health check returns healthy status"""
|
||||
url = reverse("core:health")
|
||||
@@ -228,6 +248,13 @@ class LivenessViewTest(APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["status"], "alive")
|
||||
|
||||
def test_liveness_bypasses_global_throttling(self):
|
||||
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
|
||||
response = self.client.get(reverse("core:liveness"))
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["status"], "alive")
|
||||
|
||||
|
||||
class ReadinessViewTest(APITestCase):
|
||||
"""Tests for ReadinessView"""
|
||||
@@ -245,6 +272,13 @@ class ReadinessViewTest(APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["status"], "ready")
|
||||
|
||||
def test_readiness_bypasses_global_throttling(self):
|
||||
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
|
||||
response = self.client.get(reverse("core:readiness"))
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["status"], "ready")
|
||||
|
||||
def test_readiness_returns_not_ready_on_db_error(self):
|
||||
original_connection = core_views.connection
|
||||
|
||||
@@ -262,7 +296,8 @@ class ReadinessViewTest(APITestCase):
|
||||
try:
|
||||
core_views.connection = _BrokenConnection()
|
||||
url = reverse("core:readiness")
|
||||
response = self.client.get(url)
|
||||
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
|
||||
response = self.client.get(url)
|
||||
finally:
|
||||
core_views.connection = original_connection
|
||||
|
||||
@@ -270,6 +305,65 @@ class ReadinessViewTest(APITestCase):
|
||||
self.assertEqual(response.data["status"], "not_ready")
|
||||
|
||||
|
||||
class HealthProbeThrottlingTest(APITestCase):
|
||||
def test_openapi_keeps_rate_limit_only_on_detailed_health(self):
|
||||
generator = OpenAPISchemaGenerator(
|
||||
info=openapi.Info(title="Health probes", default_version="v1"),
|
||||
patterns=[
|
||||
path("health/", HealthCheckView.as_view()),
|
||||
path("health/live/", core_views.LivenessView.as_view()),
|
||||
path("health/ready/", core_views.ReadinessView.as_view()),
|
||||
],
|
||||
)
|
||||
schema = generator.get_schema(request=None, public=True)
|
||||
responses = {
|
||||
f"{schema.base_path.rstrip('/')}{url}": item["get"].responses
|
||||
for url, item in schema.paths.items()
|
||||
}
|
||||
|
||||
self.assertNotIn("429", responses["/health/live/"])
|
||||
self.assertNotIn("429", responses["/health/ready/"])
|
||||
self.assertIn("503", responses["/health/ready/"])
|
||||
self.assertIn("429", responses["/health/"])
|
||||
|
||||
def test_probes_do_not_consume_or_obey_exhausted_anonymous_quota(self):
|
||||
probe_cache = LocMemCache(self.id(), {})
|
||||
probe_cache.clear()
|
||||
self.addCleanup(probe_cache.clear)
|
||||
|
||||
class OnePerMinuteAnonThrottle(AnonRateThrottle):
|
||||
rate = "1/min"
|
||||
cache = probe_cache
|
||||
|
||||
with (
|
||||
patch.object(APIView, "throttle_classes", [OnePerMinuteAnonThrottle]),
|
||||
patch.object(
|
||||
HealthCheckView, "_check_database", return_value={"status": "up"}
|
||||
),
|
||||
patch.object(
|
||||
HealthCheckView, "_check_redis", return_value={"status": "up"}
|
||||
),
|
||||
):
|
||||
for quota_exhausted in (False, True):
|
||||
for _ in range(3):
|
||||
for name, expected_status in (
|
||||
("liveness", "alive"),
|
||||
("readiness", "ready"),
|
||||
):
|
||||
with self.subTest(probe=name, quota_exhausted=quota_exhausted):
|
||||
response = self.client.get(reverse(f"core:{name}"))
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["status"], expected_status)
|
||||
|
||||
if not quota_exhausted:
|
||||
response = self.client.get(reverse("core:health"))
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
response = self.client.get(reverse("core:health"))
|
||||
self.assertEqual(
|
||||
response.status_code, status.HTTP_429_TOO_MANY_REQUESTS
|
||||
)
|
||||
|
||||
|
||||
class APIVersioningURLTest(APITestCase):
|
||||
"""Tests for API versioning URL structure"""
|
||||
|
||||
|
||||
@@ -38,6 +38,10 @@ from rest_framework.test import APITestCase
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
|
||||
def _source_records(groups, source_group):
|
||||
return [record for group in groups for record in group[source_group]]
|
||||
|
||||
|
||||
class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
"""Checks admin-only source-record export contract."""
|
||||
|
||||
@@ -95,6 +99,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
self.assertEqual(generation.export_year, timezone.localdate().year)
|
||||
self.assertIn('"artifacts_count": 25', command_output.getvalue())
|
||||
|
||||
def test_previous_json_generation_requires_rebuild(self):
|
||||
build_source_record_export_artifacts()
|
||||
manifest_path = Path(self.export_directory.name) / "current.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["version"], 3)
|
||||
manifest["version"] = 2
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
|
||||
load_current_source_record_export_generation()
|
||||
|
||||
def test_generation_contains_only_records_from_its_calendar_year(self):
|
||||
export_year = 2026
|
||||
generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC)
|
||||
@@ -174,7 +189,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
if artifact.source_group == SourceGroup.PLANNED_INSPECTIONS.value
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
inspection_rows = json.loads(inspection_path.read_text(encoding="utf-8"))
|
||||
inspection_rows = _source_records(
|
||||
json.loads(inspection_path.read_text(encoding="utf-8")),
|
||||
SourceGroup.PLANNED_INSPECTIONS.value,
|
||||
)
|
||||
self.assertEqual(
|
||||
{row["uid"] for row in inspection_rows},
|
||||
{str(current_record.pk), str(dateless_current_record.pk)},
|
||||
@@ -185,7 +203,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == SourceGroup.FINANCIAL_INDICATORS.value
|
||||
)
|
||||
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
|
||||
financial_rows = _source_records(
|
||||
json.loads(financial_path.read_text(encoding="utf-8")),
|
||||
SourceGroup.FINANCIAL_INDICATORS.value,
|
||||
)
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in financial_rows],
|
||||
[str(current_financial_record.pk)],
|
||||
@@ -308,8 +329,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
self.assertIn("payload.risk.score", rows[0])
|
||||
self.assertIn("payload.tags", rows[0])
|
||||
|
||||
financial_rows = json.loads(
|
||||
archive.read("financial-indicators.json").decode("utf-8"),
|
||||
financial_groups = json.loads(
|
||||
archive.read("financial-indicators.json").decode("utf-8")
|
||||
)
|
||||
self.assertEqual(financial_groups[0]["okpo"], "12345678")
|
||||
financial_rows = _source_records(
|
||||
financial_groups, SourceGroup.FINANCIAL_INDICATORS.value
|
||||
)
|
||||
self.assertEqual(financial_rows[0]["Наименование"], organization.full_name)
|
||||
self.assertEqual(
|
||||
@@ -582,6 +607,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
for file_format in ("json", "csv", "xlsx"):
|
||||
artifact_path = export_directory / f"artifact.{file_format}"
|
||||
_render_source_group_artifact(
|
||||
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
|
||||
row_spool_path=spool_path,
|
||||
output_path=artifact_path,
|
||||
headers=headers,
|
||||
@@ -623,18 +649,30 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
self.assertFalse(queryset.ordered)
|
||||
self.assertFalse(queryset.query.default_ordering)
|
||||
|
||||
def test_json_artifact_reuses_valid_canonical_spool_without_copying_it(self):
|
||||
def test_json_artifact_groups_nonadjacent_records_by_okpo(self):
|
||||
organization = Organization.objects.create(
|
||||
name='ООО "JSON без копии"',
|
||||
inn="7707083813",
|
||||
okpo="00123456",
|
||||
)
|
||||
other_organization = Organization.objects.create(
|
||||
name='ООО "Вторая организация"',
|
||||
inn="7707083814",
|
||||
okpo="87654321",
|
||||
)
|
||||
extension = PlannedInspectionExtension.objects.create(
|
||||
organization=organization,
|
||||
title="Плановые проверки Генпрокуратуры России",
|
||||
)
|
||||
for index in range(2):
|
||||
other_extension = PlannedInspectionExtension.objects.create(
|
||||
organization=other_organization,
|
||||
title="Плановые проверки Генпрокуратуры России",
|
||||
)
|
||||
for index, record_extension in enumerate(
|
||||
(extension, other_extension, extension)
|
||||
):
|
||||
OrganizationSourceRecord.objects.create(
|
||||
extension=extension,
|
||||
extension=record_extension,
|
||||
record_type="inspection",
|
||||
source="inspections",
|
||||
external_id=f"INSP-JSON-{index}",
|
||||
@@ -652,6 +690,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
)
|
||||
|
||||
_render_source_group_artifact(
|
||||
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
|
||||
row_spool_path=spool_path,
|
||||
output_path=artifact_path,
|
||||
headers=headers,
|
||||
@@ -659,9 +698,21 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
records_count=records_count,
|
||||
)
|
||||
|
||||
self.assertEqual(records_count, 2)
|
||||
self.assertEqual(len(json.loads(spool_path.read_text())), 2)
|
||||
self.assertTrue(os.path.samefile(spool_path, artifact_path))
|
||||
self.assertEqual(records_count, 3)
|
||||
self.assertEqual(len(json.loads(spool_path.read_text())), 3)
|
||||
groups = json.loads(artifact_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
{group["okpo"] for group in groups}, {"00123456", "87654321"}
|
||||
)
|
||||
grouped = {
|
||||
group["okpo"]: group[SourceGroup.PLANNED_INSPECTIONS.value]
|
||||
for group in groups
|
||||
}
|
||||
self.assertEqual(len(grouped["00123456"]), 2)
|
||||
self.assertEqual(len(grouped["87654321"]), 1)
|
||||
self.assertNotIn("ОКПО", grouped["00123456"][0])
|
||||
self.assertFalse(os.path.samefile(spool_path, artifact_path))
|
||||
self.assertFalse(list(Path(temporary_directory).glob("*.sqlite3")))
|
||||
|
||||
def test_generation_publishes_complete_matrix_and_keeps_previous_on_failure(self):
|
||||
first_generation = build_source_record_export_artifacts()
|
||||
|
||||
26
tests/test_deployment_boundaries.py
Normal file
26
tests/test_deployment_boundaries.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
WORKFLOWS = Path(__file__).resolve().parents[1] / ".gitea" / "workflows"
|
||||
|
||||
|
||||
def test_workflows_do_not_deploy_to_center():
|
||||
assert not (WORKFLOWS / "deploy-customer-main.yml").exists()
|
||||
for path in WORKFLOWS.glob("*.y*ml"):
|
||||
workflow = yaml.safe_load(path.read_text())
|
||||
assert "deploy_customer_main" not in workflow["jobs"]
|
||||
content = path.read_text()
|
||||
for forbidden in ("CUSTOMER_DEPLOY_", "10.0.10.174", "/ecos/release-service"):
|
||||
assert forbidden not in content, (path.name, forbidden)
|
||||
|
||||
|
||||
def test_ci_does_not_prune_shared_runner_resources():
|
||||
for path in WORKFLOWS.glob("*.y*ml"):
|
||||
content = path.read_text()
|
||||
for command in (
|
||||
"docker system prune",
|
||||
"docker builder prune",
|
||||
"docker buildx prune",
|
||||
):
|
||||
assert command not in content, (path.name, command)
|
||||
Reference in New Issue
Block a user