Compare commits

...

4 Commits

Author SHA1 Message Date
Aleksandr Meshchryakov
3d10e806eb fix: group JSON source exports by organization
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 1m17s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 0s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 1s
2026-09-27 10:27:06 +02:00
Aleksandr Meshchryakov
fe1e5f79bd fix: exclude infrastructure probes from API throttling
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 49s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 1s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 1s
CI/CD Pipeline / Quality Gate (push) Successful in 49s
CI/CD Pipeline / Build and Push Images (push) Successful in 0s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
2026-09-16 09:39:56 +02:00
avm
66fcacc45c Merge pull request 'Merge dev into main for customer release' (#39) from feature/customer-release-20260811 into main
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 1m11s
CI/CD Pipeline / Build and Push Images (push) Successful in 0s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 0s
Deploy Customer Main / Build, Push, Deploy (push) Successful in 11m34s
2026-08-11 13:30:24 +03:00
23ef455122 Merge dev into main for customer release
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 1m10s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 0s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 0s
CI/CD Pipeline / Quality Gate (push) Successful in 35m48s
CI/CD Pipeline / Build and Push Images (push) Successful in 0s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
2026-08-11 12:16:08 +02:00
10 changed files with 332 additions and 231 deletions

View File

@@ -45,14 +45,10 @@ jobs:
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Free Docker space
- name: Inspect Docker space
run: |
set -euo pipefail
docker system df || true
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker system prune --all --force --volumes || true
docker system df || true
- name: Run quality in golden image
env:
@@ -104,8 +100,6 @@ jobs:
}
if ! docker buildx imagetools inspect "${CI_GOLDEN_REF}:${GOLDEN_TAG}" >/dev/null 2>&1 || ! image_usable; then
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker buildx build \
-f ./docker/Dockerfile \
--target ci-deps-base \
@@ -182,15 +176,11 @@ jobs:
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Free Docker build space
- name: Inspect Docker build space
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }}
run: |
set -euo pipefail
docker system df || true
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker system prune --all --force --volumes || true
docker system df || true
- name: Build and push dev images
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && !contains(github.event.head_commit.message, '#no_image') }}
@@ -274,8 +264,6 @@ jobs:
return 0
fi
docker buildx prune --all --force || true
docker builder prune --all --force || true
docker buildx build \
-f ./docker/Dockerfile \
--target "${target}" \

View File

@@ -1,192 +0,0 @@
name: Deploy Customer Main
on:
push:
branches:
- main
workflow_dispatch:
concurrency:
group: mostovik-backend-customer-main-${{ github.ref }}
cancel-in-progress: true
env:
CUSTOMER_REGISTRY_HOST: registry.dev.nii-ecos.ru
CUSTOMER_REGISTRY_NAMESPACE: avm
CUSTOMER_WEB_IMAGE: mostovik-backend-web
CUSTOMER_CELERY_IMAGE: mostovik-backend-celery
CUSTOMER_DEPLOY_USER: ecos
CUSTOMER_DEPLOY_HOST: 10.0.10.174
CUSTOMER_SSH_PROXY_HOST: root@10.10.0.121
CUSTOMER_COMPOSE_FILE: /ecos/docker-compose.yml
CUSTOMER_DEPLOY_SCRIPT: /ecos/pull-and-run.sh
jobs:
deploy:
name: Build, Push, Deploy
runs-on: [backend-docker]
timeout-minutes: 90
steps:
- name: Ensure main branch
run: |
set -euo pipefail
if [ "${GITHUB_REF_NAME}" != "main" ]; then
echo "Customer deploy is allowed only from main; current ref is ${GITHUB_REF_NAME}" >&2
exit 1
fi
- name: Checkout code
run: |
set -euo pipefail
REPO_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
BRANCH="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}"
git -c core.hooksPath=/dev/null clone --depth=1 --branch="${BRANCH}" "${REPO_URL}" .
git -c core.hooksPath=/dev/null checkout "${GITHUB_SHA}"
- name: Validate deploy inputs
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
GITEA_TOKEN: ${{ gitea.token }}
CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
run: |
set -euo pipefail
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
home_dir="${HOME:-/root}"
if [ -z "${registry_user}" ]; then
echo "Missing registry user secret: set REGISTRY_USER or REGISTRY_USERNAME" >&2
exit 1
fi
if [ -z "${registry_password}" ]; then
echo "Missing registry password secret: set REGISTRY_TOKEN or REGISTRY_PASSWORD" >&2
exit 1
fi
if [ -z "${CUSTOMER_DEPLOY_SSH_KEY:-}" ] \
&& [ -z "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ] \
&& [ -z "${DEPLOY_SSH_KEY:-}" ] \
&& [ ! -f "${home_dir}/.ssh/ci-key" ] \
&& [ ! -f "/root/.ssh/ci-key" ]; then
echo "Missing customer SSH key: set CUSTOMER_DEPLOY_SSH_KEY_B64, CUSTOMER_DEPLOY_SSH_KEY, DEPLOY_SSH_KEY, or install ~/.ssh/ci-key on the runner" >&2
exit 1
fi
- name: Setup Docker Buildx
run: |
set -euo pipefail
if ! docker buildx inspect customer-builder >/dev/null 2>&1; then
docker buildx create --name customer-builder --driver docker-container --use
else
docker buildx use customer-builder
fi
docker buildx inspect --bootstrap
- name: Build and push customer images
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
GITEA_TOKEN: ${{ gitea.token }}
run: |
set -euo pipefail
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)"
registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}"
web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}"
celery_ref="${registry_path}/${CUSTOMER_CELERY_IMAGE}"
printf '%s' "${registry_password}" \
| docker login "${CUSTOMER_REGISTRY_HOST}" \
-u "${registry_user}" \
--password-stdin
docker buildx build \
--platform linux/amd64 \
-f ./docker/Dockerfile \
--target runtime-web \
--build-arg INSTALL_DEV=false \
--push \
-t "${web_ref}:latest" \
-t "${web_ref}:${sha_short}" \
.
docker buildx build \
--platform linux/amd64 \
-f ./docker/Dockerfile \
--target runtime-celery \
--build-arg INSTALL_DEV=false \
--push \
-t "${celery_ref}:latest" \
-t "${celery_ref}:${sha_short}" \
.
- name: Deploy customer stack
env:
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
GITEA_TOKEN: ${{ gitea.token }}
CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
run: |
set -euo pipefail
registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
case "${registry_user}" in
*[!A-Za-z0-9._@-]*)
echo "Registry user contains unsupported characters" >&2
exit 1
;;
esac
home_dir="${HOME:-/root}"
mkdir -p "${home_dir}/.ssh"
key_path="${home_dir}/.ssh/customer_deploy_key"
if [ -f "/root/.ssh/ci-key" ]; then
cp "/root/.ssh/ci-key" "${key_path}"
elif [ -f "${home_dir}/.ssh/ci-key" ]; then
cp "${home_dir}/.ssh/ci-key" "${key_path}"
elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then
printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}"
elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then
printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}"
elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then
printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}"
else
echo "Customer deploy SSH key is unavailable" >&2
exit 1
fi
chmod 600 "${key_path}"
ssh_common=(
-i "${key_path}"
-o BatchMode=yes
-o IdentitiesOnly=yes \
-o StrictHostKeyChecking=no
-o UserKnownHostsFile=/dev/null
-o GlobalKnownHostsFile=/dev/null
-o LogLevel=ERROR
-o ConnectTimeout=15
)
proxy_command="ssh -i ${key_path} -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=15 -W %h:%p ${CUSTOMER_SSH_PROXY_HOST}"
remote_command="set -euo pipefail
flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'"
ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true"
printf '%s' "${registry_password}" \
| ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \
"docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin"
ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}"

View File

@@ -0,0 +1,5 @@
# Use only with the verified deployed image and a views-only application diff.
ARG HEALTH_BASE_IMAGE
FROM ${HEALTH_BASE_IMAGE}
COPY --chown=0:0 --chmod=0644 src/apps/core/views.py /app/src/apps/core/views.py

36
docs/health-probes.md Normal file
View File

@@ -0,0 +1,36 @@
# Health probes
`GET /health/live/` and `GET /health/ready/` are unauthenticated infrastructure
probes and do not consume API rate limits. Liveness reports the running process;
readiness checks the database and still returns HTTP 503 when it is unavailable.
The comprehensive `/health/` endpoint and ordinary API endpoints retain their
configured throttling.
## Internal-main health hotfix
The internal-main Docker probe runs every 10 seconds. Applying the shared
anonymous limit of 100 requests/hour to it causes false HTTP 429 failures.
Customer deployment workflows have been removed. Do not rerun historical
customer workflows: they execute the configuration from their original commit.
For this views-only fix, `docker/Dockerfile.health-hotfix` can build a web image
from the exact deployed `repository@sha256:digest`, preserving its dependencies,
startup command and all other application files. Before using this recipe,
verify that the base image matches the source baseline and that the application
diff contains only the probe changes in `src/apps/core/views.py`. Label the new
image with the committed source revision and publish an immutable digest.
The normal internal-main backend release requires `--refresh-data` and replaces
the database/media clone. It must not be used for this health-only fix. Instead,
under the existing release lock, verify that the candidate has no pending
migrations, keep a private copy of the current manifest, and change only
`MOSTOVIK_BACKEND_WEB_IMAGE`. Recreate only `mostovik-web` with `--no-deps` and the
already-pulled image. Accept the manifest atomically after HTTP probes and Docker
health succeed; restore the previous web image and manifest if they fail.
Preserve database/media selections, Redis state and every other service image.
Acceptance includes repeated probes beyond the anonymous request allowance,
database-failure regression coverage, and a control proving that ordinary
endpoints are still throttled. Do not disable throttling globally or clear Redis
to make the healthcheck pass.

View File

@@ -53,9 +53,22 @@ XLSX или JSON заново. Он упаковывает файлы после
больше из-за разбиения крупных XLSX. Если финансовые показатели выбраны вместе
с другим форматом, в ZIP для них всё равно включается JSON.
Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`,
CSV и XLSX начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`,
`КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого
`payload`. Все записи текущего года включаются в публичные файлы, а техническое
`payload`. JSON каждого источника — массив организаций с `okpo` и массивом
записей под ключом группы API. Например, `bankruptcy-procedures.json`:
```json
[{"okpo":"12345678","bankruptcy":[{"uid":"...","source_group":"bankruptcy","payload.case_number":"..."}]}]
```
Записи одной организации собираются вместе независимо от порядка чтения из БД.
`ОКПО` переносится из записи в `okpo` оболочки; остальные поля записи остаются
в прежнем виде. При отсутствии ОКПО экспортируется `null`, а для группировки
используются ИНН, ОГРН и наименование, чтобы не смешивать разные организации.
Вложенные финансовые строки `financial_lines` сохраняются внутри финансовой
записи. При выборе нескольких источников ZIP содержит отдельный JSON-файл для
каждого. Все записи текущего года включаются в публичные файлы, а техническое
наименование внешнего поставщика нейтрализуется. Исходные значения в БД
сохраняются для работы интеграции и дедупликации.
@@ -71,8 +84,9 @@ Celery Beat запускает
1. читает каждую группу из БД один раз без глобальной сортировки миллионов строк;
выборка текущего года использует функциональный индекс по году строковой
`record_date`, а записи без предметной даты — индекс `created_at`;
2. пишет compact JSON-массив на диск и использует его как готовый JSON без второй копии;
3. потоково создаёт CSV и XLSX без накопления всех строк в памяти;
2. пишет плоский spool на диск, затем группирует JSON по ОКПО через временный
дисковый индекс SQLite без накопления всех строк в памяти;
3. потоково создаёт CSV и XLSX из того же spool;
4. разбивает XLSX по умолчанию по 100 000 строк на отдельные файлы, ограничивая временный XML и не превышая лимит Excel;
5. записывает размеры файлов, номера частей и календарный `export_year` в manifest;
6. атомарно переключает `current.json` только после готовности всей матрицы;
@@ -84,9 +98,11 @@ Celery Beat запускает
### Расчёт диска
Атомарная публикация требует одновременно хранить уже опубликованные поколения
и одно новое поколение в staging. Минимальный запас под артефакты рассчитывается
как `(SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP + 1) * размер поколения`, плюс
рабочий запас файловой системы. Исторический снимок dev от 2026-08-03 до
и одно новое поколение в staging. Во время группировки JSON дополнительно
создаётся временная база SQLite размером порядка исходного spool. Минимальный
запас под артефакты рассчитывается как
`(SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP + 1) * размер поколения`, плюс
spool, временная база SQLite и рабочий запас файловой системы. Исторический снимок dev от 2026-08-03 до
ограничения по году занимал 16,55 ГБ (54 физических файла); актуальный размер
годового поколения нужно брать из `total_size` результата команды сборки.
Временная копия целого ZIP при скачивании не создаётся.
@@ -104,7 +120,10 @@ Celery Beat запускает
Celery worker. В Docker Compose используется `./media:/app/media`.
После первого развёртывания готовое поколение можно создать сразу, не ожидая
ночного расписания:
ночного расписания. После обновления формата JSON поколения с manifest версии 2
не раздаются: до первой успешной сборки версии 3 API возвращает
`503 source_export_not_ready`. Запустите команду ниже после развёртывания,
чтобы восстановить доступность выгрузок сразу:
```bash
PYTHONPATH=src uv run python src/manage.py build_source_record_exports

View File

@@ -167,6 +167,7 @@ class LivenessView(APIView):
permission_classes = [AllowAny]
authentication_classes = []
throttle_classes = []
@swagger_auto_schema(
tags=[HEALTH_TAG],
@@ -174,7 +175,7 @@ class LivenessView(APIView):
operation_description="Возвращает 200 если приложение запущено.",
responses={
200: "Приложение запущено",
**ErrorResponses.PUBLIC,
500: CommonResponses.SERVER_ERROR,
},
)
def get(self, request: Request) -> Response:
@@ -191,6 +192,7 @@ class ReadinessView(APIView):
permission_classes = [AllowAny]
authentication_classes = []
throttle_classes = []
@swagger_auto_schema(
tags=[HEALTH_TAG],
@@ -201,7 +203,7 @@ class ReadinessView(APIView):
responses={
200: "Приложение готово обрабатывать запросы",
503: CommonResponses.SERVICE_UNAVAILABLE,
**ErrorResponses.PUBLIC,
500: CommonResponses.SERVER_ERROR,
},
)
def get(self, request: Request) -> Response:

View File

@@ -8,6 +8,7 @@ import os
import re
import secrets
import shutil
import sqlite3
import zipfile
from collections.abc import Iterable, Iterator, Sequence
from dataclasses import dataclass
@@ -38,7 +39,7 @@ EXPORT_FORMAT_XLSX = "xlsx"
EXPORT_FORMAT_JSON = "json"
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
EXPORT_MANIFEST_VERSION = 2
EXPORT_MANIFEST_VERSION = 3
CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json"
GENERATION_MANIFEST_FILE_NAME = "manifest.json"
GENERATION_DIRECTORY_NAME = "generations"
@@ -233,6 +234,7 @@ def build_source_record_export_artifacts(
file_format=file_format,
)
artifact_paths = _render_source_group_artifact(
source_group=source_group,
row_spool_path=row_spool_path,
output_path=staging_directory / file_name,
headers=headers,
@@ -645,6 +647,7 @@ def _spool_source_group_rows(
def _render_source_group_artifact(
*,
source_group: str,
row_spool_path: Path,
output_path: Path,
headers: Sequence[str],
@@ -665,7 +668,11 @@ def _render_source_group_artifact(
headers=headers,
records_count=records_count,
)
_render_json_file(row_spool_path=row_spool_path, output_path=output_path)
_render_json_file(
source_group=source_group,
row_spool_path=row_spool_path,
output_path=output_path,
)
return (output_path,)
@@ -767,10 +774,75 @@ def _build_xlsx_part_path(
def _render_json_file(
*,
source_group: str,
row_spool_path: Path,
output_path: Path,
) -> None:
os.link(row_spool_path, output_path)
grouping_path = output_path.with_name(f".{output_path.name}.grouping.sqlite3")
grouping = sqlite3.connect(grouping_path)
try:
# The published spool can contain millions of unsorted records. Sort on
# local disk instead of sorting the source queryset or loading it in RAM.
grouping.execute("PRAGMA journal_mode=OFF")
grouping.execute(
"CREATE TABLE rows (group_key TEXT, position INTEGER, okpo TEXT, data TEXT)"
)
batch: list[tuple[str, int, str | None, str]] = []
for position, row in enumerate(_iter_spooled_rows(row_spool_path)):
raw_okpo = row.pop("ОКПО", None)
okpo = str(raw_okpo) if raw_okpo else None
fallback = [row.get("ИНН"), row.get("ОГРН"), row.get("Наименование")]
if not any(fallback):
fallback.append(row.get("uid"))
group_key = json.dumps(
[okpo] if okpo else [None, *fallback],
ensure_ascii=False,
)
batch.append(
(
group_key,
position,
okpo,
json.dumps(row, ensure_ascii=False, separators=(",", ":")),
)
)
if len(batch) >= SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE:
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
batch.clear()
if batch:
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
grouping.commit()
grouping.execute("CREATE INDEX rows_by_group ON rows (group_key, position)")
with output_path.open("w", encoding="utf-8") as output:
output.write("[")
current_group: str | None = None
first_record = True
for group_key, okpo, data in grouping.execute(
"SELECT group_key, okpo, data FROM rows ORDER BY group_key, position"
):
if group_key != current_group:
if current_group is not None:
output.write("]},")
output.write(
'{"okpo":'
+ json.dumps(okpo, ensure_ascii=False)
+ ","
+ json.dumps(source_group)
+ ":["
)
current_group = group_key
first_record = True
if not first_record:
output.write(",")
output.write(data)
first_record = False
if current_group is not None:
output.write("]}")
output.write("]")
finally:
grouping.close()
grouping_path.unlink(missing_ok=True)
def _build_record_row(

View File

@@ -3,18 +3,32 @@
import sys
import types
from datetime import timedelta
from unittest.mock import patch
from apps.core import views as core_views
from apps.core.views import HealthCheckView
from django.urls import reverse
from django.core.cache.backends.locmem import LocMemCache
from django.urls import path, reverse
from django.utils import timezone
from drf_yasg import openapi
from drf_yasg.generators import OpenAPISchemaGenerator
from rest_framework import status
from rest_framework.test import APIRequestFactory, APITestCase
from rest_framework.throttling import AnonRateThrottle
from rest_framework.views import APIView
from tests.apps.user.factories import UserFactory
from tests.utils.fixtures import fake
class _RejectAllThrottle:
def allow_request(self, request, view):
return False
def wait(self):
return None
class HealthCheckViewTest(APITestCase):
"""Tests for HealthCheckView"""
@@ -23,6 +37,12 @@ class HealthCheckViewTest(APITestCase):
url = reverse("core:health")
self.assertEqual(url, "/health/")
def test_detailed_health_check_keeps_global_throttling(self):
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(reverse("core:health"))
self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
def test_health_check_success(self):
"""Test health check returns healthy status"""
url = reverse("core:health")
@@ -228,6 +248,13 @@ class LivenessViewTest(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "alive")
def test_liveness_bypasses_global_throttling(self):
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(reverse("core:liveness"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "alive")
class ReadinessViewTest(APITestCase):
"""Tests for ReadinessView"""
@@ -245,6 +272,13 @@ class ReadinessViewTest(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "ready")
def test_readiness_bypasses_global_throttling(self):
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(reverse("core:readiness"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], "ready")
def test_readiness_returns_not_ready_on_db_error(self):
original_connection = core_views.connection
@@ -262,7 +296,8 @@ class ReadinessViewTest(APITestCase):
try:
core_views.connection = _BrokenConnection()
url = reverse("core:readiness")
response = self.client.get(url)
with patch.object(APIView, "throttle_classes", [_RejectAllThrottle]):
response = self.client.get(url)
finally:
core_views.connection = original_connection
@@ -270,6 +305,65 @@ class ReadinessViewTest(APITestCase):
self.assertEqual(response.data["status"], "not_ready")
class HealthProbeThrottlingTest(APITestCase):
def test_openapi_keeps_rate_limit_only_on_detailed_health(self):
generator = OpenAPISchemaGenerator(
info=openapi.Info(title="Health probes", default_version="v1"),
patterns=[
path("health/", HealthCheckView.as_view()),
path("health/live/", core_views.LivenessView.as_view()),
path("health/ready/", core_views.ReadinessView.as_view()),
],
)
schema = generator.get_schema(request=None, public=True)
responses = {
f"{schema.base_path.rstrip('/')}{url}": item["get"].responses
for url, item in schema.paths.items()
}
self.assertNotIn("429", responses["/health/live/"])
self.assertNotIn("429", responses["/health/ready/"])
self.assertIn("503", responses["/health/ready/"])
self.assertIn("429", responses["/health/"])
def test_probes_do_not_consume_or_obey_exhausted_anonymous_quota(self):
probe_cache = LocMemCache(self.id(), {})
probe_cache.clear()
self.addCleanup(probe_cache.clear)
class OnePerMinuteAnonThrottle(AnonRateThrottle):
rate = "1/min"
cache = probe_cache
with (
patch.object(APIView, "throttle_classes", [OnePerMinuteAnonThrottle]),
patch.object(
HealthCheckView, "_check_database", return_value={"status": "up"}
),
patch.object(
HealthCheckView, "_check_redis", return_value={"status": "up"}
),
):
for quota_exhausted in (False, True):
for _ in range(3):
for name, expected_status in (
("liveness", "alive"),
("readiness", "ready"),
):
with self.subTest(probe=name, quota_exhausted=quota_exhausted):
response = self.client.get(reverse(f"core:{name}"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data["status"], expected_status)
if not quota_exhausted:
response = self.client.get(reverse("core:health"))
self.assertEqual(response.status_code, status.HTTP_200_OK)
response = self.client.get(reverse("core:health"))
self.assertEqual(
response.status_code, status.HTTP_429_TOO_MANY_REQUESTS
)
class APIVersioningURLTest(APITestCase):
"""Tests for API versioning URL structure"""

View File

@@ -38,6 +38,10 @@ from rest_framework.test import APITestCase
from tests.apps.user.factories import UserFactory
def _source_records(groups, source_group):
return [record for group in groups for record in group[source_group]]
class OrganizationSourceRecordExportApiV2Test(APITestCase):
"""Checks admin-only source-record export contract."""
@@ -95,6 +99,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertEqual(generation.export_year, timezone.localdate().year)
self.assertIn('"artifacts_count": 25', command_output.getvalue())
def test_previous_json_generation_requires_rebuild(self):
build_source_record_export_artifacts()
manifest_path = Path(self.export_directory.name) / "current.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
self.assertEqual(manifest["version"], 3)
manifest["version"] = 2
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
load_current_source_record_export_generation()
def test_generation_contains_only_records_from_its_calendar_year(self):
export_year = 2026
generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC)
@@ -174,7 +189,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
if artifact.source_group == SourceGroup.PLANNED_INSPECTIONS.value
and artifact.file_format == "json"
)
inspection_rows = json.loads(inspection_path.read_text(encoding="utf-8"))
inspection_rows = _source_records(
json.loads(inspection_path.read_text(encoding="utf-8")),
SourceGroup.PLANNED_INSPECTIONS.value,
)
self.assertEqual(
{row["uid"] for row in inspection_rows},
{str(current_record.pk), str(dateless_current_record.pk)},
@@ -185,7 +203,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
for artifact in generation.artifacts
if artifact.source_group == SourceGroup.FINANCIAL_INDICATORS.value
)
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
financial_rows = _source_records(
json.loads(financial_path.read_text(encoding="utf-8")),
SourceGroup.FINANCIAL_INDICATORS.value,
)
self.assertEqual(
[row["uid"] for row in financial_rows],
[str(current_financial_record.pk)],
@@ -308,8 +329,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertIn("payload.risk.score", rows[0])
self.assertIn("payload.tags", rows[0])
financial_rows = json.loads(
archive.read("financial-indicators.json").decode("utf-8"),
financial_groups = json.loads(
archive.read("financial-indicators.json").decode("utf-8")
)
self.assertEqual(financial_groups[0]["okpo"], "12345678")
financial_rows = _source_records(
financial_groups, SourceGroup.FINANCIAL_INDICATORS.value
)
self.assertEqual(financial_rows[0]["Наименование"], organization.full_name)
self.assertEqual(
@@ -582,6 +607,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
for file_format in ("json", "csv", "xlsx"):
artifact_path = export_directory / f"artifact.{file_format}"
_render_source_group_artifact(
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
row_spool_path=spool_path,
output_path=artifact_path,
headers=headers,
@@ -623,18 +649,30 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertFalse(queryset.ordered)
self.assertFalse(queryset.query.default_ordering)
def test_json_artifact_reuses_valid_canonical_spool_without_copying_it(self):
def test_json_artifact_groups_nonadjacent_records_by_okpo(self):
organization = Organization.objects.create(
name='ООО "JSON без копии"',
inn="7707083813",
okpo="00123456",
)
other_organization = Organization.objects.create(
name='ООО "Вторая организация"',
inn="7707083814",
okpo="87654321",
)
extension = PlannedInspectionExtension.objects.create(
organization=organization,
title="Плановые проверки Генпрокуратуры России",
)
for index in range(2):
other_extension = PlannedInspectionExtension.objects.create(
organization=other_organization,
title="Плановые проверки Генпрокуратуры России",
)
for index, record_extension in enumerate(
(extension, other_extension, extension)
):
OrganizationSourceRecord.objects.create(
extension=extension,
extension=record_extension,
record_type="inspection",
source="inspections",
external_id=f"INSP-JSON-{index}",
@@ -652,6 +690,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
)
_render_source_group_artifact(
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
row_spool_path=spool_path,
output_path=artifact_path,
headers=headers,
@@ -659,9 +698,21 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
records_count=records_count,
)
self.assertEqual(records_count, 2)
self.assertEqual(len(json.loads(spool_path.read_text())), 2)
self.assertTrue(os.path.samefile(spool_path, artifact_path))
self.assertEqual(records_count, 3)
self.assertEqual(len(json.loads(spool_path.read_text())), 3)
groups = json.loads(artifact_path.read_text(encoding="utf-8"))
self.assertEqual(
{group["okpo"] for group in groups}, {"00123456", "87654321"}
)
grouped = {
group["okpo"]: group[SourceGroup.PLANNED_INSPECTIONS.value]
for group in groups
}
self.assertEqual(len(grouped["00123456"]), 2)
self.assertEqual(len(grouped["87654321"]), 1)
self.assertNotIn("ОКПО", grouped["00123456"][0])
self.assertFalse(os.path.samefile(spool_path, artifact_path))
self.assertFalse(list(Path(temporary_directory).glob("*.sqlite3")))
def test_generation_publishes_complete_matrix_and_keeps_previous_on_failure(self):
first_generation = build_source_record_export_artifacts()

View File

@@ -0,0 +1,26 @@
from pathlib import Path
import yaml
WORKFLOWS = Path(__file__).resolve().parents[1] / ".gitea" / "workflows"
def test_workflows_do_not_deploy_to_center():
assert not (WORKFLOWS / "deploy-customer-main.yml").exists()
for path in WORKFLOWS.glob("*.y*ml"):
workflow = yaml.safe_load(path.read_text())
assert "deploy_customer_main" not in workflow["jobs"]
content = path.read_text()
for forbidden in ("CUSTOMER_DEPLOY_", "10.0.10.174", "/ecos/release-service"):
assert forbidden not in content, (path.name, forbidden)
def test_ci_does_not_prune_shared_runner_resources():
for path in WORKFLOWS.glob("*.y*ml"):
content = path.read_text()
for command in (
"docker system prune",
"docker builder prune",
"docker buildx prune",
):
assert command not in content, (path.name, command)