From a91ed1f1ae7dec3ba695dcf58dce3da1c06c9200 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 4 Mar 2026 15:36:57 +0100 Subject: [PATCH] feat(registry): add new endpoints for registers, exchange, and backups; update routing and configurations --- .gitignore | 1 + .python-version | 2 +- .qoder/rules/main.md | 422 ------------ Makefile | 12 +- docker-compose.prod.yml | 2 +- docker-compose.service.yml | 1 - docker/postgres/init.sql | 2 +- scripts/run-tests-prod.sh | 17 + src/apps/backups/__init__.py | 2 + src/apps/backups/admin.py | 39 ++ src/apps/backups/apps.py | 13 + src/apps/backups/migrations/0001_initial.py | 44 ++ src/apps/backups/migrations/__init__.py | 0 src/apps/backups/models.py | 95 +++ src/apps/backups/serializers.py | 16 + src/apps/backups/services.py | 502 +++++++++++++++ src/apps/backups/tasks.py | 117 ++++ src/apps/backups/urls.py | 13 + src/apps/backups/views.py | 104 +++ src/apps/core/views.py | 10 +- src/apps/exchange/__init__.py | 1 + src/apps/exchange/admin.py | 25 + src/apps/exchange/apps.py | 9 + src/apps/exchange/migrations/0001_initial.py | 41 ++ src/apps/exchange/migrations/__init__.py | 0 src/apps/exchange/models.py | 43 ++ src/apps/exchange/serializers.py | 88 +++ src/apps/exchange/services.py | 447 +++++++++++++ src/apps/exchange/tasks.py | 67 ++ src/apps/exchange/urls.py | 13 + src/apps/exchange/views.py | 157 +++++ .../migrations/0009_auto_20260304_1159.py | 40 ++ .../0010_link_registry_organizations.py | 166 +++++ ...1_add_normalized_date_and_amount_fields.py | 91 +++ src/apps/registers/__init__.py | 1 + src/apps/registers/admin.py | 98 +++ src/apps/registers/apps.py | 9 + src/apps/registers/migrations/0001_initial.py | 64 ++ .../migrations/0002_auto_20260304_1038.py | 366 +++++++++++ ...003_add_unique_active_membership_period.py | 55 ++ src/apps/registers/migrations/__init__.py | 0 src/apps/registers/models.py | 208 ++++++ src/apps/registers/pagination.py | 29 + src/apps/registers/serializers.py | 122 ++++ src/apps/registers/services.py | 598 ++++++++++++++++++ src/apps/registers/urls.py | 28 + src/apps/registers/views.py | 369 +++++++++++ .../0005_create_default_admin_superuser.py | 52 ++ src/apps/user/models.py | 4 +- src/apps/user/serializers.py | 2 +- src/apps/user/views.py | 31 +- src/core/api_v1_urls.py | 12 + src/settings/base.py | 16 + src/settings/production.py | 47 +- src/settings/test_postgres.py | 31 + tests/apps/backups/__init__.py | 1 + tests/apps/backups/test_views.py | 168 +++++ tests/apps/core/test_admin.py | 4 +- tests/apps/core/test_background_jobs.py | 1 + tests/apps/core/test_cache.py | 4 +- tests/apps/core/test_exception_handler.py | 5 +- tests/apps/core/test_logging.py | 3 +- tests/apps/core/test_management_commands.py | 4 +- tests/apps/core/test_middleware.py | 6 +- tests/apps/core/test_mixins.py | 8 +- tests/apps/core/test_openapi.py | 3 +- tests/apps/core/test_signals.py | 8 +- tests/apps/core/test_tasks.py | 1 + tests/apps/core/test_views.py | 40 +- tests/apps/core/test_viewsets.py | 12 +- tests/apps/exchange/__init__.py | 0 tests/apps/exchange/factories.py | 23 + tests/apps/exchange/test_services.py | 25 + tests/apps/exchange/test_views.py | 122 ++++ tests/apps/parsers/test_admin.py | 23 +- tests/apps/parsers/test_checko.py | 81 +-- tests/apps/parsers/test_checko_parsers.py | 21 +- tests/apps/parsers/test_clients.py | 31 +- tests/apps/parsers/test_fns_upload.py | 8 +- tests/apps/parsers/test_services.py | 198 +++++- tests/apps/parsers/test_views.py | 11 +- tests/apps/registers/__init__.py | 0 tests/apps/registers/factories.py | 66 ++ tests/apps/registers/test_views.py | 416 ++++++++++++ tests/apps/user/test_admin.py | 14 +- tests/apps/user/test_serializers.py | 10 +- tests/apps/user/test_views.py | 12 +- tests/utils/__init__.py | 2 +- tests/utils/fixtures.py | 30 +- tests/utils/http_server.py | 5 +- 90 files changed, 5488 insertions(+), 622 deletions(-) delete mode 100644 .qoder/rules/main.md create mode 100755 scripts/run-tests-prod.sh create mode 100644 src/apps/backups/__init__.py create mode 100644 src/apps/backups/admin.py create mode 100644 src/apps/backups/apps.py create mode 100644 src/apps/backups/migrations/0001_initial.py create mode 100644 src/apps/backups/migrations/__init__.py create mode 100644 src/apps/backups/models.py create mode 100644 src/apps/backups/serializers.py create mode 100644 src/apps/backups/services.py create mode 100644 src/apps/backups/tasks.py create mode 100644 src/apps/backups/urls.py create mode 100644 src/apps/backups/views.py create mode 100644 src/apps/exchange/__init__.py create mode 100644 src/apps/exchange/admin.py create mode 100644 src/apps/exchange/apps.py create mode 100644 src/apps/exchange/migrations/0001_initial.py create mode 100644 src/apps/exchange/migrations/__init__.py create mode 100644 src/apps/exchange/models.py create mode 100644 src/apps/exchange/serializers.py create mode 100644 src/apps/exchange/services.py create mode 100644 src/apps/exchange/tasks.py create mode 100644 src/apps/exchange/urls.py create mode 100644 src/apps/exchange/views.py create mode 100644 src/apps/parsers/migrations/0009_auto_20260304_1159.py create mode 100644 src/apps/parsers/migrations/0010_link_registry_organizations.py create mode 100644 src/apps/parsers/migrations/0011_add_normalized_date_and_amount_fields.py create mode 100644 src/apps/registers/__init__.py create mode 100644 src/apps/registers/admin.py create mode 100644 src/apps/registers/apps.py create mode 100644 src/apps/registers/migrations/0001_initial.py create mode 100644 src/apps/registers/migrations/0002_auto_20260304_1038.py create mode 100644 src/apps/registers/migrations/0003_add_unique_active_membership_period.py create mode 100644 src/apps/registers/migrations/__init__.py create mode 100644 src/apps/registers/models.py create mode 100644 src/apps/registers/pagination.py create mode 100644 src/apps/registers/serializers.py create mode 100644 src/apps/registers/services.py create mode 100644 src/apps/registers/urls.py create mode 100644 src/apps/registers/views.py create mode 100644 src/apps/user/migrations/0005_create_default_admin_superuser.py create mode 100644 src/settings/test_postgres.py create mode 100644 tests/apps/backups/__init__.py create mode 100644 tests/apps/backups/test_views.py create mode 100644 tests/apps/exchange/__init__.py create mode 100644 tests/apps/exchange/factories.py create mode 100644 tests/apps/exchange/test_services.py create mode 100644 tests/apps/exchange/test_views.py create mode 100644 tests/apps/registers/__init__.py create mode 100644 tests/apps/registers/factories.py create mode 100644 tests/apps/registers/test_views.py diff --git a/.gitignore b/.gitignore index b35c28c..ebc84b9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ Thumbs.db data/ .zed/ .env.prod +tmp/ diff --git a/.python-version b/.python-version index c00391b..641602f 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11.14 \ No newline at end of file +3.11.14 diff --git a/.qoder/rules/main.md b/.qoder/rules/main.md deleted file mode 100644 index de8d1c1..0000000 --- a/.qoder/rules/main.md +++ /dev/null @@ -1,422 +0,0 @@ ---- -trigger: always_on ---- - -## 0) Язык и стиль общения (СТРОГО) -- ИИ-агент **ВСЕГДА отвечает на русском языке** -- Английский допускается ТОЛЬКО для: - - имён библиотек - - имён классов, функций, переменных - - CLI-команд -- Тон: инженерный, практичный, без воды и маркетинга - ---- - -## 1) Базовые принципы (НЕ ОБСУЖДАЮТСЯ) -Проект разрабатывается строго по принципам: - -- **SOLID** -- **KISS** -- **DRY** - -Правила приоритета: -- красиво vs просто → **простота** -- умно vs поддерживаемо → **поддерживаемость** -- магия vs явность → **явность** - ---- - -## 2) Контекст проекта -- ОС: **Astra Linux 1.8** -- Python: **3.11.2** -- Django: **3.x (указано 3.14)** -- Django REST Framework (DRF) -- Celery -- PostgreSQL **15.10** -- Apache **2.4.57** -- mod_wsgi **4.9.4** - -### Инструменты разработки -- **uv** -- **виртуальная среда** -- **pre-commit** -- **Gitea Actions (CI)** -- Тесты: `django test` -- Линтинг: `ruff` -- **ЗАПРЕЩЕНО** создавать тестовые скрипты для демонстрации, все должно быть исправлено в рамках проекта - ---- - -## 3) Окружение и команды (СТРОГО) -Все команды: -- выполняются **только внутри виртуальной среды** -- используют **uv** -- считаются выполняемыми из корня проекта - -### ❌ Запрещено -- `pip`, `python -m pip` -- `poetry`, `pipenv`, `pipx` -- системные команды вне venv - -### ✅ Разрешено -- `uv venv` -- `source .venv/bin/activate` -- `uv add / uv remove / uv sync` -- `uv run ` - -Пример: -```bash -uv run python manage.py test -``` - ---- - -## 4) Архитектура и слои ответственности (КРИТИЧНО) - -### 4.1 View (DRF) -View отвечает ТОЛЬКО за: -- приём HTTP-запроса -- проверку прав доступа -- работу с serializer -- вызов сервисного слоя - -❌ Запрещено: -- бизнес-логика -- сложные условия -- транзакции -- сложная работа с ORM - ---- - -### 4.2 Serializer -Serializer отвечает за: -- валидацию данных -- преобразование вход/выход - -Допускается: -- field-level validation -- object-level validation - -❌ Запрещено: -- бизнес-правила -- side-effects -- сложная логика в `save()` - ---- - -### 4.3 Сервисный слой (Business Logic) -- **ВСЯ бизнес-логика живёт здесь** -- Сервисы: - - не зависят от HTTP - - легко тестируются - - управляют транзакциями -- Сервис определяет *что* делать, а не *как* отдать ответ - -Рекомендуемый паттерн: -```python -class EntityService: - @classmethod - def do_something(cls, *, data): - ... -``` - ---- - -### 4.4 Модели (ORM) -Модели должны быть: -- простыми -- декларативными - -Допускается: -- `__str__` -- простые computed properties -- минимальные helper-методы - -❌ Запрещено: -- бизнес-логика -- workflow -- сигналы как логика -- условия, зависящие от сценариев - -👉 **Любые исключения — ТОЛЬКО после обсуждения в чате.** - ---- - -## 5) Celery -- Task = **thin wrapper** -- Task вызывает сервис, а не содержит логику -- Таски: - - идемпотентны - - логируют начало и завершение -- Ретраи: - - только для временных ошибок - - с backoff - ---- - -## 6) База данных и миграции -- Любое изменение моделей → миграции обязательны -- Миграции: - - детерминированные - - без ручной магии без причины - -Проверка перед коммитом: -```bash -uv run python manage.py makemigrations --check --dry-run -``` - -PostgreSQL: -- транзакции использовать осознанно -- `select_for_update()` при гонках -- Raw SQL — только с объяснением - ---- - -## 7) Тестирование -- Любая бизнес-логика → тесты -- В первую очередь тестируется сервисный слой -- API — happy path + edge cases - -Запуск: -```bash -uv run python manage.py test -``` - ---- - -## 8) pre-commit (обязателен) -- Любой код обязан проходить pre-commit -- Агент обязан учитывать проверки форматирования и линтинга - -```bash -pre-commit run --all-files -``` - ---- - -## 9) CI (Gitea Actions) -- Используется **Gitea Actions** -- ❌ GitHub Actions запрещены -- Любые изменения: - - не должны ломать CI -- Если меняются: - - зависимости - - команды тестов - - миграции - → агент обязан указать необходимость правок workflow - ---- - -## 10) Apache + mod_wsgi -- Используется **ТОЛЬКО WSGI** -- ASGI запрещён без отдельного обсуждения -- Любые изменения в `wsgi.py`, путях, статике: - - сопровождаются пояснением - - требуют перезапуска Apache - -```bash -systemctl restart apache2 -``` - -Учитывать ограничения и права Astra Linux. - ---- - -## 11) Работа с репозиторием -- Минимальный diff — приоритет -- ❌ Не коммитить: - - `.venv` - - артефакты - - дампы БД -- Массовый рефакторинг — только по явному запросу - ---- - -## 12) Anti-patterns (ЗАПРЕЩЕНО) -- Fat Models -- God Views -- Бизнес-логика в Serializers -- Сигналы как workflow -- Магия в `save()` -- Прямые импорты моделей между apps -- Сложная логика в queryset как бизнес-правило - ---- - -## 13) Формат ответа ИИ-агента (ОБЯЗАТЕЛЬНЫЙ) -Каждый ответ должен содержать: - -1. **Что меняем** -2. **Файлы / патч** -3. **Команды (через uv)** -4. **Проверки (tests / pre-commit / CI)** -5. **Риски / замечания** - ---- - -## 14) Исключения -- ИИ-агент **НЕ внедряет исключения сам** -- Агент: - - описывает стандартное решение - - объясняет, почему оно не подходит - - запрашивает разрешение в чате - -## 15) Структура проекта и миксины (ОБЯЗАТЕЛЬНО) - -### 15.0 Правило Core-First (КРИТИЧНО) -**ПЕРЕД созданием любого нового компонента** агент ОБЯЗАН проверить модуль `apps.core`: - -``` -src/apps/core/ -├── mixins.py # Model mixins (TimestampMixin, SoftDeleteMixin, etc.) -├── services.py # BaseService, BackgroundJobService -├── views.py # Health checks, BackgroundJob API -├── viewsets.py # BaseViewSet, ReadOnlyViewSet -├── exceptions.py # APIError, NotFoundError, ValidationError -├── permissions.py # IsOwner, IsAdminOrReadOnly, etc. -├── pagination.py # CursorPagination -├── filters.py # BaseFilterSet -├── cache.py # cache_result, invalidate_cache -├── tasks.py # BaseTask для Celery -├── logging.py # StructuredLogger -├── middleware.py # RequestIDMiddleware -├── signals.py # SignalDispatcher -├── responses.py # APIResponse wrapper -├── openapi.py # api_docs decorator -└── management/commands/base.py # BaseAppCommand -``` - -**Порядок действий:** -1. Проверить `apps.core` на наличие нужного базового класса/миксина -2. Наследоваться от существующего, а не создавать с нуля -3. Если нужного нет — обсудить добавление в core - -❌ **ЗАПРЕЩЕНО:** создавать дублирующую функциональность в app-модулях - ---- - -### 15.1 Model Mixins -При создании моделей **ОБЯЗАТЕЛЬНО** использовать миксины из `apps.core.mixins`: - -| Миксин | Когда использовать | Поля | -|--------|-------------------|------| -| `TimestampMixin` | **ВСЕГДА** для любой модели | `created_at`, `updated_at` | -| `UUIDPrimaryKeyMixin` | Когда нужен UUID вместо int ID | `id` (UUID) | -| `SoftDeleteMixin` | Когда нельзя физически удалять | `is_deleted`, `deleted_at` | -| `AuditMixin` | Когда нужно знать кто создал/изменил | `created_by`, `updated_by` | -| `OrderableMixin` | Для сортируемых списков | `order` | -| `StatusMixin` | Для моделей со статусами | `status` | -| `SlugMixin` | Для URL-friendly идентификаторов | `slug` | - -**Пример правильного использования:** -```python -from apps.core.mixins import TimestampMixin, SoftDeleteMixin, AuditMixin - -class Document(TimestampMixin, SoftDeleteMixin, AuditMixin, models.Model): - """Документ с историей и мягким удалением.""" - title = models.CharField(max_length=200) - - class Meta: - ordering = ['-created_at'] -``` - -**Порядок наследования миксинов:** -1. `UUIDPrimaryKeyMixin` (если нужен) -2. `TimestampMixin` -3. `SoftDeleteMixin` (если нужен) -4. `AuditMixin` (если нужен) -5. `OrderableMixin` / `StatusMixin` / `SlugMixin` -6. `models.Model` (последним) - ---- - -### 15.2 Management Commands -Все management commands наследуются от `BaseAppCommand`: - -```python -from apps.core.management.commands.base import BaseAppCommand - -class Command(BaseAppCommand): - help = 'Описание команды' - use_transaction = True # Обернуть в транзакцию - - def add_arguments(self, parser): - super().add_arguments(parser) # Добавляет --dry-run, --silent - parser.add_argument('--my-arg', type=str) - - def execute_command(self, *args, **options): - items = MyModel.objects.all() - - for item in self.progress_iter(items, desc="Обработка"): - if not self.dry_run: - self.process(item) - - return "Обработано успешно" -``` - -**Возможности BaseAppCommand:** -- `--dry-run` — тестовый запуск без изменений -- `--silent` — минимальный вывод -- `self.progress_iter()` — прогресс-бар -- `self.timed_operation()` — измерение времени -- `self.confirm()` — подтверждение -- `self.log_info/success/warning/error()` — логирование - ---- - -### 15.3 Background Jobs (Celery) -Для отслеживания статуса фоновых задач использовать `BackgroundJob`: - -```python -# В сервисе при запуске задачи -from apps.core.services import BackgroundJobService - -job = BackgroundJobService.create_job( - task_id=task.id, - task_name="apps.myapp.tasks.process_data", - user_id=request.user.id, -) - -# В Celery таске -from apps.core.models import BackgroundJob - -@shared_task(bind=True) -def my_task(self, data): - job = BackgroundJob.objects.get(task_id=self.request.id) - job.mark_started() - - for i, item in enumerate(items): - process(item) - job.update_progress(i * 100 // len(items), "Обработка...") - - job.complete(result={"processed": len(items)}) -``` - -**API эндпоинты:** -- `GET /api/v1/jobs/` — список задач пользователя -- `GET /api/v1/jobs/{task_id}/` — статус конкретной задачи - ---- - -### 15.4 Factories (тестирование) -Все фабрики используют `factory_boy` + `faker`: - -```python -import factory -from faker import Faker - -fake = Faker("ru_RU") - -class MyModelFactory(factory.django.DjangoModelFactory): - class Meta: - model = MyModel - - name = factory.LazyAttribute(lambda _: fake.word()) - email = factory.LazyAttribute(lambda _: fake.unique.email()) -``` - -**Правила:** -- Никакого хардкода в тестах (`"test@example.com"` → `fake.email()`) -- Использовать `fake.unique.*` для уникальных полей -- Локаль: `Faker("ru_RU")` для русских данных - diff --git a/Makefile b/Makefile index 0b92c3f..b5caa8f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install setup-dev dev-up dev-down prod-up prod-down logs test test-cov test-fast test-parallel test-failfast lint format type-check security-check pre-commit pre-push migrate createsuperuser shell clean +.PHONY: help install setup-dev dev-up dev-down prod-up prod-down logs test test-prod test-cov test-fast test-parallel test-failfast lint format type-check security-check pre-commit pre-push migrate createsuperuser shell clean COMPOSE_DEV = docker compose -f docker-compose.dev.yml COMPOSE_PROD = docker compose -f docker-compose.prod.yml --env-file .env.prod @@ -13,6 +13,7 @@ help: @echo " make prod-down - Stop prod compose" @echo " make logs - Follow logs from dev compose" @echo " make test - Run tests (use TARGET=... to filter)" + @echo " make test-prod - Run production-like tests (PostgreSQL + migrations)" @echo " make pre-commit - Run all pre-commit checks" @echo " make pre-push - Run pre-push checks (tests)" @echo " make migrate - Run Django migrations" @@ -58,6 +59,15 @@ test: ./scripts/run-tests.sh; \ fi +test-prod: + @if [ "$(TARGET)" ]; then \ + echo "Running production-like tests: $(TARGET)"; \ + ./scripts/run-tests-prod.sh "$(TARGET)"; \ + else \ + echo "Running full production-like test suite"; \ + ./scripts/run-tests-prod.sh; \ + fi + test-cov: ./scripts/run-tests.sh ../tests --cov=src --cov-report=term-missing --cov-report=xml --cov-report=html diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b2189c9..73e2e4b 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -26,7 +26,7 @@ services: web: build: *web-build - image: ${WEB_IMAGE:-mostovik/web:latest} + image: http://10.10.0.10:3000/v2/avm/mostovik-backend:dev container_name: mostovik_web restart: unless-stopped env_file: diff --git a/docker-compose.service.yml b/docker-compose.service.yml index ab92f4c..801815e 100644 --- a/docker-compose.service.yml +++ b/docker-compose.service.yml @@ -31,4 +31,3 @@ services: interval: 30s timeout: 10s retries: 3 - diff --git a/docker/postgres/init.sql b/docker/postgres/init.sql index 48e8a44..6ce5e2c 100644 --- a/docker/postgres/init.sql +++ b/docker/postgres/init.sql @@ -15,4 +15,4 @@ ALTER SYSTEM SET effective_cache_size = '1GB'; ALTER SYSTEM SET maintenance_work_mem = '64MB'; ALTER SYSTEM SET checkpoint_completion_target = 0.9; ALTER SYSTEM SET wal_buffers = '16MB'; -ALTER SYSTEM SET default_statistics_target = 100; \ No newline at end of file +ALTER SYSTEM SET default_statistics_target = 100; diff --git a/scripts/run-tests-prod.sh b/scripts/run-tests-prod.sh new file mode 100755 index 0000000..44abba0 --- /dev/null +++ b/scripts/run-tests-prod.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Production-like Django test runner (PostgreSQL + migrations) + +set -euo pipefail + +cd "$(dirname "$0")/../src" || exit 1 + +export PYTHONPATH=. +export DJANGO_SETTINGS_MODULE=settings.test_postgres + +COMMON_ADDOPTS="--verbose --tb=short --reuse-db --strict-markers --strict-config --color=yes" + +if [ "$#" -eq 0 ]; then + uv run pytest ../tests --migrations -o "addopts=${COMMON_ADDOPTS}" +else + uv run pytest "$@" --migrations -o "addopts=${COMMON_ADDOPTS}" +fi diff --git a/src/apps/backups/__init__.py b/src/apps/backups/__init__.py new file mode 100644 index 0000000..a1f4418 --- /dev/null +++ b/src/apps/backups/__init__.py @@ -0,0 +1,2 @@ +"""Приложение экспорта защищённых резервных архивов.""" + diff --git a/src/apps/backups/admin.py b/src/apps/backups/admin.py new file mode 100644 index 0000000..1e19498 --- /dev/null +++ b/src/apps/backups/admin.py @@ -0,0 +1,39 @@ +"""Admin для приложения backups.""" + +from apps.backups.models import BackupExportJob +from django.contrib import admin + + +@admin.register(BackupExportJob) +class BackupExportJobAdmin(admin.ModelAdmin): + """Admin для задач backup-экспорта.""" + + list_display = [ + "id", + "actual_date", + "status", + "task_id", + "requested_by", + "organizations_count", + "archive_size", + "started_at", + "completed_at", + ] + list_filter = ["status", "actual_date", "created_at", "completed_at"] + search_fields = ["task_id", "checksum_sha256", "archive_filename"] + readonly_fields = [ + "task_id", + "archive_path", + "archive_filename", + "checksum_filename", + "checksum_sha256", + "archive_size", + "organizations_count", + "error", + "started_at", + "completed_at", + "created_at", + "updated_at", + ] + ordering = ["-actual_date", "-created_at"] + diff --git a/src/apps/backups/apps.py b/src/apps/backups/apps.py new file mode 100644 index 0000000..fb4ae92 --- /dev/null +++ b/src/apps/backups/apps.py @@ -0,0 +1,13 @@ +"""Конфигурация приложения backups.""" + +from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ + + +class BackupsConfig(AppConfig): + """AppConfig приложения backup-экспорта.""" + + default_auto_field = "django.db.models.BigAutoField" + name = "apps.backups" + verbose_name = _("Резервные копии") + diff --git a/src/apps/backups/migrations/0001_initial.py b/src/apps/backups/migrations/0001_initial.py new file mode 100644 index 0000000..2f73236 --- /dev/null +++ b/src/apps/backups/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 3.2.25 on 2026-03-04 12:34 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='BackupExportJob', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')), + ('actual_date', models.DateField(db_index=True, unique=True, verbose_name='дата актуальности')), + ('status', models.CharField(choices=[('pending', 'Ожидает'), ('started', 'Выполняется'), ('success', 'Успешно'), ('failure', 'Ошибка')], db_index=True, default='pending', max_length=20, verbose_name='статус')), + ('task_id', models.CharField(blank=True, db_index=True, max_length=255, verbose_name='ID задачи Celery')), + ('archive_path', models.TextField(blank=True, verbose_name='путь к архиву')), + ('archive_filename', models.CharField(blank=True, max_length=255, verbose_name='имя архива')), + ('checksum_filename', models.CharField(blank=True, max_length=255, verbose_name='имя файла контрольной суммы')), + ('checksum_sha256', models.CharField(blank=True, max_length=64, verbose_name='контрольная сумма SHA256')), + ('archive_size', models.PositiveIntegerField(blank=True, null=True, verbose_name='размер архива')), + ('organizations_count', models.PositiveIntegerField(default=0, verbose_name='количество организаций')), + ('error', models.TextField(blank=True, verbose_name='ошибка')), + ('started_at', models.DateTimeField(blank=True, null=True, verbose_name='время начала')), + ('completed_at', models.DateTimeField(blank=True, null=True, verbose_name='время завершения')), + ('requested_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='backup_export_jobs', to=settings.AUTH_USER_MODEL, verbose_name='запрошено пользователем')), + ], + options={ + 'verbose_name': 'задача backup-экспорта', + 'verbose_name_plural': 'задачи backup-экспорта', + 'db_table': 'backups_export_job', + 'ordering': ['-actual_date', '-created_at'], + }, + ), + ] diff --git a/src/apps/backups/migrations/__init__.py b/src/apps/backups/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/apps/backups/models.py b/src/apps/backups/models.py new file mode 100644 index 0000000..4d8d649 --- /dev/null +++ b/src/apps/backups/models.py @@ -0,0 +1,95 @@ +"""Модели приложения backups.""" + +from apps.core.mixins import TimestampMixin +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class BackupExportJob(TimestampMixin, models.Model): + """Задача формирования экспортного backup-архива на конкретную дату.""" + + class Status(models.TextChoices): + PENDING = "pending", _("Ожидает") + STARTED = "started", _("Выполняется") + SUCCESS = "success", _("Успешно") + FAILURE = "failure", _("Ошибка") + + actual_date = models.DateField( + _("дата актуальности"), + unique=True, + db_index=True, + ) + status = models.CharField( + _("статус"), + max_length=20, + choices=Status.choices, + default=Status.PENDING, + db_index=True, + ) + task_id = models.CharField( + _("ID задачи Celery"), + max_length=255, + blank=True, + db_index=True, + ) + requested_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="backup_export_jobs", + verbose_name=_("запрошено пользователем"), + ) + archive_path = models.TextField( + _("путь к архиву"), + blank=True, + ) + archive_filename = models.CharField( + _("имя архива"), + max_length=255, + blank=True, + ) + checksum_filename = models.CharField( + _("имя файла контрольной суммы"), + max_length=255, + blank=True, + ) + checksum_sha256 = models.CharField( + _("контрольная сумма SHA256"), + max_length=64, + blank=True, + ) + archive_size = models.PositiveIntegerField( + _("размер архива"), + null=True, + blank=True, + ) + organizations_count = models.PositiveIntegerField( + _("количество организаций"), + default=0, + ) + error = models.TextField( + _("ошибка"), + blank=True, + ) + started_at = models.DateTimeField( + _("время начала"), + null=True, + blank=True, + ) + completed_at = models.DateTimeField( + _("время завершения"), + null=True, + blank=True, + ) + + class Meta: + db_table = "backups_export_job" + verbose_name = _("задача backup-экспорта") + verbose_name_plural = _("задачи backup-экспорта") + ordering = ["-actual_date", "-created_at"] + + def __str__(self) -> str: + return f"Backup {self.actual_date} [{self.status}]" + diff --git a/src/apps/backups/serializers.py b/src/apps/backups/serializers.py new file mode 100644 index 0000000..fa259fe --- /dev/null +++ b/src/apps/backups/serializers.py @@ -0,0 +1,16 @@ +"""Сериализаторы API экспорта резервных архивов.""" + +from rest_framework import serializers + + +class BackupExportRequestSerializer(serializers.Serializer): + """Параметры экспорта защищённого backup архива.""" + + actual_date = serializers.DateField( + required=False, + help_text=( + "Дата актуальности организаций из реестров. " + "Если не передана, используется текущая дата." + ), + ) + diff --git a/src/apps/backups/services.py b/src/apps/backups/services.py new file mode 100644 index 0000000..d4f9c53 --- /dev/null +++ b/src/apps/backups/services.py @@ -0,0 +1,502 @@ +"""Сервисы создания защищённых backup-архивов.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import struct +import zlib +from collections.abc import Iterable +from contextlib import suppress +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from io import BytesIO +from pathlib import Path +from uuid import UUID +from zipfile import ZIP_DEFLATED, ZipFile + +from apps.backups.models import BackupExportJob +from apps.parsers.models import ( + FinancialReport, + FinancialReportLine, + IndustrialCertificateRecord, + InspectionRecord, + ManufacturerRecord, + ProcurementRecord, +) +from apps.registers.models import ( + Organization, + Register, + RegisterUpload, + RegistryMembershipPeriod, +) +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from django.conf import settings +from django.db import IntegrityError, transaction +from django.db.models import Model, Q +from django.utils import timezone + + +class BackupExportError(ValueError): + """Ошибка формирования backup-архива.""" + + +@dataclass(frozen=True) +class BackupArtifact: + """Итоговый артефакт backup-экспорта.""" + + archive_bytes: bytes + archive_filename: str + bin_filename: str + checksum_filename: str + checksum_sha256: str + organizations_count: int + actual_date: date + + +@dataclass(frozen=True) +class BackupRequestResult: + """Результат обработки запроса на экспорт backup.""" + + action: str + message: str + actual_date: date + task_id: str + + +class BackupExportService: + """Сервис экспорта актуальных организаций и связанных данных в bin-архив.""" + + MAGIC = b"MSBK" + BIN_FORMAT_VERSION = 1 + AAD = b"mostovik-backup-v1" + + @classmethod + def build_backup_archive(cls, *, actual_date: date | None = None) -> BackupArtifact: + snapshot_date = actual_date or timezone.localdate() + active_org_ids = cls._get_active_organization_ids(snapshot_date) + if not active_org_ids: + raise BackupExportError("Нет актуальных организаций для экспорта") + + export_payload = cls._build_export_payload( + actual_date=snapshot_date, + active_org_ids=active_org_ids, + ) + payload_bytes = cls._serialize_payload(export_payload) + compressed_payload = zlib.compress(payload_bytes, level=9) + + encrypted_payload, crypto_header = cls._encrypt_payload(compressed_payload) + bin_bytes = cls._build_bin_container( + encrypted_payload=encrypted_payload, + header_payload=crypto_header + | { + "actual_date": snapshot_date.isoformat(), + "organizations_count": len(active_org_ids), + "plaintext_sha256": hashlib.sha256(payload_bytes).hexdigest(), + "compressed_sha256": hashlib.sha256(compressed_payload).hexdigest(), + "ciphertext_sha256": hashlib.sha256(encrypted_payload).hexdigest(), + }, + ) + + checksum_sha256 = hashlib.sha256(bin_bytes).hexdigest() + timestamp = timezone.now().strftime("%Y%m%d_%H%M%S") + bin_filename = f"mostovik_backup_{timestamp}.bin" + checksum_filename = f"{bin_filename}.sha256" + archive_filename = f"mostovik_backup_{timestamp}.zip" + archive_bytes = cls._build_zip_archive( + bin_filename=bin_filename, + bin_bytes=bin_bytes, + checksum_filename=checksum_filename, + checksum_sha256=checksum_sha256, + ) + + return BackupArtifact( + archive_bytes=archive_bytes, + archive_filename=archive_filename, + bin_filename=bin_filename, + checksum_filename=checksum_filename, + checksum_sha256=checksum_sha256, + organizations_count=len(active_org_ids), + actual_date=snapshot_date, + ) + + @classmethod + def _get_active_organization_ids(cls, actual_date: date) -> list[int]: + return list( + RegistryMembershipPeriod.objects.filter(started_at__lte=actual_date) + .filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date)) + .values_list("organization_id", flat=True) + .distinct() + ) + + @classmethod + def _build_export_payload( + cls, + *, + actual_date: date, + active_org_ids: list[int], + ) -> dict: + active_periods = RegistryMembershipPeriod.objects.filter( + organization_id__in=active_org_ids, + started_at__lte=actual_date, + ).filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date)) + + register_ids = list(active_periods.values_list("registry_id", flat=True).distinct()) + upload_ids = list( + active_periods.values_list("started_by_upload_id", flat=True).distinct() + ) + + reports_qs = FinancialReport.objects.filter( + registry_organization_id__in=active_org_ids + ) + report_ids = list(reports_qs.values_list("id", flat=True)) + + export_map: dict[type[Model], Iterable] = { + Organization: Organization.objects.filter(id__in=active_org_ids).order_by("id"), + Register: Register.objects.filter(id__in=register_ids).order_by("name"), + RegisterUpload: RegisterUpload.objects.filter(id__in=upload_ids).order_by("id"), + RegistryMembershipPeriod: active_periods.order_by( + "registry_id", + "organization_id", + "started_at", + ), + IndustrialCertificateRecord: IndustrialCertificateRecord.objects.filter( + registry_organization_id__in=active_org_ids + ).order_by("id"), + ManufacturerRecord: ManufacturerRecord.objects.filter( + registry_organization_id__in=active_org_ids + ).order_by("id"), + InspectionRecord: InspectionRecord.objects.filter( + registry_organization_id__in=active_org_ids + ).order_by("id"), + ProcurementRecord: ProcurementRecord.objects.filter( + registry_organization_id__in=active_org_ids + ).order_by("id"), + FinancialReport: reports_qs.order_by("id"), + FinancialReportLine: FinancialReportLine.objects.filter( + report_id__in=report_ids + ).order_by("id"), + } + + schema: dict[str, dict] = {} + data: dict[str, list[dict]] = {} + for model, queryset in export_map.items(): + model_label = model._meta.label + schema[model_label] = cls._build_model_schema(model) + data[model_label] = cls._serialize_queryset(model=model, queryset=queryset) + + return { + "format": "mostovik-backup-payload", + "version": cls.BIN_FORMAT_VERSION, + "generated_at": timezone.now().isoformat(), + "actual_date": actual_date.isoformat(), + "organizations_count": len(active_org_ids), + "schema": schema, + "data": data, + } + + @classmethod + def _build_model_schema(cls, model: type[Model]) -> dict: + fields = [] + for field in model._meta.local_fields: + field_meta = { + "name": field.name, + "attname": field.attname, + "column": field.column, + "type": field.get_internal_type(), + "null": field.null, + "blank": field.blank, + "primary_key": field.primary_key, + "unique": field.unique, + "db_index": field.db_index, + "is_relation": field.is_relation, + } + + if field.is_relation and field.related_model is not None: + on_delete_handler = getattr(field.remote_field.on_delete, "__name__", "unknown") + field_meta["related_model"] = field.related_model._meta.label + field_meta["on_delete"] = on_delete_handler + + fields.append(field_meta) + + indexes = [ + {"name": index.name, "fields": list(index.fields)} + for index in model._meta.indexes + ] + + constraints = [] + for constraint in model._meta.constraints: + entry = { + "name": constraint.name, + "type": constraint.__class__.__name__, + } + if hasattr(constraint, "fields"): + entry["fields"] = list(constraint.fields) + if getattr(constraint, "condition", None) is not None: + entry["condition"] = str(constraint.condition) + constraints.append(entry) + + return { + "app_label": model._meta.app_label, + "model_name": model._meta.model_name, + "db_table": model._meta.db_table, + "fields": fields, + "indexes": indexes, + "constraints": constraints, + } + + @classmethod + def _serialize_queryset(cls, *, model: type[Model], queryset: Iterable) -> list[dict]: + field_names = [field.attname for field in model._meta.local_fields] + serialized = [] + for row in queryset.values(*field_names).iterator(chunk_size=1000): + serialized.append( + { + key: cls._normalize_value(value) + for key, value in row.items() + } + ) + return serialized + + @classmethod + def _normalize_value(cls, value): + if isinstance(value, datetime | date): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, UUID): + return str(value) + if isinstance(value, bytes): + return { + "__type__": "bytes", + "base64": base64.b64encode(value).decode("ascii"), + } + return value + + @classmethod + def _serialize_payload(cls, payload: dict) -> bytes: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + @classmethod + def _encrypt_payload(cls, payload_bytes: bytes) -> tuple[bytes, dict]: + raw_key = cls._read_encryption_key() + nonce = os.urandom(12) + encrypted_payload = AESGCM(raw_key).encrypt(nonce, payload_bytes, cls.AAD) + + return encrypted_payload, { + "algorithm": "AES-256-GCM", + "key_id": getattr(settings, "BACKUP_KEY_ID", "default"), + "nonce": base64.urlsafe_b64encode(nonce).decode("ascii"), + "aad": base64.urlsafe_b64encode(cls.AAD).decode("ascii"), + } + + @classmethod + def _read_encryption_key(cls) -> bytes: + key_raw = getattr(settings, "BACKUP_ENCRYPTION_KEY", "") + if not key_raw: + raise BackupExportError("Не задан BACKUP_ENCRYPTION_KEY в настройках") + + try: + normalized_key = key_raw + ("=" * (-len(key_raw) % 4)) + decoded_key = base64.urlsafe_b64decode(normalized_key) + except Exception as exc: # noqa: BLE001 + raise BackupExportError( + "BACKUP_ENCRYPTION_KEY должен быть base64-url кодированным ключом" + ) from exc + + if len(decoded_key) != 32: + raise BackupExportError( + "BACKUP_ENCRYPTION_KEY после декодирования должен быть 32 байта" + ) + + return decoded_key + + @classmethod + def _build_bin_container(cls, *, encrypted_payload: bytes, header_payload: dict) -> bytes: + header = { + "format": "mostovik-backup-bin", + "version": cls.BIN_FORMAT_VERSION, + "generated_at": timezone.now().isoformat(), + } | header_payload + header_bytes = json.dumps( + header, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + if len(header_bytes) > 2**32 - 1: + raise BackupExportError("Заголовок backup контейнера слишком большой") + + return ( + cls.MAGIC + + bytes([cls.BIN_FORMAT_VERSION]) + + struct.pack(">I", len(header_bytes)) + + header_bytes + + encrypted_payload + ) + + @classmethod + def _build_zip_archive( + cls, + *, + bin_filename: str, + bin_bytes: bytes, + checksum_filename: str, + checksum_sha256: str, + ) -> bytes: + checksum_content = f"{checksum_sha256} {bin_filename}\n".encode() + + stream = BytesIO() + with ZipFile(stream, mode="w", compression=ZIP_DEFLATED) as archive: + archive.writestr(bin_filename, bin_bytes) + archive.writestr(checksum_filename, checksum_content) + + return stream.getvalue() + + +class BackupExportJobService: + """Сервис оркестрации асинхронного формирования и отдачи backup-архивов.""" + + @classmethod + @transaction.atomic + def check_or_start_job( + cls, + *, + actual_date: date, + requested_by_id: int | None, + ) -> BackupRequestResult: + job = cls._get_job_for_update(actual_date) + existing_job_result = cls._result_for_existing_job(actual_date=actual_date, job=job) + if existing_job_result is not None: + return existing_job_result + + if job: + cls._cleanup_job_artifact(job) + job.delete() + + try: + new_job = BackupExportJob.objects.create( + actual_date=actual_date, + requested_by_id=requested_by_id, + status=BackupExportJob.Status.PENDING, + ) + except IntegrityError: + concurrent_job = cls._get_job_for_update(actual_date) + concurrent_job_result = cls._result_for_existing_job( + actual_date=actual_date, + job=concurrent_job, + ) + if concurrent_job_result is not None: + return concurrent_job_result + + if concurrent_job: + cls._cleanup_job_artifact(concurrent_job) + concurrent_job.delete() + + new_job = BackupExportJob.objects.create( + actual_date=actual_date, + requested_by_id=requested_by_id, + status=BackupExportJob.Status.PENDING, + ) + + from apps.backups.tasks import generate_backup_for_date + + task = generate_backup_for_date.delay(job_id=new_job.id) + new_job.task_id = task.id or "" + new_job.save(update_fields=["task_id", "updated_at"]) + + return BackupRequestResult( + action="started", + message="Формирование бэкапа запущено.", + actual_date=actual_date, + task_id=new_job.task_id, + ) + + @classmethod + def _result_for_existing_job( + cls, + *, + actual_date: date, + job: BackupExportJob | None, + ) -> BackupRequestResult | None: + if job is None: + return None + + if job.status in (BackupExportJob.Status.PENDING, BackupExportJob.Status.STARTED): + return BackupRequestResult( + action="wait", + message="Бэкап формируется, пожалуйста подождите.", + actual_date=actual_date, + task_id=job.task_id, + ) + + if job.status == BackupExportJob.Status.SUCCESS and cls._archive_exists(job): + return BackupRequestResult( + action="download", + message="Бэкап готов к скачиванию.", + actual_date=actual_date, + task_id=job.task_id, + ) + + return None + + @classmethod + @transaction.atomic + def consume_ready_archive(cls, *, actual_date: date) -> BackupArtifact: + job = cls._get_job_for_update(actual_date) + if not job: + raise BackupExportError("Задача бэкапа не найдена") + + if job.status != BackupExportJob.Status.SUCCESS: + raise BackupExportError("Бэкап еще не готов") + + if not cls._archive_exists(job): + job.delete() + raise BackupExportError("Файл бэкапа отсутствует, запустите формирование снова") + + archive_path = Path(job.archive_path) + archive_bytes = archive_path.read_bytes() + archive_filename = job.archive_filename or archive_path.name + + with suppress(Exception): + archive_path.unlink(missing_ok=True) + + artifact = BackupArtifact( + archive_bytes=archive_bytes, + archive_filename=archive_filename, + bin_filename="", + checksum_filename=job.checksum_filename, + checksum_sha256=job.checksum_sha256 or hashlib.sha256(archive_bytes).hexdigest(), + organizations_count=job.organizations_count, + actual_date=job.actual_date, + ) + job.delete() + return artifact + + @classmethod + def _get_job_for_update(cls, actual_date: date) -> BackupExportJob | None: + return ( + BackupExportJob.objects.select_for_update() + .filter(actual_date=actual_date) + .first() + ) + + @classmethod + def _archive_exists(cls, job: BackupExportJob) -> bool: + return bool(job.archive_path) and Path(job.archive_path).is_file() + + @classmethod + def _cleanup_job_artifact(cls, job: BackupExportJob) -> None: + if not job.archive_path: + return + with suppress(Exception): + Path(job.archive_path).unlink(missing_ok=True) diff --git a/src/apps/backups/tasks.py b/src/apps/backups/tasks.py new file mode 100644 index 0000000..c8e057e --- /dev/null +++ b/src/apps/backups/tasks.py @@ -0,0 +1,117 @@ +"""Celery-задачи приложения backups.""" + +from __future__ import annotations + +import logging +import uuid +from pathlib import Path + +from apps.backups.models import BackupExportJob +from apps.backups.services import BackupExportService +from apps.core.services import BackgroundJobService +from celery import shared_task +from django.conf import settings +from django.utils import timezone + +logger = logging.getLogger(__name__) + + +def _resolve_backup_target_path(file_name: str) -> Path: + target_dir = Path(getattr(settings, "BACKUP_EXPORT_DIRECTORY", settings.MEDIA_ROOT)) + target_dir.mkdir(parents=True, exist_ok=True) + + target = target_dir / file_name + if target.exists(): + suffix = uuid.uuid4().hex[:8] + target = target_dir / f"{target.stem}_{suffix}{target.suffix}" + return target + + +@shared_task(bind=True) +def generate_backup_for_date(self, job_id: int) -> dict: + """Сформировать backup-архив по записи BackupExportJob.""" + task_id = self.request.id or str(uuid.uuid4()) + job = BackupExportJob.objects.filter(id=job_id).first() + if not job: + return {"status": "skipped", "reason": "job_not_found"} + + background_job = BackgroundJobService.get_by_task_id_or_none(task_id) + if not background_job: + background_job = BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.backups.tasks.generate_backup_for_date", + user_id=job.requested_by_id, + meta={"actual_date": job.actual_date.isoformat()}, + ) + + job.task_id = task_id + job.status = BackupExportJob.Status.STARTED + job.error = "" + job.started_at = timezone.now() + job.completed_at = None + job.save( + update_fields=[ + "task_id", + "status", + "error", + "started_at", + "completed_at", + "updated_at", + ] + ) + background_job.mark_started() + background_job.update_progress(10, "Подготовка backup-данных") + + try: + artifact = BackupExportService.build_backup_archive(actual_date=job.actual_date) + background_job.update_progress(70, "Запись архива на диск") + + archive_path = _resolve_backup_target_path(artifact.archive_filename) + archive_path.write_bytes(artifact.archive_bytes) + + job.status = BackupExportJob.Status.SUCCESS + job.archive_path = str(archive_path) + job.archive_filename = artifact.archive_filename + job.checksum_filename = artifact.checksum_filename + job.checksum_sha256 = artifact.checksum_sha256 + job.archive_size = len(artifact.archive_bytes) + job.organizations_count = artifact.organizations_count + job.completed_at = timezone.now() + job.error = "" + job.save( + update_fields=[ + "status", + "archive_path", + "archive_filename", + "checksum_filename", + "checksum_sha256", + "archive_size", + "organizations_count", + "completed_at", + "error", + "updated_at", + ] + ) + + result = { + "status": "success", + "actual_date": job.actual_date.isoformat(), + "archive_filename": job.archive_filename, + "checksum_sha256": job.checksum_sha256, + "organizations_count": job.organizations_count, + } + background_job.complete(result=result) + return result + + except Exception as exc: # noqa: BLE001 + logger.exception( + "Backup export failed for actual_date=%s, job_id=%s", + job.actual_date, + job.id, + ) + job.status = BackupExportJob.Status.FAILURE + job.error = str(exc) + job.completed_at = timezone.now() + job.save(update_fields=["status", "error", "completed_at", "updated_at"]) + background_job.fail(error=str(exc)) + raise diff --git a/src/apps/backups/urls.py b/src/apps/backups/urls.py new file mode 100644 index 0000000..daad737 --- /dev/null +++ b/src/apps/backups/urls.py @@ -0,0 +1,13 @@ +"""URL-конфигурация приложения backups.""" + +from apps.backups.views import BackupExportView +from django.urls import path + +app_name = "backups" + +backups_urlpatterns = [ + path("export/", BackupExportView.as_view(), name="export"), +] + +urlpatterns = [] + diff --git a/src/apps/backups/views.py b/src/apps/backups/views.py new file mode 100644 index 0000000..5f79588 --- /dev/null +++ b/src/apps/backups/views.py @@ -0,0 +1,104 @@ +"""API views экспорта защищённых backup архивов.""" + +from apps.backups.serializers import BackupExportRequestSerializer +from apps.backups.services import BackupExportError, BackupExportJobService +from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag +from django.http import HttpResponse +from django.utils import timezone +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response +from rest_framework.views import APIView + +BACKUPS_TAG = swagger_tag("Резервные копии", "backups") + + +class BackupExportView(APIView): + """Асинхронный экспорт защищённого backup архива.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[BACKUPS_TAG], + operation_summary="Сформировать backup архив", + operation_description=( + "Асинхронное формирование архива `.zip` на дату актуальности.\n" + "Логика endpoint:\n" + "- если задача на дату уже выполняется: вернуть 'подождите'\n" + "- если задачи нет: запустить формирование\n" + "- если задача завершена успешно: отдать архив и удалить его\n\n" + "Внутри архива:\n" + "- бинарный зашифрованный backup `.bin`\n" + "- контрольная сумма `.sha256`\n\n" + "Экспортирует только актуальные организации из реестров на указанную дату " + "и все связанные записи из parser-таблиц." + ), + request_body=BackupExportRequestSerializer, + responses={ + 200: openapi.Response( + description="Готовый backup-архив (application/zip)", + schema=openapi.Schema(type=openapi.TYPE_FILE), + ), + 202: openapi.Response( + description="Задача запущена или выполняется", + schema=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "status": openapi.Schema(type=openapi.TYPE_STRING), + "message": openapi.Schema(type=openapi.TYPE_STRING), + "actual_date": openapi.Schema( + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, + ), + "task_id": openapi.Schema(type=openapi.TYPE_STRING), + }, + ), + ), + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN, + }, + ) + def post(self, request): + serializer = BackupExportRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + actual_date = serializer.validated_data.get("actual_date") or timezone.localdate() + + try: + result = BackupExportJobService.check_or_start_job( + actual_date=actual_date, + requested_by_id=request.user.id if request.user.is_authenticated else None, + ) + except BackupExportError as exc: + raise ValidationError({"backup": str(exc)}) from exc + + if result.action in {"started", "wait"}: + return Response( + { + "status": result.action, + "message": result.message, + "actual_date": result.actual_date.isoformat(), + "task_id": result.task_id, + }, + status=status.HTTP_202_ACCEPTED, + ) + + try: + artifact = BackupExportJobService.consume_ready_archive( + actual_date=result.actual_date + ) + except BackupExportError as exc: + raise ValidationError({"backup": str(exc)}) from exc + + response = HttpResponse(artifact.archive_bytes, content_type="application/zip") + response.status_code = status.HTTP_200_OK + response["Content-Disposition"] = ( + f'attachment; filename="{artifact.archive_filename}"' + ) + response["X-Backup-SHA256"] = artifact.checksum_sha256 + response["X-Backup-Checksum-File"] = artifact.checksum_filename + response["X-Backup-Organizations"] = str(artifact.organizations_count) + response["X-Backup-Actual-Date"] = artifact.actual_date.isoformat() + return response diff --git a/src/apps/core/views.py b/src/apps/core/views.py index 2071f96..5c2a521 100644 --- a/src/apps/core/views.py +++ b/src/apps/core/views.py @@ -17,6 +17,7 @@ from django.conf import settings from django.db import connection from drf_yasg.utils import swagger_auto_schema from rest_framework import status +from rest_framework.exceptions import ValidationError from rest_framework.permissions import AllowAny from rest_framework.request import Request from rest_framework.response import Response @@ -270,7 +271,14 @@ class BackgroundJobListView(APIView): from apps.core.services import BackgroundJobService status_filter = request.query_params.get("status") - limit = min(int(request.query_params.get("limit", 50)), 100) + limit_raw = request.query_params.get("limit", "50") + try: + limit = int(limit_raw) + except (TypeError, ValueError) as exc: + raise ValidationError({"limit": "Параметр limit должен быть целым числом"}) from exc + if limit < 1: + raise ValidationError({"limit": "Параметр limit должен быть больше 0"}) + limit = min(limit, 100) jobs = BackgroundJobService.get_user_jobs( user_id=request.user.id, diff --git a/src/apps/exchange/__init__.py b/src/apps/exchange/__init__.py new file mode 100644 index 0000000..43c8d21 --- /dev/null +++ b/src/apps/exchange/__init__.py @@ -0,0 +1 @@ +"""Приложение для обмена данными с внешней БД.""" diff --git a/src/apps/exchange/admin.py b/src/apps/exchange/admin.py new file mode 100644 index 0000000..29c9c09 --- /dev/null +++ b/src/apps/exchange/admin.py @@ -0,0 +1,25 @@ +"""Admin configuration for exchange app.""" + +from apps.exchange.models import ExchangeConnection +from django.contrib import admin + + +@admin.register(ExchangeConnection) +class ExchangeConnectionAdmin(admin.ModelAdmin): + """Admin для подключений обмена.""" + + list_display = [ + "id", + "server", + "port", + "username", + "database_name", + "schema_name", + "is_active", + "last_checked_at", + "created_at", + ] + list_filter = ["is_active", "created_at", "last_checked_at"] + search_fields = ["server", "username", "database_name", "schema_name"] + readonly_fields = ["created_at", "updated_at", "last_checked_at", "last_error"] + ordering = ["-is_active", "-created_at"] diff --git a/src/apps/exchange/apps.py b/src/apps/exchange/apps.py new file mode 100644 index 0000000..cd6c130 --- /dev/null +++ b/src/apps/exchange/apps.py @@ -0,0 +1,9 @@ +from django.apps import AppConfig + + +class ExchangeConfig(AppConfig): + """Конфигурация приложения обмена данными.""" + + default_auto_field = "django.db.models.BigAutoField" + name = "apps.exchange" + verbose_name = "Обмен данными" diff --git a/src/apps/exchange/migrations/0001_initial.py b/src/apps/exchange/migrations/0001_initial.py new file mode 100644 index 0000000..9b7d8d8 --- /dev/null +++ b/src/apps/exchange/migrations/0001_initial.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.25 on 2026-03-04 11:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ExchangeConnection', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')), + ('server', models.CharField(max_length=255, verbose_name='сервер')), + ('port', models.PositiveIntegerField(default=5432, verbose_name='порт')), + ('username', models.CharField(max_length=255, verbose_name='пользователь')), + ('password', models.TextField(help_text='Хранится в открытом виде', verbose_name='пароль')), + ('database_name', models.CharField(max_length=255, verbose_name='имя БД')), + ('schema_name', models.CharField(default='public', max_length=255, verbose_name='имя схемы')), + ('is_active', models.BooleanField(db_index=True, default=False, verbose_name='активное')), + ('last_checked_at', models.DateTimeField(blank=True, null=True, verbose_name='последняя проверка')), + ('last_error', models.TextField(blank=True, verbose_name='последняя ошибка')), + ], + options={ + 'verbose_name': 'подключение обмена', + 'verbose_name_plural': 'подключения обмена', + 'db_table': 'exchange_connection', + 'ordering': ['-is_active', '-created_at'], + }, + ), + migrations.AddConstraint( + model_name='exchangeconnection', + constraint=models.UniqueConstraint(condition=models.Q(('is_active', True)), fields=('is_active',), name='unique_active_exchange_connection'), + ), + ] diff --git a/src/apps/exchange/migrations/__init__.py b/src/apps/exchange/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/apps/exchange/models.py b/src/apps/exchange/models.py new file mode 100644 index 0000000..ed731a0 --- /dev/null +++ b/src/apps/exchange/models.py @@ -0,0 +1,43 @@ +"""Модели приложения обмена данными.""" + +from apps.core.mixins import TimestampMixin +from django.db import models +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + + +class ExchangeConnection(TimestampMixin, models.Model): + """Подключение к целевой БД для обмена данными.""" + + server = models.CharField(_("сервер"), max_length=255) + port = models.PositiveIntegerField(_("порт"), default=5432) + username = models.CharField(_("пользователь"), max_length=255) + password = models.TextField(_("пароль"), help_text=_("Хранится в открытом виде")) + database_name = models.CharField(_("имя БД"), max_length=255) + schema_name = models.CharField(_("имя схемы"), max_length=255, default="public") + is_active = models.BooleanField(_("активное"), default=False, db_index=True) + last_checked_at = models.DateTimeField( + _("последняя проверка"), + null=True, + blank=True, + ) + last_error = models.TextField(_("последняя ошибка"), blank=True) + + class Meta: + db_table = "exchange_connection" + verbose_name = _("подключение обмена") + verbose_name_plural = _("подключения обмена") + ordering = ["-is_active", "-created_at"] + constraints = [ + models.UniqueConstraint( + fields=["is_active"], + condition=Q(is_active=True), + name="unique_active_exchange_connection", + ) + ] + + def __str__(self) -> str: + return ( + f"{self.username}@{self.server}:{self.port}/{self.database_name}" + f"[{self.schema_name}]" + ) diff --git a/src/apps/exchange/serializers.py b/src/apps/exchange/serializers.py new file mode 100644 index 0000000..931ce47 --- /dev/null +++ b/src/apps/exchange/serializers.py @@ -0,0 +1,88 @@ +"""Сериализаторы приложения обмена данными.""" + +from apps.exchange.models import ExchangeConnection +from rest_framework import serializers + + +class ExchangeConnectionSerializer(serializers.ModelSerializer): + """Сериализатор подключения без выдачи пароля в ответах.""" + + class Meta: + model = ExchangeConnection + fields = [ + "id", + "server", + "port", + "username", + "database_name", + "schema_name", + "is_active", + "last_checked_at", + "last_error", + "created_at", + "updated_at", + ] + read_only_fields = fields + + +class ExchangeConnectionCreateSerializer(serializers.Serializer): + """Входные данные для создания активного подключения.""" + + server = serializers.CharField(max_length=255) + port = serializers.IntegerField(min_value=1, max_value=65535) + username = serializers.CharField(max_length=255) + password = serializers.CharField() + database_name = serializers.CharField(max_length=255) + schema_name = serializers.RegexField( + regex=r"^[A-Za-z_][A-Za-z0-9_]*$", + max_length=255, + error_messages={ + "invalid": ( + "Имя схемы должно начинаться с буквы/_, " + "содержать только буквы, цифры и _" + ) + }, + ) + + +class ExchangeCopyRequestSerializer(serializers.Serializer): + """Параметры запуска копирования данных.""" + + mode = serializers.ChoiceField( + choices=["all", "single", "selected"], + default="all", + ) + table = serializers.CharField(required=False) + tables = serializers.ListField( + child=serializers.CharField(), + required=False, + allow_empty=False, + ) + truncate_before_copy = serializers.BooleanField(default=True) + + def validate(self, attrs): + mode = attrs["mode"] + table = attrs.get("table") + tables = attrs.get("tables") + + if mode == "single" and not table: + raise serializers.ValidationError( + {"table": "Для mode=single нужно указать table"} + ) + + if mode == "selected" and not tables: + raise serializers.ValidationError( + {"tables": "Для mode=selected нужно указать tables"} + ) + + if mode != "single" and table: + raise serializers.ValidationError( + {"table": "Поле table допустимо только для mode=single"} + ) + + if mode != "selected" and tables: + raise serializers.ValidationError( + {"tables": "Поле tables допустимо только для mode=selected"} + ) + + return attrs diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py new file mode 100644 index 0000000..1d6ad50 --- /dev/null +++ b/src/apps/exchange/services.py @@ -0,0 +1,447 @@ +"""Сервисы приложения обмена данными.""" + +from __future__ import annotations + +from contextlib import suppress +from typing import Any + +from apps.exchange.models import ExchangeConnection +from django.apps import apps as django_apps +from django.db import connections, transaction +from django.utils import timezone + + +class ExchangeServiceError(ValueError): + """Ошибка операций приложения обмена данными.""" + + +class ExchangeConnectionService: + """Сервис управления подключениями и синхронизацией данных.""" + + PARSER_MODEL_LABELS = [ + "parsers.ParserLoadLog", + "parsers.IndustrialCertificateRecord", + "parsers.ManufacturerRecord", + "parsers.Proxy", + "parsers.InspectionRecord", + "parsers.ProcurementRecord", + "parsers.FinancialReport", + "parsers.FinancialReportLine", + ] + + @classmethod + @transaction.atomic + def create_active_connection_and_prepare(cls, **payload) -> ExchangeConnection: + """ + Создать активное подключение. + + В рамках одной операции: + 1. Деактивировать текущее активное подключение. + 2. Сохранить новое как активное. + 3. Проверить соединение и структуру target DB. + + Важно: сервис НЕ изменяет структуру target DB. + """ + ExchangeConnection.objects.filter(is_active=True).update(is_active=False) + connection = ExchangeConnection.objects.create(is_active=True, **payload) + + try: + alias = cls.test_connection(connection) + cls.validate_target_structure( + connection=connection, + alias=alias, + schema_name=connection.schema_name, + ) + except Exception as exc: # noqa: BLE001 + raise ExchangeServiceError(str(exc)) from exc + + connection.last_checked_at = timezone.now() + connection.last_error = "" + connection.save(update_fields=["last_checked_at", "last_error", "updated_at"]) + + return connection + + @classmethod + def get_active_connection(cls) -> ExchangeConnection: + connection = ExchangeConnection.objects.filter(is_active=True).first() + if not connection: + raise ExchangeServiceError("Активное подключение не найдено") + return connection + + @classmethod + def test_connection(cls, connection: ExchangeConnection) -> str: + alias = cls._configure_alias(connection) + + try: + db_connection = connections[alias] + db_connection.ensure_connection() + with db_connection.cursor() as cursor: + cursor.execute("SELECT 1") + except Exception as exc: # noqa: BLE001 + cls._mark_connection_error(connection, str(exc)) + raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc + + return alias + + @classmethod + def validate_target_structure( + cls, + *, + connection: ExchangeConnection, + alias: str, + schema_name: str, + models_to_copy: list | None = None, + ) -> None: + """ + Проверить структуру target DB без изменений. + + Проверяет: + - существование схемы + - наличие всех обязательных таблиц + - наличие всех обязательных колонок в таблицах + """ + try: + db_connection = connections[alias] + db_connection.ensure_connection() + required_models = models_to_copy or cls._extend_models_with_dependencies( + cls._get_parser_models() + ) + cls._validate_schema_exists(alias=alias, schema_name=schema_name) + cls._validate_tables_exist( + alias=alias, + schema_name=schema_name, + models_to_copy=required_models, + ) + cls._validate_columns_exist( + alias=alias, + schema_name=schema_name, + models_to_copy=required_models, + ) + except ExchangeServiceError as exc: + cls._mark_connection_error(connection, str(exc)) + raise + except Exception as exc: # noqa: BLE001 + cls._mark_connection_error(connection, str(exc)) + raise ExchangeServiceError( + f"Ошибка проверки структуры целевой БД: {exc}" + ) from exc + + @classmethod + def copy_parsers_data( + cls, + *, + connection: ExchangeConnection, + mode: str, + table: str | None = None, + tables: list[str] | None = None, + truncate_before_copy: bool = True, + ) -> dict[str, Any]: + """Скопировать данные из локальной БД в целевую БД.""" + alias = cls._configure_alias(connection) + selected_models = cls._resolve_models(mode=mode, table=table, tables=tables) + models_to_copy = cls._extend_models_with_dependencies(selected_models) + + try: + connections[alias].ensure_connection() + except Exception as exc: # noqa: BLE001 + cls._mark_connection_error(connection, str(exc)) + raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc + + cls.validate_target_structure( + connection=connection, + alias=alias, + schema_name=connection.schema_name, + models_to_copy=models_to_copy, + ) + + if truncate_before_copy: + cls._truncate_tables(alias=alias, models_to_copy=models_to_copy) + + copied_by_table: dict[str, int] = {} + for model in models_to_copy: + copied_by_table[model._meta.db_table] = cls._copy_model_data( + model=model, + alias=alias, + truncate_before_copy=truncate_before_copy, + ) + + total_rows = sum(copied_by_table.values()) + + connection.last_checked_at = timezone.now() + connection.last_error = "" + connection.save(update_fields=["last_checked_at", "last_error", "updated_at"]) + + return { + "mode": mode, + "tables": list(copied_by_table.keys()), + "rows_by_table": copied_by_table, + "total_rows": total_rows, + "truncate_before_copy": truncate_before_copy, + } + + @classmethod + def _configure_alias(cls, connection: ExchangeConnection) -> str: + alias = f"exchange_target_{connection.id}" + + config = { + "ENGINE": "django.db.backends.postgresql", + "NAME": connection.database_name, + "USER": connection.username, + "PASSWORD": connection.password, + "HOST": connection.server, + "PORT": connection.port, + "OPTIONS": { + "options": f"-c search_path={connection.schema_name},public", + }, + "CONN_MAX_AGE": 0, + "ATOMIC_REQUESTS": False, + "AUTOCOMMIT": True, + "TIME_ZONE": None, + "TEST": {}, + } + + if alias in connections.databases: + with suppress(Exception): + connections[alias].close() + + connections.databases[alias] = config + + storage = getattr(connections, "_connections", None) + if storage is not None and hasattr(storage, "__dict__"): + storage.__dict__.pop(alias, None) + + return alias + + @classmethod + def _validate_schema_exists(cls, *, alias: str, schema_name: str) -> None: + with connections[alias].cursor() as cursor: + cursor.execute( + """ + SELECT 1 + FROM information_schema.schemata + WHERE schema_name = %s + """, + [schema_name], + ) + if cursor.fetchone() is None: + raise ExchangeServiceError( + f"Схема '{schema_name}' отсутствует в целевой БД" + ) + + @classmethod + def _validate_tables_exist( + cls, + *, + alias: str, + schema_name: str, + models_to_copy: list, + ) -> None: + expected_tables = {model._meta.db_table for model in models_to_copy} + with connections[alias].cursor() as cursor: + cursor.execute( + """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = %s + """, + [schema_name], + ) + existing_tables = {row[0] for row in cursor.fetchall()} + + missing_tables = sorted(expected_tables - existing_tables) + if missing_tables: + raise ExchangeServiceError( + "В целевой БД отсутствуют таблицы: " + ", ".join(missing_tables) + ) + + @classmethod + def _validate_columns_exist( + cls, + *, + alias: str, + schema_name: str, + models_to_copy: list, + ) -> None: + for model in models_to_copy: + table_name = model._meta.db_table + expected_columns = {field.column for field in model._meta.local_fields} + + with connections[alias].cursor() as cursor: + cursor.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + """, + [schema_name, table_name], + ) + existing_columns = {row[0] for row in cursor.fetchall()} + + missing_columns = sorted(expected_columns - existing_columns) + if missing_columns: + raise ExchangeServiceError( + f"В таблице '{table_name}' отсутствуют колонки: " + + ", ".join(missing_columns) + ) + + @classmethod + def _get_parser_models(cls) -> list: + return [django_apps.get_model(label) for label in cls.PARSER_MODEL_LABELS] + + @classmethod + def _resolve_models( + cls, + *, + mode: str, + table: str | None, + tables: list[str] | None, + ) -> list: + parser_models = cls._get_parser_models() + + if mode == "all": + return parser_models + + mapping: dict[str, Any] = {} + for model in parser_models: + mapping[model._meta.db_table] = model + mapping[model._meta.model_name] = model + mapping[model.__name__.lower()] = model + + requested_names: list[str] + if mode == "single": + requested_names = [table] if table else [] + else: + requested_names = tables or [] + + resolved_models = [] + for requested_name in requested_names: + model = mapping.get(requested_name) + if not model: + available = ", ".join(sorted(m._meta.db_table for m in parser_models)) + raise ExchangeServiceError( + f"Неизвестная таблица '{requested_name}'. Доступные: {available}" + ) + resolved_models.append(model) + + return resolved_models + + @classmethod + def _extend_models_with_dependencies(cls, models_to_copy: list) -> list: + """Добавить обязательные зависимые модели для корректного copy.""" + if not cls._requires_registry_organizations(models_to_copy): + return models_to_copy + + organization_model = django_apps.get_model("registers.Organization") + ordered_models = [organization_model, *models_to_copy] + + unique_models = [] + seen = set() + for model in ordered_models: + model_key = (model._meta.app_label, model._meta.model_name) + if model_key in seen: + continue + seen.add(model_key) + unique_models.append(model) + + return unique_models + + @classmethod + def _requires_registry_organizations(cls, models_to_copy: list) -> bool: + return any( + any(field.name == "registry_organization" for field in model._meta.local_fields) + for model in models_to_copy + ) + + @classmethod + def _truncate_tables(cls, *, alias: str, models_to_copy: list) -> None: + with connections[alias].cursor() as cursor: + for model in reversed(models_to_copy): + cursor.execute( + f'TRUNCATE TABLE "{model._meta.db_table}" RESTART IDENTITY CASCADE' + ) + + @classmethod + def _copy_model_data( + cls, + *, + model, + alias: str, + truncate_before_copy: bool, + chunk_size: int = 1000, + ) -> int: + field_names = [field.attname for field in model._meta.local_fields] + queryset = model.objects.using("default").all().order_by("pk") + + total_processed = 0 + batch = [] + pk_name = model._meta.pk.attname + + for source_obj in queryset.iterator(chunk_size=chunk_size): + row_data = {field_name: getattr(source_obj, field_name) for field_name in field_names} + batch.append(model(**row_data)) + + if len(batch) >= chunk_size: + total_processed += cls._insert_batch( + model=model, + alias=alias, + batch=batch, + pk_name=pk_name, + chunk_size=chunk_size, + truncate_before_copy=truncate_before_copy, + ) + batch = [] + + if batch: + total_processed += cls._insert_batch( + model=model, + alias=alias, + batch=batch, + pk_name=pk_name, + chunk_size=chunk_size, + truncate_before_copy=truncate_before_copy, + ) + + return total_processed + + @classmethod + def _insert_batch( + cls, + *, + model, + alias: str, + batch: list, + pk_name: str, + chunk_size: int, + truncate_before_copy: bool, + ) -> int: + if truncate_before_copy: + model.objects.using(alias).bulk_create( + batch, + batch_size=chunk_size, + ignore_conflicts=False, + ) + return len(batch) + + pk_values = [getattr(item, pk_name) for item in batch] + existing_before = set( + model.objects.using(alias) + .filter(**{f"{pk_name}__in": pk_values}) + .values_list(pk_name, flat=True) + ) + model.objects.using(alias).bulk_create( + batch, + batch_size=chunk_size, + ignore_conflicts=True, + ) + existing_after = set( + model.objects.using(alias) + .filter(**{f"{pk_name}__in": pk_values}) + .values_list(pk_name, flat=True) + ) + return len(existing_after - existing_before) + + @classmethod + def _mark_connection_error(cls, connection: ExchangeConnection, error_message: str) -> None: + connection.last_checked_at = timezone.now() + connection.last_error = error_message + connection.save(update_fields=["last_checked_at", "last_error", "updated_at"]) diff --git a/src/apps/exchange/tasks.py b/src/apps/exchange/tasks.py new file mode 100644 index 0000000..fff9684 --- /dev/null +++ b/src/apps/exchange/tasks.py @@ -0,0 +1,67 @@ +"""Celery-задачи приложения exchange.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from apps.core.services import BackgroundJobService +from apps.exchange.models import ExchangeConnection +from apps.exchange.services import ExchangeConnectionService +from celery import shared_task + +logger = logging.getLogger(__name__) + + +@shared_task(bind=True) +def copy_parsers_data_async( + self, + *, + connection_id: int, + payload: dict[str, Any], + requested_by_id: int | None = None, +) -> dict[str, Any]: + """Асинхронное копирование parser-данных в target DB.""" + task_id = self.request.id or str(uuid.uuid4()) + background_job = BackgroundJobService.get_by_task_id_or_none(task_id) + if not background_job: + background_job = BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.exchange.tasks.copy_parsers_data_async", + user_id=requested_by_id, + meta={ + "connection_id": connection_id, + **payload, + }, + ) + + connection = ExchangeConnection.objects.filter(id=connection_id, is_active=True).first() + if connection is None: + background_job.fail(error="Активное подключение не найдено") + raise ValueError(f"Active exchange connection not found: {connection_id}") + + background_job.mark_started() + background_job.update_progress(10, "Проверка структуры целевой БД") + + try: + result = ExchangeConnectionService.copy_parsers_data( + connection=connection, + **payload, + ) + background_job.update_progress(90, "Фиксация результата") + output = { + "status": "success", + "connection_id": connection_id, + **result, + } + background_job.complete(result=output) + return output + except Exception as exc: # noqa: BLE001 + logger.exception( + "Exchange copy failed (connection_id=%s, task_id=%s)", + connection_id, + task_id, + ) + background_job.fail(error=str(exc)) + raise diff --git a/src/apps/exchange/urls.py b/src/apps/exchange/urls.py new file mode 100644 index 0000000..3965581 --- /dev/null +++ b/src/apps/exchange/urls.py @@ -0,0 +1,13 @@ +"""URL конфигурация приложения exchange.""" + +from apps.exchange.views import ExchangeConnectionListCreateView, ExchangeCopyDataView +from django.urls import path + +app_name = "exchange" + +exchange_urlpatterns = [ + path("connections/", ExchangeConnectionListCreateView.as_view(), name="connections"), + path("copy/", ExchangeCopyDataView.as_view(), name="copy"), +] + +urlpatterns = [] diff --git a/src/apps/exchange/views.py b/src/apps/exchange/views.py new file mode 100644 index 0000000..e8c2b6b --- /dev/null +++ b/src/apps/exchange/views.py @@ -0,0 +1,157 @@ +"""API views для обмена данными с внешней БД.""" + +from contextlib import suppress + +from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag +from apps.core.response import api_created_response, api_response +from apps.core.services import BackgroundJobService +from apps.exchange.models import ExchangeConnection +from apps.exchange.serializers import ( + ExchangeConnectionCreateSerializer, + ExchangeConnectionSerializer, + ExchangeCopyRequestSerializer, +) +from apps.exchange.services import ExchangeConnectionService, ExchangeServiceError +from apps.exchange.tasks import copy_parsers_data_async +from django.db import IntegrityError +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import IsAdminUser +from rest_framework.views import APIView + +EXCHANGE_TAG = swagger_tag("Обмен данными", "exchange") + + +class ExchangeConnectionListCreateView(APIView): + """API списка и создания подключений обмена.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[EXCHANGE_TAG], + operation_summary="Список подключений", + operation_description=( + "Возвращает список всех сохранённых подключений для обмена.\n" + "Пароль в ответ не возвращается." + ), + responses={ + 200: ExchangeConnectionSerializer(many=True), + **ErrorResponses.ADMIN, + }, + ) + def get(self, request): + queryset = ExchangeConnection.objects.all().order_by("-is_active", "-created_at") + serializer = ExchangeConnectionSerializer(queryset, many=True) + return api_response(serializer.data, status_code=status.HTTP_200_OK) + + @swagger_auto_schema( + tags=[EXCHANGE_TAG], + operation_summary="Создать активное подключение", + operation_description=( + "Создаёт новое подключение к целевой БД как активное.\n" + "Перед созданием деактивирует текущее активное подключение.\n" + "После сохранения проверяет соединение и валидирует структуру целевой БД." + ), + request_body=ExchangeConnectionCreateSerializer, + responses={ + 201: ExchangeConnectionSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN, + }, + ) + def post(self, request): + serializer = ExchangeConnectionCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + connection = ExchangeConnectionService.create_active_connection_and_prepare( + **serializer.validated_data + ) + except ExchangeServiceError as exc: + raise ValidationError({"connection": str(exc)}) from exc + + output = ExchangeConnectionSerializer(connection) + return api_created_response(output.data) + + +class ExchangeCopyDataView(APIView): + """API запуска копирования данных в целевую БД.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[EXCHANGE_TAG], + operation_summary="Копировать данные parsers в target DB", + operation_description=( + "Асинхронно запускает копирование данных из локальной БД " + "в активную целевую БД.\n" + "Перед копированием выполняется только проверка структуры " + "(без изменения схемы/миграций).\n" + "Поддерживает режимы: all / single / selected." + ), + request_body=ExchangeCopyRequestSerializer, + responses={ + 202: openapi.Response( + description="Копирование поставлено в очередь", + schema=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "status": openapi.Schema(type=openapi.TYPE_STRING), + "message": openapi.Schema(type=openapi.TYPE_STRING), + "task_id": openapi.Schema(type=openapi.TYPE_STRING), + "connection_id": openapi.Schema(type=openapi.TYPE_INTEGER), + "mode": openapi.Schema(type=openapi.TYPE_STRING), + "truncate_before_copy": openapi.Schema( + type=openapi.TYPE_BOOLEAN + ), + }, + ), + ), + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN, + }, + ) + def post(self, request): + serializer = ExchangeCopyRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + active_connection = ExchangeConnectionService.get_active_connection() + task = copy_parsers_data_async.delay( + connection_id=active_connection.id, + payload=serializer.validated_data, + requested_by_id=request.user.id if request.user.is_authenticated else None, + ) + + # Предсоздаём запись для мгновенного отслеживания в /api/v1/jobs/{task_id}/ + with suppress(IntegrityError): + BackgroundJobService.create_job( + task_id=task.id, + task_name="apps.exchange.tasks.copy_parsers_data_async", + user_id=request.user.id if request.user.is_authenticated else None, + meta={ + "connection_id": active_connection.id, + "mode": serializer.validated_data["mode"], + "table": serializer.validated_data.get("table"), + "tables": serializer.validated_data.get("tables"), + "truncate_before_copy": serializer.validated_data.get( + "truncate_before_copy" + ), + }, + ) + except ExchangeServiceError as exc: + raise ValidationError({"copy": str(exc)}) from exc + + return api_response( + { + "status": "started", + "message": "Копирование запущено в фоне.", + "task_id": task.id, + "connection_id": active_connection.id, + "mode": serializer.validated_data["mode"], + "truncate_before_copy": serializer.validated_data["truncate_before_copy"], + }, + status_code=status.HTTP_202_ACCEPTED, + ) diff --git a/src/apps/parsers/migrations/0009_auto_20260304_1159.py b/src/apps/parsers/migrations/0009_auto_20260304_1159.py new file mode 100644 index 0000000..3d0edbb --- /dev/null +++ b/src/apps/parsers/migrations/0009_auto_20260304_1159.py @@ -0,0 +1,40 @@ +# Generated by Django 3.2.25 on 2026-03-04 11:59 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("registers", "0002_auto_20260304_1038"), + ("parsers", "0008_add_load_log_unique_constraint"), + ] + + operations = [ + migrations.AddField( + model_name="financialreport", + name="registry_organization", + field=models.ForeignKey(blank=True, help_text="Связь с организацией из приложения реестров", null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="financial_reports", to="registers.organization", verbose_name="организация из реестров"), + ), + migrations.AddField( + model_name="industrialcertificaterecord", + name="registry_organization", + field=models.ForeignKey(blank=True, help_text="Связь с организацией из приложения реестров", null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="industrial_certificate_records", to="registers.organization", verbose_name="организация из реестров"), + ), + migrations.AddField( + model_name="inspectionrecord", + name="registry_organization", + field=models.ForeignKey(blank=True, help_text="Связь с организацией из приложения реестров", null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="inspection_records", to="registers.organization", verbose_name="организация из реестров"), + ), + migrations.AddField( + model_name="manufacturerrecord", + name="registry_organization", + field=models.ForeignKey(blank=True, help_text="Связь с организацией из приложения реестров", null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="manufacturer_records", to="registers.organization", verbose_name="организация из реестров"), + ), + migrations.AddField( + model_name="procurementrecord", + name="registry_organization", + field=models.ForeignKey(blank=True, help_text="Связь с организацией из приложения реестров", null=True, on_delete=django.db.models.deletion.SET_NULL, related_name="procurement_records", to="registers.organization", verbose_name="организация из реестров"), + ), + ] diff --git a/src/apps/parsers/migrations/0010_link_registry_organizations.py b/src/apps/parsers/migrations/0010_link_registry_organizations.py new file mode 100644 index 0000000..f19ce52 --- /dev/null +++ b/src/apps/parsers/migrations/0010_link_registry_organizations.py @@ -0,0 +1,166 @@ +# Generated by Django 3.2.25 on 2026-03-04 + +from collections import defaultdict + +from django.db import migrations + + +def _normalize_identifier(value): + if value is None: + return None + + as_str = str(value).strip() + if not as_str or not as_str.isdigit(): + return None + + return int(as_str) + + +def _resolve_organization_id(*, inn, ogrn, by_pair, by_inn, by_ogrn): + if inn is not None and ogrn is not None: + by_pair_id = by_pair.get((inn, ogrn)) + if by_pair_id is not None: + return by_pair_id + + if inn is not None and inn in by_inn: + return by_inn[inn] + + if ogrn is not None and ogrn in by_ogrn: + return by_ogrn[ogrn] + + return None + + +def _build_organization_lookup(organization_model): + by_pair = {} + by_inn_candidates = defaultdict(list) + by_ogrn_candidates = defaultdict(list) + + organizations = organization_model.objects.all().values("id", "mn_inn", "mn_ogrn") + for organization in organizations: + organization_id = organization["id"] + inn = organization["mn_inn"] + ogrn = organization["mn_ogrn"] + + by_pair[(inn, ogrn)] = organization_id + by_inn_candidates[inn].append(organization_id) + by_ogrn_candidates[ogrn].append(organization_id) + + by_inn = { + inn: ids[0] + for inn, ids in by_inn_candidates.items() + if len(set(ids)) == 1 + } + by_ogrn = { + ogrn: ids[0] + for ogrn, ids in by_ogrn_candidates.items() + if len(set(ids)) == 1 + } + + return by_pair, by_inn, by_ogrn + + +def _link_model_records( + *, + model, + inn_field, + ogrn_field, + by_pair, + by_inn, + by_ogrn, + batch_size=1000, +): + field_names = ["id"] + if inn_field: + field_names.append(inn_field) + if ogrn_field: + field_names.append(ogrn_field) + + queryset = model.objects.filter(registry_organization__isnull=True).only(*field_names) + updates = [] + + for record in queryset.iterator(chunk_size=batch_size): + inn = _normalize_identifier(getattr(record, inn_field)) if inn_field else None + ogrn = _normalize_identifier(getattr(record, ogrn_field)) if ogrn_field else None + + organization_id = _resolve_organization_id( + inn=inn, + ogrn=ogrn, + by_pair=by_pair, + by_inn=by_inn, + by_ogrn=by_ogrn, + ) + if organization_id is None: + continue + + record.registry_organization_id = organization_id + updates.append(record) + + if len(updates) >= batch_size: + model.objects.bulk_update(updates, ["registry_organization"], batch_size=batch_size) + updates = [] + + if updates: + model.objects.bulk_update(updates, ["registry_organization"], batch_size=batch_size) + + +def link_registry_organizations(apps, schema_editor): + organization_model = apps.get_model("registers", "Organization") + industrial_model = apps.get_model("parsers", "IndustrialCertificateRecord") + manufacturer_model = apps.get_model("parsers", "ManufacturerRecord") + inspection_model = apps.get_model("parsers", "InspectionRecord") + procurement_model = apps.get_model("parsers", "ProcurementRecord") + financial_report_model = apps.get_model("parsers", "FinancialReport") + + by_pair, by_inn, by_ogrn = _build_organization_lookup(organization_model) + + _link_model_records( + model=industrial_model, + inn_field="inn", + ogrn_field="ogrn", + by_pair=by_pair, + by_inn=by_inn, + by_ogrn=by_ogrn, + ) + _link_model_records( + model=manufacturer_model, + inn_field="inn", + ogrn_field="ogrn", + by_pair=by_pair, + by_inn=by_inn, + by_ogrn=by_ogrn, + ) + _link_model_records( + model=inspection_model, + inn_field="inn", + ogrn_field="ogrn", + by_pair=by_pair, + by_inn=by_inn, + by_ogrn=by_ogrn, + ) + _link_model_records( + model=procurement_model, + inn_field="customer_inn", + ogrn_field="customer_ogrn", + by_pair=by_pair, + by_inn=by_inn, + by_ogrn=by_ogrn, + ) + _link_model_records( + model=financial_report_model, + inn_field=None, + ogrn_field="ogrn", + by_pair=by_pair, + by_inn=by_inn, + by_ogrn=by_ogrn, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0009_auto_20260304_1159"), + ] + + operations = [ + migrations.RunPython(link_registry_organizations, migrations.RunPython.noop), + ] diff --git a/src/apps/parsers/migrations/0011_add_normalized_date_and_amount_fields.py b/src/apps/parsers/migrations/0011_add_normalized_date_and_amount_fields.py new file mode 100644 index 0000000..054a2a9 --- /dev/null +++ b/src/apps/parsers/migrations/0011_add_normalized_date_and_amount_fields.py @@ -0,0 +1,91 @@ +# Generated by Django 3.2.25 on 2026-03-04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0010_link_registry_organizations"), + ] + + operations = [ + migrations.AddField( + model_name="industrialcertificaterecord", + name="issue_date_normalized", + field=models.DateField( + blank=True, + db_index=True, + help_text="Нормализованная дата выдачи сертификата", + null=True, + verbose_name="дата выдачи (нормализованная)", + ), + ), + migrations.AddField( + model_name="industrialcertificaterecord", + name="expiry_date_normalized", + field=models.DateField( + blank=True, + db_index=True, + help_text="Нормализованная дата окончания действия", + null=True, + verbose_name="дата окончания (нормализованная)", + ), + ), + migrations.AddField( + model_name="inspectionrecord", + name="start_date_normalized", + field=models.DateField( + blank=True, + db_index=True, + help_text="Нормализованная дата начала проверки", + null=True, + verbose_name="дата начала (нормализованная)", + ), + ), + migrations.AddField( + model_name="inspectionrecord", + name="end_date_normalized", + field=models.DateField( + blank=True, + db_index=True, + help_text="Нормализованная дата окончания проверки", + null=True, + verbose_name="дата окончания (нормализованная)", + ), + ), + migrations.AddField( + model_name="procurementrecord", + name="max_price_amount", + field=models.DecimalField( + blank=True, + db_index=True, + decimal_places=2, + help_text="Нормализованная числовая НМЦ", + max_digits=20, + null=True, + verbose_name="НМЦ (число)", + ), + ), + migrations.AddField( + model_name="procurementrecord", + name="publish_date_normalized", + field=models.DateField( + blank=True, + db_index=True, + help_text="Нормализованная дата публикации извещения", + null=True, + verbose_name="дата публикации (нормализованная)", + ), + ), + migrations.AddField( + model_name="procurementrecord", + name="end_date_normalized", + field=models.DateField( + blank=True, + db_index=True, + help_text="Нормализованная дата окончания подачи заявок", + null=True, + verbose_name="дата окончания (нормализованная)", + ), + ), + ] diff --git a/src/apps/registers/__init__.py b/src/apps/registers/__init__.py new file mode 100644 index 0000000..f729fe8 --- /dev/null +++ b/src/apps/registers/__init__.py @@ -0,0 +1 @@ +"""Приложение для управления реестрами организаций.""" diff --git a/src/apps/registers/admin.py b/src/apps/registers/admin.py new file mode 100644 index 0000000..7a2c3db --- /dev/null +++ b/src/apps/registers/admin.py @@ -0,0 +1,98 @@ +"""Admin configuration for registers app.""" + +from apps.registers.models import ( + Organization, + Register, + RegisterUpload, + RegistryMembershipPeriod, +) +from django.contrib import admin + + +@admin.register(Register) +class RegisterAdmin(admin.ModelAdmin): + """Admin для реестров.""" + + list_display = ["name", "id", "active_organizations_count", "created_at"] + search_fields = ["name", "id"] + readonly_fields = ["id", "created_at", "updated_at"] + ordering = ["name"] + + def active_organizations_count(self, obj): + return obj.membership_periods.filter(ended_at__isnull=True).count() + + active_organizations_count.short_description = "Активных организаций" + + +@admin.register(Organization) +class OrganizationAdmin(admin.ModelAdmin): + """Admin для канонических организаций.""" + + list_display = [ + "pn_name_short", + "mn_ogrn", + "mn_inn", + "in_kpp", + "mn_okpo", + "created_at", + ] + list_filter = ["created_at"] + search_fields = ["pn_name", "mn_ogrn", "mn_inn", "in_kpp", "mn_okpo"] + readonly_fields = ["created_at", "updated_at"] + ordering = ["pn_name"] + + def pn_name_short(self, obj): + name = obj.pn_name or "" + return name[:60] + "..." if len(name) > 60 else name + + pn_name_short.short_description = "Организация" + pn_name_short.admin_order_field = "pn_name" + + +@admin.register(RegisterUpload) +class RegisterUploadAdmin(admin.ModelAdmin): + """Admin для загрузок реестров.""" + + list_display = [ + "id", + "registry", + "actual_date", + "rows_count", + "file_name", + "uploaded_by", + "created_at", + ] + list_filter = ["registry", "actual_date", "created_at"] + search_fields = ["file_name", "file_hash", "registry__name"] + readonly_fields = ["created_at", "updated_at", "file_hash"] + ordering = ["-actual_date", "-created_at"] + + +@admin.register(RegistryMembershipPeriod) +class RegistryMembershipPeriodAdmin(admin.ModelAdmin): + """Admin для периодов участия в реестрах.""" + + list_display = [ + "registry", + "organization_short", + "started_at", + "ended_at", + "started_by_upload", + "ended_by_upload", + ] + list_filter = ["registry", "started_at", "ended_at"] + search_fields = [ + "registry__name", + "organization__pn_name", + "organization__mn_ogrn", + "organization__mn_inn", + ] + readonly_fields = ["created_at", "updated_at"] + ordering = ["registry__name", "-started_at"] + + def organization_short(self, obj): + name = obj.organization.pn_name or "" + return name[:60] + "..." if len(name) > 60 else name + + organization_short.short_description = "Организация" + organization_short.admin_order_field = "organization__pn_name" diff --git a/src/apps/registers/apps.py b/src/apps/registers/apps.py new file mode 100644 index 0000000..a05bc64 --- /dev/null +++ b/src/apps/registers/apps.py @@ -0,0 +1,9 @@ +from django.apps import AppConfig + + +class RegistersConfig(AppConfig): + """Конфигурация приложения реестров.""" + + default_auto_field = "django.db.models.BigAutoField" + name = "apps.registers" + verbose_name = "Реестры организаций" diff --git a/src/apps/registers/migrations/0001_initial.py b/src/apps/registers/migrations/0001_initial.py new file mode 100644 index 0000000..0c9255e --- /dev/null +++ b/src/apps/registers/migrations/0001_initial.py @@ -0,0 +1,64 @@ +# Generated by Django 3.2.25 on 2026-03-04 10:17 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Register', + fields=[ + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(help_text='Человекочитаемое название реестра', max_length=255, unique=True, verbose_name='название')), + ], + options={ + 'verbose_name': 'реестр', + 'verbose_name_plural': 'реестры', + 'db_table': 'registers_register', + 'ordering': ['name'], + }, + ), + migrations.CreateModel( + name='Organization', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')), + ('pn_name', models.TextField(help_text='Полное наименование организации', verbose_name='наименование')), + ('mn_ogrn', models.BigIntegerField(db_index=True, help_text='ОГРН организации', verbose_name='ОГРН')), + ('mn_inn', models.BigIntegerField(db_index=True, help_text='ИНН организации', verbose_name='ИНН')), + ('in_kpp', models.BigIntegerField(blank=True, db_index=True, help_text='КПП организации', null=True, verbose_name='КПП')), + ('mn_okpo', models.TextField(db_index=True, help_text='ОКПО (текст с обязательной числовой валидацией)', validators=[django.core.validators.RegexValidator(message='ОКПО должен содержать только цифры', regex='^\\d+$')], verbose_name='ОКПО')), + ('registry', models.ForeignKey(help_text='Реестр, к которому относится организация', on_delete=django.db.models.deletion.CASCADE, related_name='organizations', to='registers.register', verbose_name='реестр')), + ], + options={ + 'verbose_name': 'организация', + 'verbose_name_plural': 'организации', + 'db_table': 'registers_organization', + 'ordering': ['pn_name'], + }, + ), + migrations.AddIndex( + model_name='organization', + index=models.Index(fields=['registry', 'mn_ogrn'], name='registers_o_registr_e00175_idx'), + ), + migrations.AddIndex( + model_name='organization', + index=models.Index(fields=['registry', 'mn_inn'], name='registers_o_registr_6ab4dd_idx'), + ), + migrations.AddIndex( + model_name='organization', + index=models.Index(fields=['registry', 'mn_okpo'], name='registers_o_registr_629dae_idx'), + ), + ] diff --git a/src/apps/registers/migrations/0002_auto_20260304_1038.py b/src/apps/registers/migrations/0002_auto_20260304_1038.py new file mode 100644 index 0000000..6d90a5c --- /dev/null +++ b/src/apps/registers/migrations/0002_auto_20260304_1038.py @@ -0,0 +1,366 @@ +# Generated by Django 3.2.25 on 2026-03-04 10:38 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django.db.models.expressions +from django.utils import timezone + + +def migrate_legacy_organization_memberships(apps, schema_editor): + """ + Перенос текущего состояния из старой модели Organization(registry FK) + в периодную модель membership. + + Исторических дат из прошлых загрузок нет, поэтому все legacy записи + считаются активными с даты миграции. + """ + + Organization = apps.get_model("registers", "Organization") + RegisterUpload = apps.get_model("registers", "RegisterUpload") + RegistryMembershipPeriod = apps.get_model("registers", "RegistryMembershipPeriod") + + db_alias = schema_editor.connection.alias + snapshot_date = timezone.localdate() + + organizations = list(Organization.objects.using(db_alias).order_by("id")) + + uploads_by_registry: dict = {} + rows_count_by_registry: dict = {} + canonical_by_identity: dict = {} + membership_seen: set[tuple[int, int]] = set() + duplicate_organization_ids: list[int] = [] + + for organization in organizations: + identity = (organization.mn_ogrn, organization.mn_inn) + + canonical_org_id = canonical_by_identity.get(identity) + if canonical_org_id is None: + canonical_org_id = organization.id + canonical_by_identity[identity] = canonical_org_id + else: + duplicate_organization_ids.append(organization.id) + + upload = uploads_by_registry.get(organization.registry_id) + if upload is None: + upload = RegisterUpload.objects.using(db_alias).create( + registry_id=organization.registry_id, + actual_date=snapshot_date, + file_name="legacy_migration.xlsx", + file_hash=( + f"legacy-{organization.registry_id}-{snapshot_date.isoformat()}" + ), + rows_count=0, + ) + uploads_by_registry[organization.registry_id] = upload + rows_count_by_registry[organization.registry_id] = 0 + + membership_key = (organization.registry_id, canonical_org_id) + if membership_key in membership_seen: + continue + + membership_seen.add(membership_key) + RegistryMembershipPeriod.objects.using(db_alias).create( + registry_id=organization.registry_id, + organization_id=canonical_org_id, + started_at=snapshot_date, + started_by_upload_id=upload.id, + ) + rows_count_by_registry[organization.registry_id] += 1 + + for registry_id, upload in uploads_by_registry.items(): + RegisterUpload.objects.using(db_alias).filter(id=upload.id).update( + rows_count=rows_count_by_registry.get(registry_id, 0) + ) + + if duplicate_organization_ids: + Organization.objects.using(db_alias).filter( + id__in=duplicate_organization_ids + ).delete() + + +def reverse_migrate_legacy_organization_memberships(apps, schema_editor): + """Обратный перенос не поддерживается (noop).""" + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("registers", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="RegisterUpload", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Дата и время создания записи", + verbose_name="создано", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="Дата и время последнего обновления", + verbose_name="обновлено", + ), + ), + ( + "actual_date", + models.DateField( + db_index=True, + help_text="Дата среза реестра, к которой относится загрузка", + verbose_name="дата актуальности", + ), + ), + ( + "file_name", + models.CharField( + help_text="Оригинальное имя загруженного файла", + max_length=255, + verbose_name="имя файла", + ), + ), + ( + "file_hash", + models.CharField( + db_index=True, + help_text="SHA-256 хеш загруженного файла", + max_length=64, + verbose_name="хеш файла", + ), + ), + ( + "rows_count", + models.PositiveIntegerField( + default=0, + help_text="Количество строк организаций в файле", + verbose_name="количество строк", + ), + ), + ], + options={ + "verbose_name": "загрузка реестра", + "verbose_name_plural": "загрузки реестров", + "db_table": "registers_upload", + "ordering": ["-actual_date", "-created_at"], + }, + ), + migrations.CreateModel( + name="RegistryMembershipPeriod", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Дата и время создания записи", + verbose_name="создано", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="Дата и время последнего обновления", + verbose_name="обновлено", + ), + ), + ( + "started_at", + models.DateField( + db_index=True, + help_text="Дата, с которой организация входит в реестр", + verbose_name="дата входа", + ), + ), + ( + "ended_at", + models.DateField( + blank=True, + db_index=True, + help_text="Дата, с которой организация больше не входит в реестр", + null=True, + verbose_name="дата выхода", + ), + ), + ], + options={ + "verbose_name": "период участия", + "verbose_name_plural": "периоды участия", + "db_table": "registers_membership_period", + "ordering": ["-started_at", "registry_id"], + }, + ), + migrations.AddField( + model_name="registerupload", + name="registry", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="uploads", + to="registers.register", + verbose_name="реестр", + ), + ), + migrations.AddField( + model_name="registerupload", + name="uploaded_by", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="register_uploads", + to=settings.AUTH_USER_MODEL, + verbose_name="загружено пользователем", + ), + ), + migrations.AddField( + model_name="registrymembershipperiod", + name="ended_by_upload", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="ended_periods", + to="registers.registerupload", + verbose_name="загрузка выхода", + ), + ), + migrations.AddField( + model_name="registrymembershipperiod", + name="organization", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="membership_periods", + to="registers.organization", + verbose_name="организация", + ), + ), + migrations.AddField( + model_name="registrymembershipperiod", + name="registry", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="membership_periods", + to="registers.register", + verbose_name="реестр", + ), + ), + migrations.AddField( + model_name="registrymembershipperiod", + name="started_by_upload", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="started_periods", + to="registers.registerupload", + verbose_name="загрузка входа", + ), + ), + migrations.RunPython( + migrate_legacy_organization_memberships, + reverse_migrate_legacy_organization_memberships, + ), + migrations.RemoveIndex( + model_name="organization", + name="registers_o_registr_e00175_idx", + ), + migrations.RemoveIndex( + model_name="organization", + name="registers_o_registr_6ab4dd_idx", + ), + migrations.RemoveIndex( + model_name="organization", + name="registers_o_registr_629dae_idx", + ), + migrations.RemoveField( + model_name="organization", + name="registry", + ), + migrations.AddIndex( + model_name="organization", + index=models.Index( + fields=["mn_ogrn", "mn_inn"], + name="registers_o_mn_ogrn_d3d935_idx", + ), + ), + migrations.AddIndex( + model_name="organization", + index=models.Index( + fields=["mn_okpo"], + name="registers_o_mn_okpo_ffb72b_idx", + ), + ), + migrations.AddConstraint( + model_name="organization", + constraint=models.UniqueConstraint( + fields=("mn_ogrn", "mn_inn"), + name="unique_registers_organization_identity", + ), + ), + migrations.AddIndex( + model_name="registrymembershipperiod", + index=models.Index( + fields=["registry", "started_at"], + name="registers_m_registr_3292a6_idx", + ), + ), + migrations.AddIndex( + model_name="registrymembershipperiod", + index=models.Index( + fields=["registry", "ended_at"], + name="registers_m_registr_edbdd9_idx", + ), + ), + migrations.AddIndex( + model_name="registrymembershipperiod", + index=models.Index( + fields=["organization", "started_at"], + name="registers_m_organiz_138ba3_idx", + ), + ), + migrations.AddConstraint( + model_name="registrymembershipperiod", + constraint=models.CheckConstraint( + check=models.Q(("ended_at__isnull", True), ("ended_at__gte", django.db.models.expressions.F("started_at"),), _connector="OR"), + name="check_membership_period_dates", + ), + ), + migrations.AddIndex( + model_name="registerupload", + index=models.Index( + fields=["registry", "actual_date"], + name="registers_u_registr_e67ec5_idx", + ), + ), + migrations.AddIndex( + model_name="registerupload", + index=models.Index( + fields=["registry", "file_hash"], + name="registers_u_registr_4a5919_idx", + ), + ), + ] diff --git a/src/apps/registers/migrations/0003_add_unique_active_membership_period.py b/src/apps/registers/migrations/0003_add_unique_active_membership_period.py new file mode 100644 index 0000000..2ac1ebd --- /dev/null +++ b/src/apps/registers/migrations/0003_add_unique_active_membership_period.py @@ -0,0 +1,55 @@ +# Generated by Django 3.2.25 on 2026-03-04 + +from django.db import migrations, models +from django.db.models import Count + + +def deduplicate_active_membership_periods(apps, schema_editor): + RegistryMembershipPeriod = apps.get_model("registers", "RegistryMembershipPeriod") + db_alias = schema_editor.connection.alias + + duplicate_groups = ( + RegistryMembershipPeriod.objects.using(db_alias) + .filter(ended_at__isnull=True) + .values("registry_id", "organization_id") + .annotate(active_count=Count("id")) + .filter(active_count__gt=1) + ) + + for group in duplicate_groups: + period_ids = list( + RegistryMembershipPeriod.objects.using(db_alias) + .filter( + registry_id=group["registry_id"], + organization_id=group["organization_id"], + ended_at__isnull=True, + ) + .order_by("-started_at", "-id") + .values_list("id", flat=True) + ) + if len(period_ids) <= 1: + continue + RegistryMembershipPeriod.objects.using(db_alias).filter( + id__in=period_ids[1:] + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("registers", "0002_auto_20260304_1038"), + ] + + operations = [ + migrations.RunPython( + deduplicate_active_membership_periods, + migrations.RunPython.noop, + ), + migrations.AddConstraint( + model_name="registrymembershipperiod", + constraint=models.UniqueConstraint( + fields=("registry", "organization"), + condition=models.Q(ended_at__isnull=True), + name="unique_active_membership_period", + ), + ), + ] diff --git a/src/apps/registers/migrations/__init__.py b/src/apps/registers/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/apps/registers/models.py b/src/apps/registers/models.py new file mode 100644 index 0000000..26e4217 --- /dev/null +++ b/src/apps/registers/models.py @@ -0,0 +1,208 @@ +"""Модели приложения реестров организаций.""" + +from apps.core.mixins import TimestampMixin, UUIDPrimaryKeyMixin +from django.conf import settings +from django.core.validators import RegexValidator +from django.db import models +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ + + +class Register(UUIDPrimaryKeyMixin, TimestampMixin, models.Model): + """Справочник реестров.""" + + name = models.CharField( + _("название"), + max_length=255, + unique=True, + help_text=_("Человекочитаемое название реестра"), + ) + + class Meta: + db_table = "registers_register" + verbose_name = _("реестр") + verbose_name_plural = _("реестры") + ordering = ["name"] + + def __str__(self) -> str: + return self.name + + +class Organization(TimestampMixin, models.Model): + """Каноническая организация для всех реестров.""" + + pn_name = models.TextField( + _("наименование"), + help_text=_("Полное наименование организации"), + ) + mn_ogrn = models.BigIntegerField( + _("ОГРН"), + db_index=True, + help_text=_("ОГРН организации"), + ) + mn_inn = models.BigIntegerField( + _("ИНН"), + db_index=True, + help_text=_("ИНН организации"), + ) + in_kpp = models.BigIntegerField( + _("КПП"), + db_index=True, + null=True, + blank=True, + help_text=_("КПП организации"), + ) + mn_okpo = models.TextField( + _("ОКПО"), + db_index=True, + validators=[ + RegexValidator( + regex=r"^\d+$", + message=_("ОКПО должен содержать только цифры"), + ) + ], + help_text=_("ОКПО (текст с обязательной числовой валидацией)"), + ) + + class Meta: + db_table = "registers_organization" + verbose_name = _("организация") + verbose_name_plural = _("организации") + ordering = ["pn_name"] + indexes = [ + models.Index(fields=["mn_ogrn", "mn_inn"]), + models.Index(fields=["mn_okpo"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["mn_ogrn", "mn_inn"], + name="unique_registers_organization_identity", + ), + ] + + def __str__(self) -> str: + return f"{self.pn_name[:60]} ({self.mn_ogrn}/{self.mn_inn})" + + +class RegisterUpload(TimestampMixin, models.Model): + """Факт загрузки снимка реестра на конкретную дату актуальности.""" + + registry = models.ForeignKey( + Register, + on_delete=models.CASCADE, + related_name="uploads", + verbose_name=_("реестр"), + ) + actual_date = models.DateField( + _("дата актуальности"), + db_index=True, + help_text=_("Дата среза реестра, к которой относится загрузка"), + ) + file_name = models.CharField( + _("имя файла"), + max_length=255, + help_text=_("Оригинальное имя загруженного файла"), + ) + file_hash = models.CharField( + _("хеш файла"), + max_length=64, + db_index=True, + help_text=_("SHA-256 хеш загруженного файла"), + ) + rows_count = models.PositiveIntegerField( + _("количество строк"), + default=0, + help_text=_("Количество строк организаций в файле"), + ) + uploaded_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="register_uploads", + verbose_name=_("загружено пользователем"), + ) + + class Meta: + db_table = "registers_upload" + verbose_name = _("загрузка реестра") + verbose_name_plural = _("загрузки реестров") + ordering = ["-actual_date", "-created_at"] + indexes = [ + models.Index(fields=["registry", "actual_date"]), + models.Index(fields=["registry", "file_hash"]), + ] + + def __str__(self) -> str: + return f"{self.registry.name} @ {self.actual_date}" + + +class RegistryMembershipPeriod(TimestampMixin, models.Model): + """Период присутствия организации в конкретном реестре.""" + + registry = models.ForeignKey( + Register, + on_delete=models.CASCADE, + related_name="membership_periods", + verbose_name=_("реестр"), + ) + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + related_name="membership_periods", + verbose_name=_("организация"), + ) + started_at = models.DateField( + _("дата входа"), + db_index=True, + help_text=_("Дата, с которой организация входит в реестр"), + ) + ended_at = models.DateField( + _("дата выхода"), + null=True, + blank=True, + db_index=True, + help_text=_("Дата, с которой организация больше не входит в реестр"), + ) + started_by_upload = models.ForeignKey( + RegisterUpload, + on_delete=models.PROTECT, + related_name="started_periods", + verbose_name=_("загрузка входа"), + ) + ended_by_upload = models.ForeignKey( + RegisterUpload, + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="ended_periods", + verbose_name=_("загрузка выхода"), + ) + + class Meta: + db_table = "registers_membership_period" + verbose_name = _("период участия") + verbose_name_plural = _("периоды участия") + ordering = ["-started_at", "registry_id"] + indexes = [ + models.Index(fields=["registry", "started_at"]), + models.Index(fields=["registry", "ended_at"]), + models.Index(fields=["organization", "started_at"]), + ] + constraints = [ + models.CheckConstraint( + check=Q(ended_at__isnull=True) | Q(ended_at__gte=models.F("started_at")), + name="check_membership_period_dates", + ), + models.UniqueConstraint( + fields=["registry", "organization"], + condition=Q(ended_at__isnull=True), + name="unique_active_membership_period", + ), + ] + + def __str__(self) -> str: + return ( + f"{self.registry.name}: {self.organization.pn_name[:40]} " + f"[{self.started_at} - {self.ended_at or '...'}]" + ) diff --git a/src/apps/registers/pagination.py b/src/apps/registers/pagination.py new file mode 100644 index 0000000..42de890 --- /dev/null +++ b/src/apps/registers/pagination.py @@ -0,0 +1,29 @@ +"""Pagination helpers for registers app.""" + +from apps.core.pagination import StandardPagination + + +class RegistersPagination(StandardPagination): + """ + Пагинация для реестров с поддержкой meta.actual_date. + + Если view выставляет атрибут `actual_date_meta`, он будет добавлен + в метаданные ответа. + """ + + _view = None + + def paginate_queryset(self, queryset, request, view=None): + self._view = view + return super().paginate_queryset(queryset, request, view=view) + + def get_paginated_response(self, data): + response = super().get_paginated_response(data) + + actual_date = getattr(self._view, "actual_date_meta", None) + if actual_date: + meta = response.data.get("meta") or {} + meta["actual_date"] = actual_date.isoformat() + response.data["meta"] = meta + + return response diff --git a/src/apps/registers/serializers.py b/src/apps/registers/serializers.py new file mode 100644 index 0000000..311165c --- /dev/null +++ b/src/apps/registers/serializers.py @@ -0,0 +1,122 @@ +"""Сериализаторы для API реестров.""" + +from apps.registers.models import ( + Organization, + Register, + RegistryMembershipPeriod, +) +from rest_framework import serializers + + +class RegisterSerializer(serializers.ModelSerializer): + """Сериализатор реестра.""" + + class Meta: + model = Register + fields = ["id", "name", "created_at", "updated_at"] + read_only_fields = fields + + +class RegistryMembershipPeriodSerializer(serializers.ModelSerializer): + """Сериализатор периода участия организации в реестре.""" + + registry_id = serializers.UUIDField(source="registry.id", read_only=True) + registry_name = serializers.CharField(source="registry.name", read_only=True) + + class Meta: + model = RegistryMembershipPeriod + fields = [ + "id", + "registry_id", + "registry_name", + "started_at", + "ended_at", + ] + read_only_fields = fields + + +class OrganizationSerializer(serializers.ModelSerializer): + """Сериализатор организации.""" + + class Meta: + model = Organization + fields = [ + "id", + "pn_name", + "mn_ogrn", + "mn_inn", + "in_kpp", + "mn_okpo", + "created_at", + "updated_at", + ] + read_only_fields = fields + + +class OrganizationDetailSerializer(OrganizationSerializer): + """Организация с историей участия в реестрах.""" + + periods = RegistryMembershipPeriodSerializer( + source="membership_periods", + many=True, + read_only=True, + ) + + class Meta(OrganizationSerializer.Meta): + fields = OrganizationSerializer.Meta.fields + ["periods"] + + +class RegisterFileUploadSerializer(serializers.Serializer): + """Сериализатор загрузки файла реестра.""" + + registry = serializers.PrimaryKeyRelatedField(queryset=Register.objects.all()) + file = serializers.FileField() + actual_date = serializers.DateField(required=False) + + def validate_file(self, value): + if not value.name.lower().endswith(".xlsx"): + raise serializers.ValidationError("Поддерживаются только файлы .xlsx") + return value + + +class OrganizationListQuerySerializer(serializers.Serializer): + """Сериализатор query-параметров списка организаций.""" + + registry = serializers.PrimaryKeyRelatedField( + queryset=Register.objects.all(), + required=False, + ) + actual_date = serializers.DateField(required=False) + search = serializers.CharField(required=False, allow_blank=True) + mn_ogrn = serializers.IntegerField(required=False, min_value=0) + mn_inn = serializers.IntegerField(required=False, min_value=0) + in_kpp = serializers.IntegerField(required=False, min_value=0) + mn_okpo = serializers.CharField(required=False, allow_blank=False) + + def validate(self, attrs): + if attrs.get("actual_date") and not attrs.get("registry"): + raise serializers.ValidationError( + {"actual_date": "Параметр actual_date допустим только вместе с registry"} + ) + return attrs + + def validate_mn_okpo(self, value): + if not value.isdigit(): + raise serializers.ValidationError("mn_okpo должен содержать только цифры") + return value + + +class RegistryOrganizationListQuerySerializer(serializers.Serializer): + """Сериализатор query-параметров списка организаций конкретного реестра.""" + + actual_date = serializers.DateField(required=False) + search = serializers.CharField(required=False, allow_blank=True) + mn_ogrn = serializers.IntegerField(required=False, min_value=0) + mn_inn = serializers.IntegerField(required=False, min_value=0) + in_kpp = serializers.IntegerField(required=False, min_value=0) + mn_okpo = serializers.CharField(required=False, allow_blank=False) + + def validate_mn_okpo(self, value): + if not value.isdigit(): + raise serializers.ValidationError("mn_okpo должен содержать только цифры") + return value diff --git a/src/apps/registers/services.py b/src/apps/registers/services.py new file mode 100644 index 0000000..50793f6 --- /dev/null +++ b/src/apps/registers/services.py @@ -0,0 +1,598 @@ +"""Сервисы загрузки и обработки реестров организаций.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from datetime import date + +from apps.registers.models import ( + Organization, + Register, + RegisterUpload, + RegistryMembershipPeriod, +) +from django.db import transaction +from django.db.models import CharField, Q +from django.db.models.functions import Cast +from django.utils import timezone +from openpyxl import load_workbook + + +class RegisterImportError(ValueError): + """Ошибка импорта Excel файла реестра.""" + + +@dataclass(frozen=True) +class ParsedOrganization: + """Промежуточная структура организации после парсинга Excel.""" + + pn_name: str + mn_ogrn: int + mn_inn: int + in_kpp: int | None + mn_okpo: str + + +class RegisterImportService: + """Сервис импорта организаций из Excel в выбранный реестр.""" + + REQUIRED_HEADERS = {"pn_name", "mn_ogrn", "mn_inn", "mn_okpo"} + + @classmethod + @transaction.atomic + def sync_registry_memberships( + cls, + *, + registry: Register, + uploaded_file, + actual_date: date | None = None, + uploaded_by=None, + ) -> dict[str, int | str]: + """ + Импортировать снимок реестра на дату актуальности. + + Алгоритм: + 1. Парсим файл и upsert-им канонические организации. + 2. Закрываем активные периоды организаций, которых нет в новом снимке. + 3. Открываем периоды для организаций, которых не было среди активных. + """ + snapshot_date = actual_date or timezone.localdate() + + parsed_rows = cls.parse_xlsx(uploaded_file) + rows = cls._ensure_unique_identities(parsed_rows) + file_hash = cls._calculate_file_hash(uploaded_file) + + cls._validate_snapshot_date(registry=registry, snapshot_date=snapshot_date) + + upload = RegisterUpload.objects.create( + registry=registry, + actual_date=snapshot_date, + file_name=uploaded_file.name, + file_hash=file_hash, + rows_count=len(rows), + uploaded_by=uploaded_by, + ) + + ( + snapshot_org_ids, + organizations_created, + organizations_updated, + ) = cls._upsert_organizations(rows) + + active_by_org = cls._get_active_periods_by_org(registry) + active_org_ids = set(active_by_org.keys()) + + closed_periods = cls._close_missing_periods( + active_by_org=active_by_org, + snapshot_org_ids=snapshot_org_ids, + snapshot_date=snapshot_date, + upload=upload, + ) + + opened_periods = cls._open_new_periods( + registry=registry, + snapshot_org_ids=snapshot_org_ids, + active_org_ids=active_org_ids, + snapshot_date=snapshot_date, + upload=upload, + ) + + active_periods_count = RegistryMembershipPeriod.objects.filter( + registry=registry, + ended_at__isnull=True, + ).count() + + return { + "upload_id": upload.id, + "registry_id": str(registry.id), + "registry_name": registry.name, + "actual_date": snapshot_date.isoformat(), + "rows_in_file": len(rows), + "organizations_created": organizations_created, + "organizations_updated": organizations_updated, + "opened_periods": opened_periods, + "closed_periods": closed_periods, + "active_periods": active_periods_count, + } + + @classmethod + def _upsert_organizations( + cls, + rows: list[ParsedOrganization], + ) -> tuple[set[int], int, int]: + snapshot_org_ids: set[int] = set() + organizations_created = 0 + organizations_updated = 0 + + for row in rows: + organization, created = Organization.objects.get_or_create( + mn_ogrn=row.mn_ogrn, + mn_inn=row.mn_inn, + defaults={ + "pn_name": row.pn_name, + "in_kpp": row.in_kpp, + "mn_okpo": row.mn_okpo, + }, + ) + + if created: + organizations_created += 1 + else: + updated = cls._update_organization_fields(organization=organization, row=row) + if updated: + organizations_updated += 1 + + snapshot_org_ids.add(organization.id) + + return snapshot_org_ids, organizations_created, organizations_updated + + @classmethod + def _update_organization_fields( + cls, + *, + organization: Organization, + row: ParsedOrganization, + ) -> bool: + update_fields: list[str] = [] + + if organization.pn_name != row.pn_name: + organization.pn_name = row.pn_name + update_fields.append("pn_name") + + if organization.in_kpp != row.in_kpp: + organization.in_kpp = row.in_kpp + update_fields.append("in_kpp") + + if organization.mn_okpo != row.mn_okpo: + organization.mn_okpo = row.mn_okpo + update_fields.append("mn_okpo") + + if not update_fields: + return False + + organization.save(update_fields=update_fields + ["updated_at"]) + return True + + @classmethod + def _get_active_periods_by_org( + cls, + registry: Register, + ) -> dict[int, RegistryMembershipPeriod]: + active_periods = ( + RegistryMembershipPeriod.objects.select_for_update() + .filter(registry=registry, ended_at__isnull=True) + .only("id", "organization_id", "started_at") + ) + return {period.organization_id: period for period in active_periods} + + @classmethod + def _close_missing_periods( + cls, + *, + active_by_org: dict[int, RegistryMembershipPeriod], + snapshot_org_ids: set[int], + snapshot_date: date, + upload: RegisterUpload, + ) -> int: + close_by_date_ids: list[int] = [] + close_by_delete_ids: list[int] = [] + + for organization_id, period in active_by_org.items(): + if organization_id in snapshot_org_ids: + continue + + # Если загружали этот же snapshot_date повторно, удаляем нулевой период. + if period.started_at == snapshot_date: + close_by_delete_ids.append(period.id) + else: + close_by_date_ids.append(period.id) + + closed_periods = 0 + + if close_by_date_ids: + closed_periods += RegistryMembershipPeriod.objects.filter( + id__in=close_by_date_ids + ).update( + ended_at=snapshot_date, + ended_by_upload=upload, + ) + + if close_by_delete_ids: + deleted_count, _ = RegistryMembershipPeriod.objects.filter( + id__in=close_by_delete_ids + ).delete() + closed_periods += deleted_count + + return closed_periods + + @classmethod + def _open_new_periods( + cls, + *, + registry: Register, + snapshot_org_ids: set[int], + active_org_ids: set[int], + snapshot_date: date, + upload: RegisterUpload, + ) -> int: + to_open_org_ids = snapshot_org_ids - active_org_ids + if not to_open_org_ids: + return 0 + + RegistryMembershipPeriod.objects.bulk_create( + [ + RegistryMembershipPeriod( + registry=registry, + organization_id=organization_id, + started_at=snapshot_date, + started_by_upload=upload, + ) + for organization_id in to_open_org_ids + ], + batch_size=1000, + ignore_conflicts=True, + ) + + return RegistryMembershipPeriod.objects.filter( + started_by_upload=upload, + ).count() + + @classmethod + def resolve_actual_date( + cls, + *, + registry: Register, + requested_date: date | None, + ) -> date: + """Вернуть effective дату среза: запрошенную или последнюю доступную.""" + if requested_date: + return requested_date + + latest_upload = ( + RegisterUpload.objects.filter(registry=registry) + .order_by("-actual_date", "-id") + .first() + ) + + if latest_upload: + return latest_upload.actual_date + + return timezone.localdate() + + @classmethod + def get_organizations_queryset( + cls, + *, + registry: Register | None = None, + actual_date: date | None = None, + search: str = "", + mn_ogrn: int | None = None, + mn_inn: int | None = None, + in_kpp: int | None = None, + mn_okpo: str | None = None, + ) -> tuple: + """Получить queryset организаций с учетом фильтров и среза по реестру.""" + queryset = Organization.objects.all().order_by("pn_name") + resolved_actual_date = None + + if registry: + resolved_actual_date = cls.resolve_actual_date( + registry=registry, + requested_date=actual_date, + ) + queryset = cls._filter_organizations_active_in_registry( + queryset=queryset, + registry=registry, + actual_date=resolved_actual_date, + ).distinct() + + queryset = cls._apply_exact_filters( + queryset, + mn_ogrn=mn_ogrn, + mn_inn=mn_inn, + in_kpp=in_kpp, + mn_okpo=mn_okpo, + ) + queryset = cls._apply_search(queryset, search.strip()) + + return queryset, resolved_actual_date + + @classmethod + def get_registry_organizations_queryset( + cls, + *, + registry: Register, + actual_date: date | None = None, + search: str = "", + mn_ogrn: int | None = None, + mn_inn: int | None = None, + in_kpp: int | None = None, + mn_okpo: str | None = None, + ) -> tuple: + """Получить queryset организаций конкретного реестра на дату.""" + resolved_actual_date = cls.resolve_actual_date( + registry=registry, + requested_date=actual_date, + ) + + queryset = cls._filter_organizations_active_in_registry( + queryset=Organization.objects.all().order_by("pn_name"), + registry=registry, + actual_date=resolved_actual_date, + ).distinct() + queryset = cls._apply_exact_filters( + queryset, + mn_ogrn=mn_ogrn, + mn_inn=mn_inn, + in_kpp=in_kpp, + mn_okpo=mn_okpo, + ) + queryset = cls._apply_search(queryset, search.strip()) + + return queryset, resolved_actual_date + + @classmethod + def _filter_organizations_active_in_registry( + cls, + *, + queryset, + registry: Register, + actual_date: date, + ): + return queryset.filter( + membership_periods__registry=registry, + membership_periods__started_at__lte=actual_date, + ).filter( + Q(membership_periods__ended_at__isnull=True) + | Q(membership_periods__ended_at__gt=actual_date) + ) + + @classmethod + def _apply_exact_filters( + cls, + queryset, + *, + mn_ogrn: int | None, + mn_inn: int | None, + in_kpp: int | None, + mn_okpo: str | None, + ): + if mn_ogrn is not None: + queryset = queryset.filter(mn_ogrn=mn_ogrn) + if mn_inn is not None: + queryset = queryset.filter(mn_inn=mn_inn) + if in_kpp is not None: + queryset = queryset.filter(in_kpp=in_kpp) + if mn_okpo: + queryset = queryset.filter(mn_okpo=mn_okpo) + + return queryset + + @classmethod + def _apply_search(cls, queryset, search_query: str): + if not search_query: + return queryset + + return queryset.annotate( + mn_ogrn_text=Cast("mn_ogrn", output_field=CharField()), + mn_inn_text=Cast("mn_inn", output_field=CharField()), + in_kpp_text=Cast("in_kpp", output_field=CharField()), + ).filter( + Q(pn_name__icontains=search_query) + | Q(mn_ogrn_text__icontains=search_query) + | Q(mn_inn_text__icontains=search_query) + | Q(in_kpp_text__icontains=search_query) + | Q(mn_okpo__icontains=search_query) + ) + + @classmethod + def parse_xlsx(cls, uploaded_file) -> list[ParsedOrganization]: + try: + uploaded_file.seek(0) + workbook = load_workbook(uploaded_file, read_only=True, data_only=True) + except Exception as exc: # noqa: BLE001 + raise RegisterImportError("Не удалось прочитать Excel файл") from exc + + try: + worksheet = workbook.active + row_iter = worksheet.iter_rows(min_row=1, values_only=True) + header_row = next(row_iter, None) + + if not header_row: + raise RegisterImportError("Файл не содержит заголовков") + + header_index_map = cls._build_header_index_map(header_row) + cls._validate_headers(header_index_map) + + organizations: list[ParsedOrganization] = [] + for row_number, row_values in enumerate(row_iter, start=2): + if cls._is_empty_row(row_values): + continue + + organizations.append( + ParsedOrganization( + pn_name=cls._as_required_text( + row_values[header_index_map["pn_name"]], + field_name="pn_name", + row_number=row_number, + ), + mn_ogrn=cls._as_required_int( + row_values[header_index_map["mn_ogrn"]], + field_name="mn_ogrn", + row_number=row_number, + ), + mn_inn=cls._as_required_int( + row_values[header_index_map["mn_inn"]], + field_name="mn_inn", + row_number=row_number, + ), + in_kpp=cls._as_optional_int( + row_values[header_index_map["in_kpp"]], + field_name="in_kpp", + row_number=row_number, + ) + if "in_kpp" in header_index_map + else None, + mn_okpo=cls._as_numeric_text( + row_values[header_index_map["mn_okpo"]], + field_name="mn_okpo", + row_number=row_number, + ), + ) + ) + + if not organizations: + raise RegisterImportError("Файл не содержит строк с организациями") + + return organizations + finally: + workbook.close() + + @classmethod + def _validate_snapshot_date(cls, *, registry: Register, snapshot_date: date) -> None: + latest_upload = ( + RegisterUpload.objects.filter(registry=registry) + .order_by("-actual_date", "-id") + .first() + ) + if latest_upload and snapshot_date < latest_upload.actual_date: + raise RegisterImportError( + "Дата актуальности не может быть раньше последней загрузки " + f"({latest_upload.actual_date.isoformat()})" + ) + + @classmethod + def _calculate_file_hash(cls, uploaded_file) -> str: + uploaded_file.seek(0) + file_content = uploaded_file.read() + uploaded_file.seek(0) + return hashlib.sha256(file_content).hexdigest() + + @classmethod + def _ensure_unique_identities( + cls, + rows: list[ParsedOrganization], + ) -> list[ParsedOrganization]: + unique_rows: list[ParsedOrganization] = [] + seen_keys: set[tuple[int, int]] = set() + + for row in rows: + key = (row.mn_ogrn, row.mn_inn) + if key in seen_keys: + raise RegisterImportError( + "Файл содержит дубли по ключу (mn_ogrn, mn_inn): " + f"{row.mn_ogrn}, {row.mn_inn}" + ) + seen_keys.add(key) + unique_rows.append(row) + + return unique_rows + + @classmethod + def _build_header_index_map(cls, header_row: tuple) -> dict[str, int]: + header_index_map: dict[str, int] = {} + for index, value in enumerate(header_row): + normalized = cls._normalize_header(value) + if normalized: + header_index_map[normalized] = index + return header_index_map + + @classmethod + def _validate_headers(cls, header_index_map: dict[str, int]) -> None: + missing = sorted(cls.REQUIRED_HEADERS - set(header_index_map.keys())) + if missing: + raise RegisterImportError( + f"Отсутствуют обязательные колонки: {', '.join(missing)}" + ) + + @staticmethod + def _normalize_header(value) -> str: + return str(value or "").strip().lower() + + @staticmethod + def _is_empty_row(row_values: tuple) -> bool: + for value in row_values: + if value is not None and str(value).strip() != "": + return False + return True + + @classmethod + def _as_required_text(cls, value, *, field_name: str, row_number: int) -> str: + text_value = str(value or "").strip() + if not text_value: + raise RegisterImportError( + f"Строка {row_number}: поле {field_name} обязательно" + ) + return text_value + + @classmethod + def _as_required_int(cls, value, *, field_name: str, row_number: int) -> int: + parsed = cls._as_optional_int( + value, + field_name=field_name, + row_number=row_number, + ) + if parsed is None: + raise RegisterImportError( + f"Строка {row_number}: поле {field_name} обязательно" + ) + return parsed + + @classmethod + def _as_optional_int(cls, value, *, field_name: str, row_number: int) -> int | None: + if value is None: + return None + + text_value = str(value).strip().replace(" ", "") + if not text_value: + return None + + if text_value.endswith(".0"): + text_value = text_value[:-2] + + if not text_value.isdigit(): + raise RegisterImportError( + f"Строка {row_number}: поле {field_name} должно быть числом" + ) + + return int(text_value) + + @classmethod + def _as_numeric_text(cls, value, *, field_name: str, row_number: int) -> str: + text_value = str(value or "").strip().replace(" ", "") + + if text_value.endswith(".0") and text_value[:-2].isdigit(): + text_value = text_value[:-2] + + if not text_value: + raise RegisterImportError( + f"Строка {row_number}: поле {field_name} обязательно" + ) + + if not text_value.isdigit(): + raise RegisterImportError( + f"Строка {row_number}: поле {field_name} должно содержать только цифры" + ) + + return text_value diff --git a/src/apps/registers/urls.py b/src/apps/registers/urls.py new file mode 100644 index 0000000..61a07b9 --- /dev/null +++ b/src/apps/registers/urls.py @@ -0,0 +1,28 @@ +"""URL конфигурация для приложения реестров.""" + +from apps.registers.views import ( + OrganizationViewSet, + RegisterUploadView, + RegisterViewSet, + RegistryOrganizationListView, +) +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +app_name = "registers" + +router = DefaultRouter() +router.register(r"registries", RegisterViewSet, basename="registries") +router.register(r"organizations", OrganizationViewSet, basename="organizations") + +registers_urlpatterns = [ + path("upload/", RegisterUploadView.as_view(), name="register-upload"), + path( + "registries//organizations/", + RegistryOrganizationListView.as_view(), + name="registry-organizations-list", + ), + path("", include(router.urls)), +] + +urlpatterns = [] diff --git a/src/apps/registers/views.py b/src/apps/registers/views.py new file mode 100644 index 0000000..bdda439 --- /dev/null +++ b/src/apps/registers/views.py @@ -0,0 +1,369 @@ +"""Views для работы с реестрами организаций.""" + +from __future__ import annotations + +from datetime import date + +from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag +from apps.registers.models import Organization, Register +from apps.registers.pagination import RegistersPagination +from apps.registers.serializers import ( + OrganizationDetailSerializer, + OrganizationListQuerySerializer, + OrganizationSerializer, + RegisterFileUploadSerializer, + RegisterSerializer, + RegistryOrganizationListQuerySerializer, +) +from apps.registers.services import RegisterImportError, RegisterImportService +from django.shortcuts import get_object_or_404 +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework.exceptions import ValidationError +from rest_framework.generics import ListAPIView +from rest_framework.parsers import MultiPartParser +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework.viewsets import ReadOnlyModelViewSet + +REGISTERS_TAG = swagger_tag("Реестры организаций", "registers") + + +class RegisterViewSet(ReadOnlyModelViewSet): + """API для просмотра списка реестров.""" + + queryset = Register.objects.all().order_by("name") + serializer_class = RegisterSerializer + permission_classes = [IsAuthenticated] + search_fields = ["name"] + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Список реестров", + operation_description="Возвращает список доступных реестров.", + responses={ + 200: RegisterSerializer(many=True), + **ErrorResponses.AUTHENTICATED, + }, + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Детали реестра", + operation_description="Возвращает информацию о конкретном реестре.", + responses={ + 200: RegisterSerializer, + **ErrorResponses.AUTHENTICATED_NOT_FOUND, + }, + ) + def retrieve(self, request, *args, **kwargs): + return super().retrieve(request, *args, **kwargs) + + +class OrganizationViewSet(ReadOnlyModelViewSet): + """ + API для просмотра организаций. + + Поддерживает глобальный поиск по бизнес-полям, а также фильтрацию + по `registry` и `actual_date` (срез организаций в реестре на дату). + """ + + queryset = Organization.objects.all().order_by("pn_name") + serializer_class = OrganizationSerializer + permission_classes = [IsAuthenticated] + pagination_class = RegistersPagination + actual_date_meta: date | None = None + + def get_serializer_class(self): + if self.action == "retrieve": + return OrganizationDetailSerializer + return OrganizationSerializer + + def get_queryset(self): + self.actual_date_meta = None + + if self.action == "retrieve": + return ( + Organization.objects.all() + .order_by("pn_name") + .prefetch_related("membership_periods__registry") + ) + + params_serializer = OrganizationListQuerySerializer(data=self.request.query_params) + params_serializer.is_valid(raise_exception=True) + + queryset, self.actual_date_meta = RegisterImportService.get_organizations_queryset( + **params_serializer.validated_data + ) + + return queryset + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Список организаций", + operation_description=( + "Возвращает список организаций.\n" + "Поддерживает фильтрацию по: mn_ogrn, mn_inn, in_kpp, mn_okpo.\n" + "Поддерживает поиск (search) по: pn_name, mn_ogrn, mn_inn, in_kpp, mn_okpo.\n" + "Опционально можно указать registry и actual_date для среза реестра на дату." + ), + manual_parameters=[ + openapi.Parameter( + name="registry", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + format=openapi.FORMAT_UUID, + required=False, + description="UUID реестра для среза", + ), + openapi.Parameter( + name="actual_date", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, + required=False, + description="Дата актуальности среза (YYYY-MM-DD)", + ), + openapi.Parameter( + name="search", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + required=False, + description="Поиск по текстовым и числовым полям организации", + ), + openapi.Parameter( + name="mn_ogrn", + in_=openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + description="Точное значение ОГРН", + ), + openapi.Parameter( + name="mn_inn", + in_=openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + description="Точное значение ИНН", + ), + openapi.Parameter( + name="in_kpp", + in_=openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + description="Точное значение КПП", + ), + openapi.Parameter( + name="mn_okpo", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + required=False, + description="Точное значение ОКПО", + ), + ], + responses={ + 200: OrganizationSerializer(many=True), + **ErrorResponses.AUTHENTICATED, + }, + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Детали организации", + operation_description="Возвращает информацию об организации и историю её периодов.", + responses={ + 200: OrganizationDetailSerializer, + **ErrorResponses.AUTHENTICATED_NOT_FOUND, + }, + ) + def retrieve(self, request, *args, **kwargs): + return super().retrieve(request, *args, **kwargs) + + +class RegistryOrganizationListView(ListAPIView): + """API списка организаций конкретного реестра на дату актуальности.""" + + serializer_class = OrganizationSerializer + permission_classes = [IsAuthenticated] + pagination_class = RegistersPagination + actual_date_meta: date | None = None + + def _get_registry(self) -> Register: + return get_object_or_404(Register, id=self.kwargs["registry_id"]) + + def get_queryset(self): + self.actual_date_meta = None + registry = self._get_registry() + + params_serializer = RegistryOrganizationListQuerySerializer( + data=self.request.query_params + ) + params_serializer.is_valid(raise_exception=True) + + queryset, self.actual_date_meta = ( + RegisterImportService.get_registry_organizations_queryset( + registry=registry, + **params_serializer.validated_data, + ) + ) + + return queryset + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Список организаций реестра", + operation_description=( + "Возвращает список организаций только для указанного реестра.\n" + "Опциональный параметр actual_date задаёт срез на дату.\n" + "Если actual_date не передан, используется последняя доступная дата загрузки." + ), + manual_parameters=[ + openapi.Parameter( + name="actual_date", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, + required=False, + description="Дата актуальности среза (YYYY-MM-DD)", + ), + openapi.Parameter( + name="search", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + required=False, + description="Поиск по текстовым и числовым полям организации", + ), + openapi.Parameter( + name="mn_ogrn", + in_=openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + description="Точное значение ОГРН", + ), + openapi.Parameter( + name="mn_inn", + in_=openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + description="Точное значение ИНН", + ), + openapi.Parameter( + name="in_kpp", + in_=openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + description="Точное значение КПП", + ), + openapi.Parameter( + name="mn_okpo", + in_=openapi.IN_QUERY, + type=openapi.TYPE_STRING, + required=False, + description="Точное значение ОКПО", + ), + ], + responses={ + 200: OrganizationSerializer(many=True), + **ErrorResponses.AUTHENTICATED, + }, + ) + def get(self, request, *args, **kwargs): + return super().get(request, *args, **kwargs) + + +class RegisterUploadView(APIView): + """API загрузки Excel файла организаций в выбранный реестр.""" + + parser_classes = [MultiPartParser] + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + tags=[REGISTERS_TAG], + operation_summary="Загрузка списка организаций", + operation_description=( + "Загружает Excel (.xlsx) с организациями в выбранный реестр.\n" + "Требуемые колонки: pn_name, mn_ogrn, mn_inn, mn_okpo.\n" + "Опциональная колонка: in_kpp.\n" + "actual_date задаёт дату актуальности среза.\n" + "Если организация исчезла из следующего среза, для неё закрывается период участия." + ), + manual_parameters=[ + openapi.Parameter( + name="registry", + in_=openapi.IN_FORM, + type=openapi.TYPE_STRING, + format=openapi.FORMAT_UUID, + required=True, + description="UUID выбранного реестра", + ), + openapi.Parameter( + name="actual_date", + in_=openapi.IN_FORM, + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, + required=False, + description="Дата актуальности (YYYY-MM-DD)", + ), + openapi.Parameter( + name="file", + in_=openapi.IN_FORM, + type=openapi.TYPE_FILE, + required=True, + description="Excel файл с организациями", + ), + ], + consumes=["multipart/form-data"], + responses={ + 201: openapi.Response( + description="Список организаций успешно загружен", + schema=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "upload_id": openapi.Schema(type=openapi.TYPE_INTEGER), + "registry_id": openapi.Schema(type=openapi.TYPE_STRING), + "registry_name": openapi.Schema(type=openapi.TYPE_STRING), + "actual_date": openapi.Schema( + type=openapi.TYPE_STRING, + format=openapi.FORMAT_DATE, + ), + "rows_in_file": openapi.Schema(type=openapi.TYPE_INTEGER), + "organizations_created": openapi.Schema( + type=openapi.TYPE_INTEGER + ), + "organizations_updated": openapi.Schema( + type=openapi.TYPE_INTEGER + ), + "opened_periods": openapi.Schema(type=openapi.TYPE_INTEGER), + "closed_periods": openapi.Schema(type=openapi.TYPE_INTEGER), + "active_periods": openapi.Schema(type=openapi.TYPE_INTEGER), + }, + ), + ), + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.AUTHENTICATED, + }, + ) + def post(self, request): + serializer = RegisterFileUploadSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + registry = serializer.validated_data["registry"] + uploaded_file = serializer.validated_data["file"] + actual_date = serializer.validated_data.get("actual_date") + + try: + result = RegisterImportService.sync_registry_memberships( + registry=registry, + uploaded_file=uploaded_file, + actual_date=actual_date, + uploaded_by=request.user, + ) + except RegisterImportError as exc: + raise ValidationError({"file": str(exc)}) from exc + + return Response(result, status=status.HTTP_201_CREATED) diff --git a/src/apps/user/migrations/0005_create_default_admin_superuser.py b/src/apps/user/migrations/0005_create_default_admin_superuser.py new file mode 100644 index 0000000..5c6e322 --- /dev/null +++ b/src/apps/user/migrations/0005_create_default_admin_superuser.py @@ -0,0 +1,52 @@ +# Generated by Django 3.2.25 on 2026-03-04 + +from django.db import migrations + +TARGET_USERNAME = "admin" +TARGET_EMAIL = "1@3.com" +TARGET_PASSWORD = "12345678" # noqa: S105 + + +def create_or_update_admin_superuser(apps, schema_editor): + User = apps.get_model("user", "User") + db_alias = schema_editor.connection.alias + manager = User.objects.using(db_alias) + + email_user = manager.filter(email=TARGET_EMAIL).first() + username_user = manager.filter(username=TARGET_USERNAME).first() + + if email_user and username_user and email_user.pk != username_user.pk: + user = email_user + can_set_username = False + elif email_user: + user = email_user + can_set_username = True + elif username_user: + user = username_user + can_set_username = True + else: + user = User(email=TARGET_EMAIL, username=TARGET_USERNAME) + can_set_username = True + + user.email = TARGET_EMAIL + if can_set_username: + user.username = TARGET_USERNAME + + user.is_staff = True + user.is_superuser = True + user.is_active = True + user.set_password(TARGET_PASSWORD) + user.save(using=db_alias) + + +class Migration(migrations.Migration): + dependencies = [ + ("user", "0004_alter_user_groups"), + ] + + operations = [ + migrations.RunPython( + create_or_update_admin_superuser, + migrations.RunPython.noop, + ), + ] diff --git a/src/apps/user/models.py b/src/apps/user/models.py index ef84f96..03cebb7 100644 --- a/src/apps/user/models.py +++ b/src/apps/user/models.py @@ -50,8 +50,8 @@ class User(AbstractUser): updated_at = models.DateTimeField(_("updated at"), auto_now=True) - USERNAME_FIELD = "email" - REQUIRED_FIELDS = ["username"] + USERNAME_FIELD = "username" + REQUIRED_FIELDS = ["email"] class Meta: db_table = "users" diff --git a/src/apps/user/serializers.py b/src/apps/user/serializers.py index a0f13f4..b6b423e 100644 --- a/src/apps/user/serializers.py +++ b/src/apps/user/serializers.py @@ -104,7 +104,7 @@ class ProfileUpdateSerializer(serializers.ModelSerializer): class LoginSerializer(serializers.Serializer): """Сериализатор для входа""" - email = serializers.EmailField(help_text="Email пользователя") + username = serializers.CharField(help_text="Username пользователя") password = serializers.CharField(help_text="Пароль") diff --git a/src/apps/user/views.py b/src/apps/user/views.py index 50ec604..9da0b7c 100644 --- a/src/apps/user/views.py +++ b/src/apps/user/views.py @@ -8,7 +8,7 @@ from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView -from rest_framework_simplejwt.tokens import RefreshToken +from rest_framework_simplejwt.views import TokenRefreshView as SimpleJWTTokenRefreshView from rest_framework_simplejwt.views import TokenVerifyView as SimpleJWTTokenVerifyView from .serializers import ( @@ -78,7 +78,9 @@ class LoginView(APIView): @swagger_auto_schema( tags=[AUTH_TAG], operation_summary="Вход", - operation_description="Аутентификация по email и паролю. Возвращает JWT токены.", + operation_description=( + "Аутентификация по username и паролю. Возвращает JWT токены." + ), request_body=LoginSerializer, responses={ 200: TokenSerializer, @@ -90,10 +92,10 @@ class LoginView(APIView): def post(self, request): serializer = LoginSerializer(data=request.data) if serializer.is_valid(): - email = serializer.validated_data["email"] + username = serializer.validated_data["username"] password = serializer.validated_data["password"] - user = authenticate(email=email, password=password) + user = authenticate(username=username, password=password) if user: tokens = UserService.get_tokens_for_user(user) return Response(tokens, status=status.HTTP_200_OK) @@ -279,7 +281,7 @@ def user_profile_detail(request): return Response(profile_data) -class TokenRefreshView(APIView): +class TokenRefreshView(SimpleJWTTokenRefreshView): """Обновление access токена через refresh токен.""" permission_classes = [AllowAny] @@ -304,23 +306,8 @@ class TokenRefreshView(APIView): **ErrorResponses.PUBLIC, }, ) - def post(self, request): - refresh_token = request.data.get("refresh") - if not refresh_token: - return Response( - {"error": "Refresh token обязателен"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - try: - refresh = RefreshToken(refresh_token) - return Response( - {"access": str(refresh.access_token), "refresh": str(refresh)} - ) - except Exception: - return Response( - {"error": "Неверный refresh token"}, status=status.HTTP_401_UNAUTHORIZED - ) + def post(self, request, *args, **kwargs): + return super().post(request, *args, **kwargs) class TokenVerifySwaggerView(SimpleJWTTokenVerifyView): diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index 119b82a..d746f88 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -10,10 +10,15 @@ API v1 URL configuration. - /api/v1/proverki/ - Единый реестр проверок - /api/v1/zakupki/ - Государственные закупки - /api/v1/fns/ - ФНС (бухгалтерская отчетность) +- /api/v1/registers/ - Реестры организаций +- /api/v1/exchange/ - Обмен с внешней БД +- /api/v1/backups/ - Экспорт защищённых backup-архивов - /api/v1/system/ - Системные (логи, прокси) - только для админов """ +from apps.backups.urls import backups_urlpatterns from apps.core.views import BackgroundJobListView, BackgroundJobStatusView +from apps.exchange.urls import exchange_urlpatterns from apps.parsers.urls import ( fns_urlpatterns, minpromtorg_urlpatterns, @@ -21,6 +26,7 @@ from apps.parsers.urls import ( system_urlpatterns, zakupki_urlpatterns, ) +from apps.registers.urls import registers_urlpatterns from django.urls import include, path app_name = "api_v1" @@ -44,6 +50,12 @@ urlpatterns = [ path("zakupki/", include((zakupki_urlpatterns, "zakupki"))), # Парсеры - ФНС бухгалтерская отчетность path("fns/", include((fns_urlpatterns, "fns"))), + # Реестры организаций + path("registers/", include((registers_urlpatterns, "registers"))), + # Обмен с внешней БД + path("exchange/", include((exchange_urlpatterns, "exchange"))), + # Backup архивы + path("backups/", include((backups_urlpatterns, "backups"))), # Системные (только админы) path("system/", include((system_urlpatterns, "system"))), ] diff --git a/src/settings/base.py b/src/settings/base.py index 331af90..48d4833 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -40,6 +40,7 @@ INSTALLED_APPS = [ "rest_framework", "django_filters", "corsheaders", + "rest_framework_simplejwt.token_blacklist", "django_celery_beat", "django_celery_results", "drf_yasg", @@ -47,6 +48,9 @@ INSTALLED_APPS = [ "apps.core", "apps.user", "apps.parsers", + "apps.registers", + "apps.exchange", + "apps.backups", ] # Jazzmin Admin Configuration @@ -78,6 +82,9 @@ JAZZMIN_SETTINGS = { "order_with_respect_to": [ "user", "parsers", + "registers", + "exchange", + "backups", "core", "django_celery_beat", ], @@ -91,6 +98,9 @@ JAZZMIN_SETTINGS = { "parsers.ParserLoadLog": "fas fa-history", "parsers.IndustrialCertificateRecord": "fas fa-certificate", "parsers.ManufacturerRecord": "fas fa-industry", + "registers.Register": "fas fa-book", + "registers.Organization": "fas fa-building", + "exchange.ExchangeConnection": "fas fa-database", "core.BackgroundJob": "fas fa-tasks", "django_celery_beat.PeriodicTask": "fas fa-clock", "django_celery_beat.CrontabSchedule": "fas fa-calendar-alt", @@ -188,6 +198,12 @@ WSGI_APPLICATION = "core.wsgi.application" ZAKUPKI_TOKEN = os.getenv("ZAKUPKI_TOKEN", "") FNS_LOCK_TTL_SECONDS = 3600 PARSER_PROXIES = [] +BACKUP_ENCRYPTION_KEY = os.getenv("BACKUP_ENCRYPTION_KEY", "") +BACKUP_KEY_ID = os.getenv("BACKUP_KEY_ID", "default") +BACKUP_EXPORT_DIRECTORY = os.getenv( + "BACKUP_EXPORT_DIRECTORY", + str(PROJECT_ROOT / "media" / "backups"), +) # Password validation diff --git a/src/settings/production.py b/src/settings/production.py index 66bf2eb..b371419 100644 --- a/src/settings/production.py +++ b/src/settings/production.py @@ -5,23 +5,48 @@ Docker Compose сеть - используются имена сервисов ( import os +from django.core.exceptions import ImproperlyConfigured + from .base import * -SECRET_KEY = os.getenv("SECRET_KEY", "production-secret-key-mostovik-change-me-2024") -DEBUG = os.getenv("DEBUG", "False").lower() == "true" -ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", "*").split(",") + +def _require_env(name: str) -> str: + value = os.getenv(name, "").strip() + if not value: + raise ImproperlyConfigured(f"{name} must be set in production") + return value + + +def _parse_allowed_hosts(raw_value: str) -> list[str]: + hosts = [host.strip() for host in raw_value.split(",") if host.strip()] + if not hosts: + raise ImproperlyConfigured("ALLOWED_HOSTS must contain at least one host") + if "*" in hosts: + raise ImproperlyConfigured( + "ALLOWED_HOSTS must not contain '*' in production" + ) + return hosts + + +SECRET_KEY = _require_env("SECRET_KEY") +DEBUG = os.getenv("DEBUG", "false").strip().lower() == "true" +if DEBUG: + raise ImproperlyConfigured("DEBUG must be False in production") +ALLOWED_HOSTS = _parse_allowed_hosts(_require_env("ALLOWED_HOSTS")) # JWT SIMPLE_JWT["SIGNING_KEY"] = SECRET_KEY -# HTTPS settings (раскомментировать если используется HTTPS) -# SECURE_SSL_REDIRECT = True -# SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") -# SECURE_HSTS_SECONDS = 31536000 -# SECURE_HSTS_INCLUDE_SUBDOMAINS = True -# SECURE_HSTS_PRELOAD = True -# SESSION_COOKIE_SECURE = True -# CSRF_COOKIE_SECURE = True +# HTTPS settings +SECURE_SSL_REDIRECT = os.getenv("SECURE_SSL_REDIRECT", "true").strip().lower() == "true" +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +SECURE_HSTS_SECONDS = int(os.getenv("SECURE_HSTS_SECONDS", "31536000")) +SECURE_HSTS_INCLUDE_SUBDOMAINS = ( + os.getenv("SECURE_HSTS_INCLUDE_SUBDOMAINS", "true").strip().lower() == "true" +) +SECURE_HSTS_PRELOAD = os.getenv("SECURE_HSTS_PRELOAD", "true").strip().lower() == "true" +SESSION_COOKIE_SECURE = True +CSRF_COOKIE_SECURE = True # Database DATABASES = { diff --git a/src/settings/test_postgres.py b/src/settings/test_postgres.py new file mode 100644 index 0000000..318579b --- /dev/null +++ b/src/settings/test_postgres.py @@ -0,0 +1,31 @@ +""" +Production-like test settings (PostgreSQL + real migrations). +""" + +import os + +from .test import * + +# Override SQLite with PostgreSQL to match production behavior. +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.getenv("TEST_POSTGRES_DB", os.getenv("POSTGRES_DB", "mostovik_test")), + "USER": os.getenv("TEST_POSTGRES_USER", os.getenv("POSTGRES_USER", "postgres")), + "PASSWORD": os.getenv( + "TEST_POSTGRES_PASSWORD", os.getenv("POSTGRES_PASSWORD", "postgres") + ), + "HOST": os.getenv("TEST_POSTGRES_HOST", os.getenv("POSTGRES_HOST", "127.0.0.1")), + "PORT": os.getenv("TEST_POSTGRES_PORT", os.getenv("POSTGRES_PORT", "5432")), + "CONN_MAX_AGE": 0, + "TEST": { + "NAME": os.getenv( + "TEST_POSTGRES_DB", + os.getenv("POSTGRES_DB", "mostovik_test"), + ), + }, + } +} + +# Enable real migrations for schema parity checks. +globals().pop("MIGRATION_MODULES", None) diff --git a/tests/apps/backups/__init__.py b/tests/apps/backups/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/apps/backups/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/apps/backups/test_views.py b/tests/apps/backups/test_views.py new file mode 100644 index 0000000..41769b0 --- /dev/null +++ b/tests/apps/backups/test_views.py @@ -0,0 +1,168 @@ +"""Tests for async backups export API.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import Mock, patch + +from apps.backups.models import BackupExportJob +from apps.backups.services import BackupExportJobService +from django.db import IntegrityError +from django.urls import reverse +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.user.factories import UserFactory + + +class BackupExportViewTest(APITestCase): + """Tests for async backup export endpoint.""" + + def setUp(self): + self.user = UserFactory.create_user() + self.admin = UserFactory.create_superuser() + self.url = reverse("api_v1:backups:export") + + def test_export_admin_only(self): + response = self.client.post(self.url, {}, format="json") + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + self.client.force_authenticate(self.user) + response = self.client.post(self.url, {}, format="json") + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + @patch("apps.backups.tasks.generate_backup_for_date.delay") + def test_export_starts_job_when_absent(self, delay_mock): + delay_mock.return_value = Mock(id="task-backup-1") + + self.client.force_authenticate(self.admin) + today = timezone.localdate() + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["status"], "started") + self.assertEqual(response.data["task_id"], "task-backup-1") + + job = BackupExportJob.objects.get(actual_date=today) + self.assertEqual(job.status, BackupExportJob.Status.PENDING) + self.assertEqual(job.task_id, "task-backup-1") + delay_mock.assert_called_once_with(job_id=job.id) + + @patch("apps.backups.tasks.generate_backup_for_date.delay") + def test_export_returns_wait_when_job_in_progress(self, delay_mock): + today = timezone.localdate() + BackupExportJob.objects.create( + actual_date=today, + status=BackupExportJob.Status.STARTED, + task_id="task-running-1", + ) + + self.client.force_authenticate(self.admin) + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["status"], "wait") + self.assertEqual(response.data["task_id"], "task-running-1") + delay_mock.assert_not_called() + + @patch("apps.backups.tasks.generate_backup_for_date.delay") + def test_export_returns_file_and_deletes_after_download(self, delay_mock): + with TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + archive_bytes = b"zip-content" + archive_path = tmp_path / "backup.zip" + archive_path.write_bytes(archive_bytes) + + today = timezone.localdate() + job = BackupExportJob.objects.create( + actual_date=today, + status=BackupExportJob.Status.SUCCESS, + task_id="task-success-1", + archive_path=str(archive_path), + archive_filename="backup.zip", + checksum_filename="backup.zip.sha256", + checksum_sha256=hashlib.sha256(archive_bytes).hexdigest(), + organizations_count=7, + ) + + self.client.force_authenticate(self.admin) + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.content, archive_bytes) + self.assertEqual(response["Content-Type"], "application/zip") + self.assertIn('attachment; filename="backup.zip"', response["Content-Disposition"]) + self.assertEqual(response["X-Backup-Organizations"], "7") + self.assertEqual( + response["X-Backup-SHA256"], + hashlib.sha256(archive_bytes).hexdigest(), + ) + + self.assertFalse(archive_path.exists()) + self.assertFalse(BackupExportJob.objects.filter(id=job.id).exists()) + delay_mock.assert_not_called() + + @patch("apps.backups.tasks.generate_backup_for_date.delay") + def test_export_restarts_when_success_job_has_no_file(self, delay_mock): + delay_mock.return_value = Mock(id="task-backup-retry") + today = timezone.localdate() + with TemporaryDirectory() as tmp_dir: + missing_archive_path = Path(tmp_dir) / "non-existent-backup.zip" + BackupExportJob.objects.create( + actual_date=today, + status=BackupExportJob.Status.SUCCESS, + task_id="task-old", + archive_path=str(missing_archive_path), + archive_filename="non-existent-backup.zip", + ) + + self.client.force_authenticate(self.admin) + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["status"], "started") + self.assertEqual(response.data["task_id"], "task-backup-retry") + delay_mock.assert_called_once() + + @patch("apps.backups.services.BackupExportJob.objects.create") + def test_check_or_start_job_handles_integrity_race(self, create_mock): + today = timezone.localdate() + concurrent_job = BackupExportJob( + actual_date=today, + status=BackupExportJob.Status.STARTED, + task_id="task-race-running", + ) + concurrent_job.save() + create_mock.side_effect = IntegrityError("duplicate key value") + + with patch.object( + BackupExportJobService, + "_get_job_for_update", + side_effect=[None, concurrent_job], + ): + result = BackupExportJobService.check_or_start_job( + actual_date=today, + requested_by_id=self.admin.id, + ) + + self.assertEqual(result.action, "wait") + self.assertEqual(result.task_id, "task-race-running") diff --git a/tests/apps/core/test_admin.py b/tests/apps/core/test_admin.py index d09fa1b..8f584dc 100644 --- a/tests/apps/core/test_admin.py +++ b/tests/apps/core/test_admin.py @@ -3,13 +3,13 @@ from datetime import timedelta from unittest.mock import patch +from apps.core.admin import BackgroundJobAdmin +from apps.core.models import BackgroundJob from django.contrib.admin.sites import AdminSite from django.contrib.messages.storage.fallback import FallbackStorage from django.test import RequestFactory, TestCase from django.utils import timezone -from apps.core.admin import BackgroundJobAdmin -from apps.core.models import BackgroundJob from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake diff --git a/tests/apps/core/test_background_jobs.py b/tests/apps/core/test_background_jobs.py index e8b10c0..f93655c 100644 --- a/tests/apps/core/test_background_jobs.py +++ b/tests/apps/core/test_background_jobs.py @@ -253,6 +253,7 @@ class BackgroundJobServiceTest(TestCase): def test_cleanup_old_jobs(self): from datetime import timedelta + from django.utils import timezone old_job = BackgroundJobService.create_job( diff --git a/tests/apps/core/test_cache.py b/tests/apps/core/test_cache.py index c6c87a5..1d93d4d 100644 --- a/tests/apps/core/test_cache.py +++ b/tests/apps/core/test_cache.py @@ -3,12 +3,13 @@ from apps.core.cache import ( CacheManager, _build_cache_key, - invalidate_cache, cache_method, cache_result, + invalidate_cache, ) from django.core.cache import cache from django.test import TestCase + from tests.utils.fixtures import fake @@ -262,6 +263,7 @@ class BuildCacheKeyTest(TestCase): def test_build_cache_key_handles_circular(self): """Test circular references fallback to str.""" + def my_function(): pass diff --git a/tests/apps/core/test_exception_handler.py b/tests/apps/core/test_exception_handler.py index 370e8af..f0174a1 100644 --- a/tests/apps/core/test_exception_handler.py +++ b/tests/apps/core/test_exception_handler.py @@ -1,13 +1,12 @@ """Tests for custom exception handler.""" +from apps.core.exception_handler import custom_exception_handler +from apps.core.exceptions import BaseAPIException from django.core.exceptions import PermissionDenied from django.http import Http404 from django.test import SimpleTestCase from rest_framework.exceptions import APIException, ValidationError -from apps.core.exception_handler import custom_exception_handler -from apps.core.exceptions import BaseAPIException - class CustomExceptionHandlerTest(SimpleTestCase): def _context(self): diff --git a/tests/apps/core/test_logging.py b/tests/apps/core/test_logging.py index 29ceac4..ed75b08 100644 --- a/tests/apps/core/test_logging.py +++ b/tests/apps/core/test_logging.py @@ -12,8 +12,7 @@ from apps.core.logging import ( ) from apps.core.middleware import RequestIDMiddleware, get_request_id from django.http import HttpResponse -from django.test import RequestFactory -from django.test import TestCase +from django.test import RequestFactory, TestCase class JSONFormatterTest(TestCase): diff --git a/tests/apps/core/test_management_commands.py b/tests/apps/core/test_management_commands.py index 395c125..73a0dc0 100644 --- a/tests/apps/core/test_management_commands.py +++ b/tests/apps/core/test_management_commands.py @@ -148,9 +148,7 @@ class BaseAppCommandTest(TestCase): from apps.core.models import BackgroundJob - self.assertTrue( - BackgroundJob.objects.filter(task_id="persist-task").exists() - ) + self.assertTrue(BackgroundJob.objects.filter(task_id="persist-task").exists()) def test_progress_iter_without_total(self): cmd = BaseAppCommand() diff --git a/tests/apps/core/test_middleware.py b/tests/apps/core/test_middleware.py index 6bf1832..942885f 100644 --- a/tests/apps/core/test_middleware.py +++ b/tests/apps/core/test_middleware.py @@ -3,7 +3,11 @@ import logging from io import StringIO -from apps.core.middleware import RequestIDMiddleware, RequestLoggingMiddleware, get_request_id +from apps.core.middleware import ( + RequestIDMiddleware, + RequestLoggingMiddleware, + get_request_id, +) from django.http import HttpResponse from django.test import RequestFactory from django.urls import reverse diff --git a/tests/apps/core/test_mixins.py b/tests/apps/core/test_mixins.py index 6a4e8a4..6afa107 100644 --- a/tests/apps/core/test_mixins.py +++ b/tests/apps/core/test_mixins.py @@ -1,9 +1,5 @@ """Тесты для Model Mixins.""" -from django.db import connection, models -from django.test import TestCase, TransactionTestCase -from django.test.utils import isolate_apps - from apps.core.mixins import ( AuditMixin, OrderableMixin, @@ -12,6 +8,10 @@ from apps.core.mixins import ( StatusMixin, TimestampMixin, ) +from django.db import connection, models +from django.test import TestCase, TransactionTestCase +from django.test.utils import isolate_apps + from tests.apps.user.factories import UserFactory diff --git a/tests/apps/core/test_openapi.py b/tests/apps/core/test_openapi.py index 9ec175f..0fbd66e 100644 --- a/tests/apps/core/test_openapi.py +++ b/tests/apps/core/test_openapi.py @@ -2,12 +2,11 @@ from __future__ import annotations +from apps.core.openapi import _get_status_description, api_docs, swagger_tag from django.test import SimpleTestCase, override_settings from drf_yasg import openapi from rest_framework import serializers -from apps.core.openapi import _get_status_description, api_docs, swagger_tag - class DummySerializer(serializers.Serializer): name = serializers.CharField() diff --git a/tests/apps/core/test_signals.py b/tests/apps/core/test_signals.py index f10bbd0..5514571 100644 --- a/tests/apps/core/test_signals.py +++ b/tests/apps/core/test_signals.py @@ -2,10 +2,6 @@ from __future__ import annotations -from django.contrib.auth import get_user_model -from django.db.models.signals import post_save -from django.test import TestCase - from apps.core.signals import ( SignalDispatcher, on_post_delete, @@ -14,6 +10,10 @@ from apps.core.signals import ( on_pre_save, signal_dispatcher, ) +from django.contrib.auth import get_user_model +from django.db.models.signals import post_save +from django.test import TestCase + from tests.utils.fixtures import fake diff --git a/tests/apps/core/test_tasks.py b/tests/apps/core/test_tasks.py index ba784e6..9212e96 100644 --- a/tests/apps/core/test_tasks.py +++ b/tests/apps/core/test_tasks.py @@ -37,6 +37,7 @@ def idempotent_task(self, marker: str): BackgroundJob.objects.create(task_id=marker, task_name="test.idem") return marker + @celery_app.task(base=TimedTask, bind=True) def timed_task(self, marker: str): return marker diff --git a/tests/apps/core/test_views.py b/tests/apps/core/test_views.py index 10eeb2e..ee94104 100644 --- a/tests/apps/core/test_views.py +++ b/tests/apps/core/test_views.py @@ -1,17 +1,18 @@ """Tests for core views (health checks)""" -from django.urls import reverse -from rest_framework import status -from rest_framework.test import APITestCase, APIRequestFactory +import sys +import types from datetime import timedelta +from apps.core import views as core_views +from apps.core.views import HealthCheckView +from django.urls import reverse +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APIRequestFactory, APITestCase + from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake -from django.utils import timezone -from apps.core.views import HealthCheckView -from apps.core import views as core_views -import sys -import types class HealthCheckViewTest(APITestCase): @@ -51,7 +52,9 @@ class HealthCheckViewTest(APITestCase): url = reverse("core:health") response = self.client.get(url) self.assertIn("redis", response.data["checks"]) - self.assertIn(response.data["checks"]["redis"]["status"], ["up", "down", "skipped"]) + self.assertIn( + response.data["checks"]["redis"]["status"], ["up", "down", "skipped"] + ) class HealthCheckStatusCombinationsTest(APITestCase): @@ -322,7 +325,9 @@ class BackgroundJobsViewTest(APITestCase): ) def test_job_status_for_owner(self): - job = self._create_job(task_id="job-owner", user_id=self.user.id, status="success") + job = self._create_job( + task_id="job-owner", user_id=self.user.id, status="success" + ) self.client.force_authenticate(self.user) url = reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) response = self.client.get(url) @@ -330,14 +335,18 @@ class BackgroundJobsViewTest(APITestCase): self.assertEqual(response.data["task_id"], job.task_id) def test_job_status_forbidden_for_other_user(self): - job = self._create_job(task_id="job-forbidden", user_id=self.user.id, status="success") + job = self._create_job( + task_id="job-forbidden", user_id=self.user.id, status="success" + ) self.client.force_authenticate(self.other) url = reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_job_status_for_admin(self): - job = self._create_job(task_id="job-admin", user_id=self.user.id, status="success") + job = self._create_job( + task_id="job-admin", user_id=self.user.id, status="success" + ) self.client.force_authenticate(self.admin) url = reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) response = self.client.get(url) @@ -364,3 +373,10 @@ class BackgroundJobsViewTest(APITestCase): response = self.client.get(url, {"limit": 2}) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertLessEqual(len(response.data), 2) + + def test_job_list_invalid_limit_returns_400(self): + self._create_job(task_id="job-invalid-limit", user_id=self.user.id, status="success") + self.client.force_authenticate(self.user) + url = reverse("api_v1:jobs:job-list") + response = self.client.get(url, {"limit": "abc"}) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/tests/apps/core/test_viewsets.py b/tests/apps/core/test_viewsets.py index 191e673..4bb4495 100644 --- a/tests/apps/core/test_viewsets.py +++ b/tests/apps/core/test_viewsets.py @@ -189,7 +189,9 @@ router.register( basename="profile-owner-public", ) router.register("bulk-proxies", BulkProxyViewSet, basename="bulk-proxy") -router.register("bulk-proxies-write", BulkWritableProxyViewSet, basename="bulk-proxy-write") +router.register( + "bulk-proxies-write", BulkWritableProxyViewSet, basename="bulk-proxy-write" +) urlpatterns = [path("", include(router.urls))] @@ -283,9 +285,7 @@ class BaseViewSetIntegrationTest(APITestCase): self.assertTrue(response.data["success"]) self.assertEqual(len(response.data["data"]), 2) self.assertIn("pagination", response.data["meta"]) - self.assertSetEqual( - set(response.data["data"][0].keys()), {"id", "address"} - ) + self.assertSetEqual(set(response.data["data"][0].keys()), {"id", "address"}) def test_list_without_pagination(self): ProxyFactory.create_batch(2) @@ -451,7 +451,9 @@ class BulkMixinIntegrationTest(APITestCase): self.assertEqual(response.data["errors"][0]["code"], "missing_ids") def test_bulk_update_empty_items(self): - response = self.client.patch("/bulk-proxies-write/bulk_update/", {}, format="json") + response = self.client.patch( + "/bulk-proxies-write/bulk_update/", {}, format="json" + ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertFalse(response.data["success"]) diff --git a/tests/apps/exchange/__init__.py b/tests/apps/exchange/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/exchange/factories.py b/tests/apps/exchange/factories.py new file mode 100644 index 0000000..0d2d63e --- /dev/null +++ b/tests/apps/exchange/factories.py @@ -0,0 +1,23 @@ +"""Фабрики для тестов exchange.""" + +import factory +from apps.exchange.models import ExchangeConnection +from faker import Faker + +fake = Faker("ru_RU") + + +class ExchangeConnectionFactory(factory.django.DjangoModelFactory): + """Фабрика для модели ExchangeConnection.""" + + class Meta: + model = ExchangeConnection + + server = factory.LazyAttribute(lambda _: fake.ipv4()) + port = 5432 + username = factory.LazyAttribute(lambda _: fake.user_name()) + password = factory.LazyAttribute(lambda _: fake.password()) + database_name = factory.Sequence(lambda n: f"target_db_{n + 1}") + schema_name = "public" + is_active = False + last_error = "" diff --git a/tests/apps/exchange/test_services.py b/tests/apps/exchange/test_services.py new file mode 100644 index 0000000..cb55c5f --- /dev/null +++ b/tests/apps/exchange/test_services.py @@ -0,0 +1,25 @@ +"""Tests for exchange services.""" + +from apps.exchange.services import ExchangeConnectionService +from apps.parsers.models import IndustrialCertificateRecord, ParserLoadLog +from apps.registers.models import Organization +from django.test import TestCase + + +class ExchangeConnectionServiceDependenciesTest(TestCase): + """Tests for dependency expansion in copy operation.""" + + def test_extend_models_without_registry_links(self): + models_to_copy = ExchangeConnectionService._extend_models_with_dependencies( + [ParserLoadLog] + ) + + self.assertEqual(models_to_copy, [ParserLoadLog]) + + def test_extend_models_adds_registers_organization_first(self): + models_to_copy = ExchangeConnectionService._extend_models_with_dependencies( + [IndustrialCertificateRecord] + ) + + self.assertEqual(models_to_copy[0], Organization) + self.assertEqual(models_to_copy[1], IndustrialCertificateRecord) diff --git a/tests/apps/exchange/test_views.py b/tests/apps/exchange/test_views.py new file mode 100644 index 0000000..111c2c3 --- /dev/null +++ b/tests/apps/exchange/test_views.py @@ -0,0 +1,122 @@ +"""Tests for exchange API views.""" + +from types import SimpleNamespace +from unittest.mock import patch + +from apps.exchange.models import ExchangeConnection +from apps.exchange.services import ExchangeServiceError +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.exchange.factories import ExchangeConnectionFactory +from tests.apps.user.factories import UserFactory + + +class ExchangeViewsTest(APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + self.admin = UserFactory.create_superuser() + self.connections_url = reverse("api_v1:exchange:connections") + self.copy_url = reverse("api_v1:exchange:copy") + + def test_connections_endpoint_admin_only(self): + response = self.client.get(self.connections_url) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + self.client.force_authenticate(self.user) + response = self.client.get(self.connections_url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + self.client.force_authenticate(self.admin) + response = self.client.get(self.connections_url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(response.data["success"]) + self.assertIsInstance(response.data["data"], list) + + @patch("apps.exchange.services.ExchangeConnectionService.validate_target_structure") + @patch("apps.exchange.services.ExchangeConnectionService.test_connection") + def test_create_connection_success(self, connection_mock, validate_mock): + old_active = ExchangeConnectionFactory(is_active=True) + + payload = { + "server": "127.0.0.1", + "port": 5432, + "username": "postgres", + "password": "secret", + "database_name": "target_db", + "schema_name": "public", + } + + self.client.force_authenticate(self.admin) + response = self.client.post(self.connections_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ExchangeConnection.objects.filter(is_active=True).count(), 1) + + new_connection = ExchangeConnection.objects.get(id=response.data["data"]["id"]) + self.assertTrue(new_connection.is_active) + + old_active.refresh_from_db() + self.assertFalse(old_active.is_active) + + connection_mock.assert_called_once() + validate_mock.assert_called_once() + + @patch("apps.exchange.services.ExchangeConnectionService.test_connection") + def test_create_connection_fail_rolls_back_active(self, connection_mock): + connection_mock.side_effect = ExchangeServiceError("Connection refused") + + old_active = ExchangeConnectionFactory(is_active=True) + + payload = { + "server": "127.0.0.1", + "port": 5432, + "username": "postgres", + "password": "secret", + "database_name": "target_db", + "schema_name": "public", + } + + self.client.force_authenticate(self.admin) + response = self.client.post(self.connections_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(ExchangeConnection.objects.count(), 1) + + old_active.refresh_from_db() + self.assertTrue(old_active.is_active) + + def test_copy_requires_active_connection(self): + self.client.force_authenticate(self.admin) + response = self.client.post(self.copy_url, {"mode": "all"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + @patch("apps.exchange.views.copy_parsers_data_async.delay") + @patch("apps.exchange.services.ExchangeConnectionService.get_active_connection") + def test_copy_all_success(self, get_active_mock, delay_mock): + active_connection = ExchangeConnectionFactory(is_active=True) + get_active_mock.return_value = active_connection + delay_mock.return_value = SimpleNamespace(id="task-123") + + self.client.force_authenticate(self.admin) + response = self.client.post(self.copy_url, {"mode": "all"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["data"]["status"], "started") + self.assertEqual(response.data["data"]["task_id"], "task-123") + self.assertEqual(response.data["data"]["connection_id"], active_connection.id) + get_active_mock.assert_called_once() + delay_mock.assert_called_once_with( + connection_id=active_connection.id, + payload={"mode": "all", "truncate_before_copy": True}, + requested_by_id=self.admin.id, + ) + + def test_copy_single_requires_table(self): + self.client.force_authenticate(self.admin) + response = self.client.post(self.copy_url, {"mode": "single"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("table", str(response.data)) diff --git a/tests/apps/parsers/test_admin.py b/tests/apps/parsers/test_admin.py index a67fcfa..3e0892d 100644 --- a/tests/apps/parsers/test_admin.py +++ b/tests/apps/parsers/test_admin.py @@ -1,9 +1,5 @@ """Tests for parsers admin configurations.""" -from django.contrib.admin.sites import AdminSite -from django.contrib.messages.storage.fallback import FallbackStorage -from django.test import RequestFactory, TestCase - from apps.parsers.admin import ( FinancialReportAdmin, HasCertificateNumberFilter, @@ -24,6 +20,10 @@ from apps.parsers.models import ( ProcurementRecord, Proxy, ) +from django.contrib.admin.sites import AdminSite +from django.contrib.messages.storage.fallback import FallbackStorage +from django.test import RequestFactory, TestCase + from tests.apps.parsers.factories import ( IndustrialCertificateRecordFactory, InspectionRecordFactory, @@ -34,7 +34,6 @@ from tests.apps.parsers.factories import ( from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake - _CYRILLIC_FINISHED = "\u0437\u0430\u0432\u0435\u0440\u0448\u0435\u043d" _CYRILLIC_PUBLISHED = "\u043e\u043f\u0443\u0431\u043b\u0438\u043a" @@ -121,7 +120,9 @@ class ParsersAdminTest(TestCase): filter_none = HasCertificateNumberFilter( request, {}, IndustrialCertificateRecord, admin ) - qs_none = filter_none.queryset(request, IndustrialCertificateRecord.objects.all()) + qs_none = filter_none.queryset( + request, IndustrialCertificateRecord.objects.all() + ) self.assertIn(record_good, qs_none) def test_manufacturer_admin_helpers(self): @@ -141,13 +142,17 @@ class ParsersAdminTest(TestCase): record = InspectionRecordFactory( organisation_name="Org" * 30, control_authority="Auth" * 20, - status=f"{_CYRILLIC_FINISHED}" + status=f"{_CYRILLIC_FINISHED}", ) self.assertTrue(admin.organisation_name_short(record).endswith("...")) self.assertTrue(admin.control_authority_short(record).endswith("...")) self.assertIn("span", str(admin.status_badge(record))) - record_progress = InspectionRecordFactory(status="\u0432 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0435") - record_cancel = InspectionRecordFactory(status="\u043e\u0442\u043c\u0435\u043d\u0435\u043d\u0430") + record_progress = InspectionRecordFactory( + status="\u0432 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0435" + ) + record_cancel = InspectionRecordFactory( + status="\u043e\u0442\u043c\u0435\u043d\u0435\u043d\u0430" + ) record_other = InspectionRecordFactory(status="unknown") self.assertIn("span", str(admin.status_badge(record_progress))) self.assertIn("span", str(admin.status_badge(record_cancel))) diff --git a/tests/apps/parsers/test_checko.py b/tests/apps/parsers/test_checko.py index 0fd410f..11f7038 100644 --- a/tests/apps/parsers/test_checko.py +++ b/tests/apps/parsers/test_checko.py @@ -3,49 +3,47 @@ from __future__ import annotations import json -import requests from urllib.parse import parse_qs -from django.test import SimpleTestCase - -from requests.adapters import BaseAdapter - +import requests from apps.parsers.clients.checko import ( - CheckoClient, + BankRequest, + CaseRole, CheckoAPIError, + CheckoClient, + CheckoConnectionError, CheckoNotFoundError, CheckoRateLimitError, CheckoValidationError, - CheckoConnectionError, - BankRequest, CompanyRequest, + ContractLaw, + ContractRole, ContractsRequest, - EntrepreneurRequest, EnforcementsRequest, + EntrepreneurRequest, FinancesRequest, InspectionsRequest, LegalCasesRequest, + ObjectType, PersonRequest, SearchRequest, - CaseRole, - ContractRole, SearchType, - ObjectType, - ContractLaw, SortOrder, ) from apps.parsers.clients.checko.datasets import ( - OKVED2, OKFS, OKOPF, OKPD, OKPD2, + OKVED2, AccountCodes, CompanyStatuses, EntrepreneurStatuses, ) +from django.test import SimpleTestCase +from requests.adapters import BaseAdapter -from tests.utils import TestHTTPServer, Response +from tests.utils import Response, TestHTTPServer from tests.utils.fixtures import fake @@ -125,7 +123,9 @@ class CheckoClientValidationTest(SimpleTestCase): def test_search_request_min_query_length(self): with self.assertRaises(CheckoValidationError) as context: self.client.search( - SearchRequest(by=SearchType.NAME, obj=ObjectType.ORGANIZATION, query="abc") + SearchRequest( + by=SearchType.NAME, obj=ObjectType.ORGANIZATION, query="abc" + ) ) self.assertIn("4", str(context.exception)) @@ -241,6 +241,7 @@ class CheckoRequestParamsTest(SimpleTestCase): bank = BankRequest(bic=_digits(9)) self.assertEqual(bank.to_params()["bic"], bank.bic) + class CheckoClientApiTest(SimpleTestCase): def test_get_company_success(self): inn = "".join(str(fake.random_int(0, 9)) for _ in range(10)) @@ -279,7 +280,11 @@ class CheckoClientApiTest(SimpleTestCase): ) client = _client_for(server) with self.assertRaises(CheckoNotFoundError): - client.get_company(CompanyRequest(inn="".join(str(fake.random_int(0, 9)) for _ in range(10)))) + client.get_company( + CompanyRequest( + inn="".join(str(fake.random_int(0, 9)) for _ in range(10)) + ) + ) def test_get_entrepreneur_success(self): ogrnip = "".join(str(fake.random_int(0, 9)) for _ in range(15)) @@ -468,7 +473,11 @@ class CheckoClientApiTest(SimpleTestCase): ) client = _client_for(server) with self.assertRaises(CheckoAPIError): - client.get_company(CompanyRequest(inn="".join(str(fake.random_int(0, 9)) for _ in range(10)))) + client.get_company( + CompanyRequest( + inn="".join(str(fake.random_int(0, 9)) for _ in range(10)) + ) + ) def test_rate_limit_error_handling(self): with TestHTTPServer() as server: @@ -478,7 +487,11 @@ class CheckoClientApiTest(SimpleTestCase): ) client = _client_for(server) with self.assertRaises(CheckoRateLimitError): - client.get_company(CompanyRequest(inn="".join(str(fake.random_int(0, 9)) for _ in range(10)))) + client.get_company( + CompanyRequest( + inn="".join(str(fake.random_int(0, 9)) for _ in range(10)) + ) + ) def test_not_found_error_handling(self): with TestHTTPServer() as server: @@ -488,7 +501,11 @@ class CheckoClientApiTest(SimpleTestCase): ) client = _client_for(server) with self.assertRaises(CheckoNotFoundError): - client.get_company(CompanyRequest(inn="".join(str(fake.random_int(0, 9)) for _ in range(10)))) + client.get_company( + CompanyRequest( + inn="".join(str(fake.random_int(0, 9)) for _ in range(10)) + ) + ) class CheckoClientExtraEndpointsTest(SimpleTestCase): @@ -564,9 +581,7 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): with TestHTTPServer() as server: server.add_route("GET", "/v2/inspections", inspections_handler) client = _client_for(server) - inspections = list( - client.iter_inspections(InspectionsRequest(inn=inn)) - ) + inspections = list(client.iter_inspections(InspectionsRequest(inn=inn))) self.assertTrue(inspections) @@ -599,9 +614,7 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): with TestHTTPServer() as server: server.add_route("GET", "/v2/inspections", inspections_handler) client = _client_for(server) - inspections = list( - client.iter_inspections(InspectionsRequest(inn=inn)) - ) + inspections = list(client.iter_inspections(InspectionsRequest(inn=inn))) self.assertEqual(len(inspections), 1) @@ -642,9 +655,7 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): page = int(params.get("page", ["1"])[0]) payload = { "data": { - "enforcements": [ - {"number": fake.bothify(text="##-####")} - ], + "enforcements": [{"number": fake.bothify(text="##-####")}], "pagination": { "total_records": 2, "total_pages": 2, @@ -664,9 +675,7 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): with TestHTTPServer() as server: server.add_route("GET", "/v2/enforcements", enforcements_handler) client = _client_for(server) - enforcements = list( - client.iter_enforcements(EnforcementsRequest(inn=inn)) - ) + enforcements = list(client.iter_enforcements(EnforcementsRequest(inn=inn))) self.assertTrue(enforcements) @@ -676,9 +685,7 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): def enforcements_handler(_req, _body): payload = { "data": { - "enforcements": [ - {"number": fake.bothify(text="##-####")} - ], + "enforcements": [{"number": fake.bothify(text="##-####")}], "pagination": { "total_records": 1, "total_pages": 1, @@ -696,9 +703,7 @@ class CheckoClientExtraEndpointsTest(SimpleTestCase): with TestHTTPServer() as server: server.add_route("GET", "/v2/enforcements", enforcements_handler) client = _client_for(server) - enforcements = list( - client.iter_enforcements(EnforcementsRequest(inn=inn)) - ) + enforcements = list(client.iter_enforcements(EnforcementsRequest(inn=inn))) self.assertEqual(len(enforcements), 1) diff --git a/tests/apps/parsers/test_checko_parsers.py b/tests/apps/parsers/test_checko_parsers.py index 9b6bcb7..3e215c3 100644 --- a/tests/apps/parsers/test_checko_parsers.py +++ b/tests/apps/parsers/test_checko_parsers.py @@ -2,9 +2,9 @@ from __future__ import annotations +from apps.parsers.clients.checko.client import CheckoClient, _map_ru_keys from django.test import SimpleTestCase -from apps.parsers.clients.checko.client import CheckoClient, _map_ru_keys from tests.utils.fixtures import fake @@ -32,7 +32,10 @@ class CheckoClientParsingTest(SimpleTestCase): self.assertEqual(mapped["inn"], data["\u0418\u041d\u041d"]) self.assertIn("legal_address", mapped) self.assertIn("records", mapped) - self.assertEqual(mapped["records"][0]["ogrn"], data["\u0417\u0430\u043f\u0438\u0441\u0438"][0]["\u041e\u0413\u0420\u041d"]) + self.assertEqual( + mapped["records"][0]["ogrn"], + data["\u0417\u0430\u043f\u0438\u0441\u0438"][0]["\u041e\u0413\u0420\u041d"], + ) def test_parse_okved_info_with_additional(self): info = self.client._parse_okved_info( @@ -96,7 +99,11 @@ class CheckoClientParsingTest(SimpleTestCase): "reg_date": str(fake.date()), "short_name": fake.company(), "full_name": fake.company(), - "status": {"code": "100", "name": "Active", "record_date": str(fake.date())}, + "status": { + "code": "100", + "name": "Active", + "record_date": str(fake.date()), + }, "legal_address": { "full_address": fake.address(), "region": {"code": "77", "name": "Moscow"}, @@ -168,7 +175,9 @@ class CheckoClientParsingTest(SimpleTestCase): "full_name": fake.company(), } ], - "branches": {"branch": [{"full_name": fake.company(), "address": fake.address()}]}, + "branches": { + "branch": [{"full_name": fake.company(), "address": fake.address()}] + }, "licenses": [{"number": "L-1", "activities": [fake.word()]}], "trademarks": [{"id": 1, "url": fake.url()}], "tax_debt": {"total": 123, "date": str(fake.date())}, @@ -239,9 +248,7 @@ class CheckoClientParsingTest(SimpleTestCase): "companies_as_founder": [ {"ogrn": _digits(13), "short_name": fake.company()} ], - "entrepreneurs": [ - {"ogrnip": _digits(15), "full_name": fake.name()} - ], + "entrepreneurs": [{"ogrnip": _digits(15), "full_name": fake.name()}], } ) self.assertEqual(len(person.disqualifications), 1) diff --git a/tests/apps/parsers/test_clients.py b/tests/apps/parsers/test_clients.py index 42792d5..9faaab5 100644 --- a/tests/apps/parsers/test_clients.py +++ b/tests/apps/parsers/test_clients.py @@ -5,8 +5,6 @@ from __future__ import annotations from urllib.parse import urlparse import requests -from requests.adapters import BaseAdapter - from apps.parsers.clients.base import ( BaseHTTPClient, ConnectionError, @@ -19,6 +17,7 @@ from apps.parsers.clients.minpromtorg.schemas import IndustrialCertificate, Manu from apps.parsers.clients.proverki import ProverkiClient from apps.parsers.clients.proverki.schemas import Inspection from django.test import TestCase, tag +from requests.adapters import BaseAdapter from tests.utils import Response, TestHTTPServer from tests.utils.fixtures import ( @@ -110,7 +109,9 @@ class BaseHTTPClientTest(TestCase): with TestHTTPServer() as server: server.add_route("POST", "/echo", echo_handler) - server.add_route("GET", "/missing", lambda _req, _body: Response(status=404)) + server.add_route( + "GET", "/missing", lambda _req, _body: Response(status=404) + ) client = BaseHTTPClient(base_url=server.base_url, adapter=server.adapter) result = client.post("/echo", data=b"ping") self.assertEqual(result, b"ping") @@ -123,7 +124,9 @@ class BaseHTTPClientTest(TestCase): def test_download_file_error(self): with TestHTTPServer() as server: - server.add_route("GET", "/missing.bin", lambda _req, _body: Response(status=404)) + server.add_route( + "GET", "/missing.bin", lambda _req, _body: Response(status=404) + ) client = BaseHTTPClient(base_url=server.base_url, adapter=server.adapter) with self.assertRaises(HTTPError): client.download_file("/missing.bin") @@ -136,7 +139,9 @@ class BaseHTTPClientTest(TestCase): def test_context_manager_closes_session(self): with TestHTTPServer() as server: server.add_json("/ping", {"ok": True}) - with BaseHTTPClient(base_url=server.base_url, adapter=server.adapter) as client: + with BaseHTTPClient( + base_url=server.base_url, adapter=server.adapter + ) as client: client.get_json("/ping") self.assertIsNotNone(client._session) self.assertIsNone(client._session) @@ -280,10 +285,7 @@ class IndustrialProductionClientTest(TestCase): def test_get_latest_file_url_selects_newest(self): client = IndustrialProductionClient() dates = sorted( - { - fake.date_between(start_date="-90d", end_date="today") - for _ in range(3) - } + {fake.date_between(start_date="-90d", end_date="today") for _ in range(3)} ) files = [] for date in dates: @@ -397,10 +399,7 @@ class ManufacturesClientTest(TestCase): def test_get_latest_file_url_selects_newest(self): client = ManufacturesClient() dates = sorted( - { - fake.date_between(start_date="-90d", end_date="today") - for _ in range(3) - } + {fake.date_between(start_date="-90d", end_date="today") for _ in range(3)} ) files = [] for date in dates: @@ -565,7 +564,7 @@ class ProverkiClientTest(TestCase): f"<КонтрольныйОрган>{authority}" "" "" - ).encode("utf-8") + ).encode() inspections = client._parse_xml_content(xml_content, None) @@ -580,7 +579,7 @@ class ProverkiClientTest(TestCase): row_inn = "".join(str(fake.random_int(0, 9)) for _ in range(10)) reg_num = "".join(str(fake.random_int(0, 9)) for _ in range(12)) element = ET.fromstring( - f"" + f'' ) # noqa: S314 client = ProverkiClient() @@ -602,7 +601,7 @@ class ProverkiClientTest(TestCase): inn = "".join(str(fake.random_int(0, 9)) for _ in range(10)) reg_num = "".join(str(fake.random_int(0, 9)) for _ in range(12)) xml_content = ( - "" + '' "" "" f"{inn}" diff --git a/tests/apps/parsers/test_fns_upload.py b/tests/apps/parsers/test_fns_upload.py index 7b7f172..ff468d2 100644 --- a/tests/apps/parsers/test_fns_upload.py +++ b/tests/apps/parsers/test_fns_upload.py @@ -27,7 +27,9 @@ def _build_fns_excel_bytes() -> bytes: year = fake.random_int(min=2020, max=2025) ws.append(["Форма №1", None, year, None]) ws.append([None, "Код", "Начало", "Конец"]) - ws.append([fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)]) + ws.append( + [fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)] + ) buf = io.BytesIO() wb.save(buf) wb.close() @@ -80,9 +82,7 @@ class FNSUploadIntegrationTest(APITestCase): report = FinancialReport.objects.first() self.assertEqual(report.external_id, external_id) self.assertEqual(report.ogrn, ogrn) - self.assertTrue( - FinancialReportLine.objects.filter(report=report).exists() - ) + self.assertTrue(FinancialReportLine.objects.filter(report=report).exists()) processed_path = os.path.join(processed_dir, filename) self.assertTrue(os.path.exists(processed_path)) diff --git a/tests/apps/parsers/test_services.py b/tests/apps/parsers/test_services.py index 91982f2..a9d9ef7 100644 --- a/tests/apps/parsers/test_services.py +++ b/tests/apps/parsers/test_services.py @@ -1,13 +1,17 @@ """Tests for parsers services.""" +from urllib.parse import urlparse + from apps.parsers.clients.minpromtorg.industrial import IndustrialProductionClient from apps.parsers.clients.minpromtorg.schemas import IndustrialCertificate, Manufacturer from apps.parsers.clients.proverki.schemas import Inspection +from apps.parsers.clients.zakupki.schemas import Procurement from apps.parsers.models import ( IndustrialCertificateRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, + ProcurementRecord, Proxy, ) from apps.parsers.services import ( @@ -15,12 +19,14 @@ from apps.parsers.services import ( InspectionService, ManufacturerService, ParserLoadLogService, + ProcurementService, ProxyService, ) +from apps.registers.models import Organization from django.test import TestCase, tag + from tests.utils import TestHTTPServer from tests.utils.fixtures import build_minpromtorg_certificates_excel, fake -from urllib.parse import urlparse from .factories import ( IndustrialCertificateRecordFactory, @@ -38,6 +44,17 @@ def _digits(length: int) -> str: def _proxy_address() -> str: return f"http://{fake.ipv4()}:{fake.port_number()}" + +def _create_registry_organization(*, inn: str, ogrn: str) -> Organization: + return Organization.objects.create( + pn_name=fake.company(), + mn_ogrn=int(ogrn), + mn_inn=int(inn), + in_kpp=int(_digits(9)), + mn_okpo=_digits(8), + ) + + class ProxyServiceTest(TestCase): """Tests for ProxyService.""" @@ -248,6 +265,59 @@ class IndustrialCertificateServiceTest(TestCase): self.assertEqual(count, 5) self.assertEqual(IndustrialCertificateRecord.objects.count(), 5) + record = IndustrialCertificateRecord.objects.first() + self.assertIsNotNone(record.issue_date_normalized) + self.assertIsNotNone(record.expiry_date_normalized) + + def test_save_certificates_links_registry_organization_when_exists(self): + """Test linking to registers organization is created when identifiers match.""" + inn = _digits(10) + ogrn = _digits(13) + organization = _create_registry_organization(inn=inn, ogrn=ogrn) + certificate_number = fake.bothify(text="??-####-#####") + + certificates = [ + IndustrialCertificate( + issue_date=str(fake.date()), + certificate_number=certificate_number, + expiry_date=str(fake.date()), + certificate_file_url=fake.url(), + organisation_name=fake.company(), + inn=inn, + ogrn=ogrn, + ) + ] + + saved = IndustrialCertificateService.save_certificates(certificates, batch_id=1) + + self.assertEqual(saved, 1) + record = IndustrialCertificateRecord.objects.get( + certificate_number=certificate_number + ) + self.assertEqual(record.registry_organization_id, organization.id) + + def test_save_certificates_keeps_null_registry_organization_when_not_found(self): + """Test parser save does not fail and keeps null when organization is absent.""" + certificate_number = fake.bothify(text="??-####-#####") + certificates = [ + IndustrialCertificate( + issue_date=str(fake.date()), + certificate_number=certificate_number, + expiry_date=str(fake.date()), + certificate_file_url=fake.url(), + organisation_name=fake.company(), + inn=_digits(10), + ogrn=_digits(13), + ) + ] + + saved = IndustrialCertificateService.save_certificates(certificates, batch_id=1) + + self.assertEqual(saved, 1) + record = IndustrialCertificateRecord.objects.get( + certificate_number=certificate_number + ) + self.assertIsNone(record.registry_organization_id) def test_save_certificates_with_chunk_size(self): """Test saving certificates in chunks.""" @@ -287,16 +357,16 @@ class IndustrialCertificateServiceTest(TestCase): results = IndustrialCertificateService.find_by_inn(inn_a) self.assertEqual(results.count(), 2) - results_batch1 = IndustrialCertificateService.find_by_inn( - inn_a, batch_id=1 - ) + results_batch1 = IndustrialCertificateService.find_by_inn(inn_a, batch_id=1) self.assertEqual(results_batch1.count(), 1) def test_find_by_certificate_number(self): """Test finding certificate by number.""" unique_number = fake.bothify(text="CERT-#####") IndustrialCertificateRecordFactory(certificate_number=unique_number) - IndustrialCertificateRecordFactory(certificate_number=fake.bothify(text="CERT-#####")) + IndustrialCertificateRecordFactory( + certificate_number=fake.bothify(text="CERT-#####") + ) results = IndustrialCertificateService.find_by_certificate_number(unique_number) self.assertEqual(results.count(), 1) @@ -332,9 +402,10 @@ class IndustrialCertificateServiceTest(TestCase): ogrn=_digits(13), ) ] - IndustrialCertificateService.save_certificates(duplicate, batch_id=2) + count2 = IndustrialCertificateService.save_certificates(duplicate, batch_id=2) # Should still be 1 record (duplicate skipped) + self.assertEqual(count2, 0) self.assertEqual(IndustrialCertificateRecord.objects.count(), 1) # Verify original data preserved @@ -369,6 +440,27 @@ class ManufacturerServiceTest(TestCase): self.assertEqual(count, 5) self.assertEqual(ManufacturerRecord.objects.count(), 5) + def test_save_manufacturers_links_registry_organization_when_exists(self): + """Test linking manufacturer to registers organization by INN/ОГРН.""" + inn = _digits(10) + ogrn = _digits(13) + organization = _create_registry_organization(inn=inn, ogrn=ogrn) + + manufacturers = [ + Manufacturer( + full_legal_name=fake.company(), + inn=inn, + ogrn=ogrn, + address=fake.address().replace("\n", ", "), + ) + ] + + saved = ManufacturerService.save_manufacturers(manufacturers, batch_id=1) + + self.assertEqual(saved, 1) + record = ManufacturerRecord.objects.get(inn=inn) + self.assertEqual(record.registry_organization_id, organization.id) + def test_save_manufacturers_with_chunk_size(self): """Test saving manufacturers in chunks.""" manufacturers = [ @@ -448,9 +540,10 @@ class ManufacturerServiceTest(TestCase): address=fake.address().replace("\n", ", "), ) ] - ManufacturerService.save_manufacturers(duplicate, batch_id=2) + count2 = ManufacturerService.save_manufacturers(duplicate, batch_id=2) # Should still be 1 record (duplicate skipped) + self.assertEqual(count2, 0) self.assertEqual(ManufacturerRecord.objects.count(), 1) # Verify original data preserved @@ -493,6 +586,39 @@ class InspectionServiceTest(TestCase): self.assertEqual(count, 5) self.assertEqual(InspectionRecord.objects.count(), 5) + record = InspectionRecord.objects.first() + self.assertIsNotNone(record.start_date_normalized) + self.assertIsNotNone(record.end_date_normalized) + + def test_save_inspections_links_registry_organization_when_exists(self): + """Test linking inspection to registers organization by INN/ОГРН.""" + inn = _digits(10) + ogrn = _digits(13) + organization = _create_registry_organization(inn=inn, ogrn=ogrn) + registration_number = _digits(12) + + inspections = [ + Inspection( + registration_number=registration_number, + inn=inn, + ogrn=ogrn, + organisation_name=fake.company(), + control_authority=fake.company(), + inspection_type=fake.word(), + inspection_form=fake.word(), + start_date=str(fake.date()), + end_date=str(fake.date()), + status=fake.word(), + legal_basis=fake.sentence(nb_words=3), + result=fake.sentence(nb_words=3), + ) + ] + + saved = InspectionService.save_inspections(inspections, batch_id=1) + + self.assertEqual(saved, 1) + record = InspectionRecord.objects.get(registration_number=registration_number) + self.assertEqual(record.registry_organization_id, organization.id) def test_save_inspections_with_chunk_size(self): """Test saving inspections in chunks.""" @@ -612,9 +738,10 @@ class InspectionServiceTest(TestCase): result=fake.sentence(nb_words=3), ) ] - InspectionService.save_inspections(duplicate, batch_id=2) + count2 = InspectionService.save_inspections(duplicate, batch_id=2) # Should still be 1 record (duplicate skipped) + self.assertEqual(count2, 0) self.assertEqual(InspectionRecord.objects.count(), 1) # Verify original data preserved @@ -626,6 +753,57 @@ class InspectionServiceTest(TestCase): self.assertEqual(record.load_batch, 1) # Original batch +class ProcurementServiceTest(TestCase): + """Tests for ProcurementService.""" + + def _build_procurement(self, **overrides) -> Procurement: + data = { + "purchase_number": _digits(19), + "purchase_name": fake.sentence(nb_words=4), + "customer_inn": _digits(10), + "customer_kpp": _digits(9), + "customer_ogrn": _digits(13), + "customer_name": fake.company(), + "max_price": "1 234 567,89", + "currency_code": "RUB", + "placement_method": fake.word(), + "publish_date": "01.03.2026", + "end_date": "2026-03-15", + "status": fake.word(), + "law_type": "44-FZ", + "purchase_object_info": fake.sentence(nb_words=4), + "href": fake.url(), + } + data.update(overrides) + return Procurement(**data) + + def test_save_procurements_sets_normalized_fields(self): + procurement = self._build_procurement() + + saved = ProcurementService.save_procurements([procurement], batch_id=1) + + self.assertEqual(saved, 1) + record = ProcurementRecord.objects.get(purchase_number=procurement.purchase_number) + self.assertEqual(str(record.max_price_amount), "1234567.89") + self.assertEqual(str(record.publish_date_normalized), "2026-03-01") + self.assertEqual(str(record.end_date_normalized), "2026-03-15") + + def test_save_procurements_duplicate_returns_zero(self): + purchase_number = _digits(19) + first = self._build_procurement(purchase_number=purchase_number) + duplicate = self._build_procurement( + purchase_number=purchase_number, + customer_name=fake.company(), + ) + + saved_first = ProcurementService.save_procurements([first], batch_id=1) + saved_second = ProcurementService.save_procurements([duplicate], batch_id=2) + + self.assertEqual(saved_first, 1) + self.assertEqual(saved_second, 0) + self.assertEqual(ProcurementRecord.objects.count(), 1) + + @tag("integration", "slow", "e2e") class EndToEndIntegrationTest(TestCase): """ @@ -668,9 +846,7 @@ class EndToEndIntegrationTest(TestCase): ) server.add_bytes(f"/files/{file_name}", excel_bytes) host = urlparse(server.base_url) - client_host = ( - f"{host.hostname}:{host.port}" if host.port else host.hostname - ) + client_host = f"{host.hostname}:{host.port}" if host.port else host.hostname with IndustrialProductionClient( host=client_host, scheme="http", diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index c6bc986..b3d69cd 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -6,13 +6,13 @@ import io import os import tempfile +from apps.parsers.models import FinancialReport, FinancialReportLine, ProcurementRecord from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse from openpyxl import Workbook from rest_framework import status from rest_framework.test import APITestCase -from apps.parsers.models import FinancialReport, FinancialReportLine, ProcurementRecord from tests.apps.parsers.factories import ( IndustrialCertificateRecordFactory, InspectionRecordFactory, @@ -34,7 +34,9 @@ def _build_fns_excel_bytes() -> bytes: year = fake.random_int(min=2020, max=2025) ws.append(["Form", None, year, None]) ws.append([None, "Code", "Start", "End"]) - ws.append([fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)]) + ws.append( + [fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)] + ) buf = io.BytesIO() wb.save(buf) wb.close() @@ -130,6 +132,7 @@ class ParsersViewSetTest(APITestCase): url = reverse("api_v1:fns:fns-reports-list") response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["data"][0]["lines_count"], 1) detail = self.client.get( reverse("api_v1:fns:fns-reports-detail", args=[report.id]) ) @@ -184,5 +187,7 @@ class ParsersViewSetTest(APITestCase): FNS_FAILED_DIRECTORY=failed_dir, ): url = reverse("api_v1:fns:fns-upload") - response = self.client.post(url, {"files": [upload]}, format="multipart") + response = self.client.post( + url, {"files": [upload]}, format="multipart" + ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/tests/apps/registers/__init__.py b/tests/apps/registers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/apps/registers/factories.py b/tests/apps/registers/factories.py new file mode 100644 index 0000000..676b55b --- /dev/null +++ b/tests/apps/registers/factories.py @@ -0,0 +1,66 @@ +"""Фабрики для тестов приложения registers.""" + +import factory +from apps.registers.models import ( + Organization, + Register, + RegisterUpload, + RegistryMembershipPeriod, +) +from django.utils import timezone +from faker import Faker + +fake = Faker("ru_RU") + + +class RegisterFactory(factory.django.DjangoModelFactory): + """Фабрика для модели Register.""" + + class Meta: + model = Register + + name = factory.Sequence(lambda n: f"Реестр {n + 1}") + + +class OrganizationFactory(factory.django.DjangoModelFactory): + """Фабрика для модели Organization.""" + + class Meta: + model = Organization + + pn_name = factory.LazyAttribute(lambda _: fake.company()) + mn_ogrn = factory.Sequence(lambda n: 10_000_000_000_000 + n) + mn_inn = factory.Sequence(lambda n: 1_000_000_000 + n) + in_kpp = factory.LazyAttribute( + lambda _: fake.random_number(digits=9, fix_len=True) + ) + mn_okpo = factory.LazyAttribute( + lambda _: str(fake.random_number(digits=8, fix_len=True)) + ) + + +class RegisterUploadFactory(factory.django.DjangoModelFactory): + """Фабрика для модели RegisterUpload.""" + + class Meta: + model = RegisterUpload + + registry = factory.SubFactory(RegisterFactory) + actual_date = factory.LazyFunction(timezone.localdate) + file_name = factory.Sequence(lambda n: f"register_{n + 1}.xlsx") + file_hash = factory.LazyAttribute(lambda _: fake.sha256(raw_output=False)) + rows_count = factory.LazyAttribute(lambda _: fake.random_int(min=1, max=1000)) + + +class RegistryMembershipPeriodFactory(factory.django.DjangoModelFactory): + """Фабрика для модели RegistryMembershipPeriod.""" + + class Meta: + model = RegistryMembershipPeriod + + registry = factory.SubFactory(RegisterFactory) + organization = factory.SubFactory(OrganizationFactory) + started_at = factory.LazyFunction(timezone.localdate) + ended_at = None + started_by_upload = factory.SubFactory(RegisterUploadFactory, registry=factory.SelfAttribute("..registry")) + ended_by_upload = None diff --git a/tests/apps/registers/test_views.py b/tests/apps/registers/test_views.py new file mode 100644 index 0000000..4330cb8 --- /dev/null +++ b/tests/apps/registers/test_views.py @@ -0,0 +1,416 @@ +"""Integration tests for registers API views.""" + +from __future__ import annotations + +import io +from datetime import date + +from apps.registers.models import Organization, RegistryMembershipPeriod +from django.core.files.uploadedfile import SimpleUploadedFile +from django.db import IntegrityError +from django.urls import reverse +from openpyxl import Workbook +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.registers.factories import ( + OrganizationFactory, + RegisterFactory, + RegisterUploadFactory, + RegistryMembershipPeriodFactory, +) +from tests.apps.user.factories import UserFactory + + +def _build_register_excel_bytes(rows: list[dict], *, with_kpp: bool = True) -> bytes: + workbook = Workbook() + worksheet = workbook.active + + headers = ["pn_name", "mn_ogrn", "mn_inn"] + if with_kpp: + headers.append("in_kpp") + headers.append("mn_okpo") + + worksheet.append(headers) + + for row in rows: + values = [row["pn_name"], row["mn_ogrn"], row["mn_inn"]] + if with_kpp: + values.append(row.get("in_kpp")) + values.append(row["mn_okpo"]) + worksheet.append(values) + + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + return buffer.getvalue() + + +def _extract_results(response_data): + if hasattr(response_data, "get"): + data = response_data.get("data") + if isinstance(data, list): + return data + + results = response_data.get("results") + if results is not None: + return results + return response_data + + +class RegistersViewsTest(APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + self.client.force_authenticate(self.user) + + def _post_upload( + self, + *, + registry, + rows: list[dict], + actual_date_value: date, + with_kpp: bool = True, + file_name: str = "registry.xlsx", + ): + content = _build_register_excel_bytes(rows, with_kpp=with_kpp) + upload = SimpleUploadedFile( + file_name, + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + return self.client.post( + reverse("api_v1:registers:register-upload"), + { + "registry": str(registry.id), + "actual_date": actual_date_value.isoformat(), + "file": upload, + }, + format="multipart", + ) + + def test_registries_list_and_retrieve(self): + registry = RegisterFactory(name="Росатом") + + list_response = self.client.get(reverse("api_v1:registers:registries-list")) + self.assertEqual(list_response.status_code, status.HTTP_200_OK) + + detail_response = self.client.get( + reverse("api_v1:registers:registries-detail", args=[registry.id]) + ) + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + self.assertEqual(detail_response.data["name"], "Росатом") + + def test_organizations_list_and_retrieve(self): + organization = OrganizationFactory() + + list_response = self.client.get(reverse("api_v1:registers:organizations-list")) + self.assertEqual(list_response.status_code, status.HTTP_200_OK) + + detail_response = self.client.get( + reverse("api_v1:registers:organizations-detail", args=[organization.id]) + ) + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + self.assertEqual(detail_response.data["id"], organization.id) + self.assertIn("periods", detail_response.data) + + def test_organizations_search_by_all_fields(self): + organization = OrganizationFactory( + pn_name='АО "Тестовая организация"', + mn_ogrn=1027600980990, + mn_inn=7601000086, + in_kpp=760401001, + mn_okpo="07506197", + ) + OrganizationFactory() + + search_values = [ + "Тестовая организация", + str(organization.mn_ogrn), + str(organization.mn_inn), + str(organization.in_kpp), + organization.mn_okpo, + ] + + for search_value in search_values: + response = self.client.get( + reverse("api_v1:registers:organizations-list"), + {"search": search_value}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + result_ids = [item["id"] for item in _extract_results(response.data)] + self.assertIn(organization.id, result_ids) + + def test_organizations_filter_by_registry_and_actual_date(self): + registry = RegisterFactory(name="Роскосмос") + + organization_old = OrganizationFactory() + organization_current = OrganizationFactory() + + upload_start = RegisterUploadFactory(registry=registry, actual_date=date(2026, 1, 1)) + upload_end = RegisterUploadFactory(registry=registry, actual_date=date(2026, 2, 1)) + + RegistryMembershipPeriodFactory( + registry=registry, + organization=organization_old, + started_at=date(2026, 1, 1), + ended_at=date(2026, 2, 1), + started_by_upload=upload_start, + ended_by_upload=upload_end, + ) + RegistryMembershipPeriodFactory( + registry=registry, + organization=organization_current, + started_at=date(2026, 2, 1), + started_by_upload=upload_end, + ) + + response_past = self.client.get( + reverse("api_v1:registers:organizations-list"), + {"registry": str(registry.id), "actual_date": "2026-01-15"}, + ) + self.assertEqual(response_past.status_code, status.HTTP_200_OK) + past_ids = {item["id"] for item in _extract_results(response_past.data)} + self.assertIn(organization_old.id, past_ids) + self.assertNotIn(organization_current.id, past_ids) + + response_latest = self.client.get( + reverse("api_v1:registers:organizations-list"), + {"registry": str(registry.id)}, + ) + self.assertEqual(response_latest.status_code, status.HTTP_200_OK) + latest_ids = {item["id"] for item in _extract_results(response_latest.data)} + self.assertNotIn(organization_old.id, latest_ids) + self.assertIn(organization_current.id, latest_ids) + + def test_registry_specific_organizations_list_endpoint(self): + registry = RegisterFactory(name="Росатом ОПК") + organization = OrganizationFactory( + mn_ogrn=1027600980990, + mn_inn=7601000086, + mn_okpo="07506197", + ) + upload = RegisterUploadFactory(registry=registry, actual_date=date(2026, 3, 1)) + RegistryMembershipPeriodFactory( + registry=registry, + organization=organization, + started_at=date(2026, 3, 1), + started_by_upload=upload, + ) + + response = self.client.get( + reverse( + "api_v1:registers:registry-organizations-list", + args=[registry.id], + ), + {"actual_date": "2026-03-15"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + results = _extract_results(response.data) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], organization.id) + + def test_upload_closes_and_reopens_period(self): + registry = RegisterFactory(name="Реестр периодов") + + org_a = { + "pn_name": 'АО "Орг А"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07506197", + } + org_b = { + "pn_name": 'АО "Орг Б"', + "mn_ogrn": "1083340004527", + "mn_inn": "3329051460", + "in_kpp": "332901001", + "mn_okpo": "07518609", + } + + first = self._post_upload( + registry=registry, + rows=[org_a], + actual_date_value=date(2026, 1, 1), + file_name="first.xlsx", + ) + self.assertEqual(first.status_code, status.HTTP_201_CREATED) + self.assertEqual(first.data["opened_periods"], 1) + self.assertEqual(first.data["closed_periods"], 0) + + second = self._post_upload( + registry=registry, + rows=[org_b], + actual_date_value=date(2026, 2, 1), + file_name="second.xlsx", + ) + self.assertEqual(second.status_code, status.HTTP_201_CREATED) + self.assertEqual(second.data["opened_periods"], 1) + self.assertEqual(second.data["closed_periods"], 1) + + third = self._post_upload( + registry=registry, + rows=[org_a], + actual_date_value=date(2026, 3, 1), + file_name="third.xlsx", + ) + self.assertEqual(third.status_code, status.HTTP_201_CREATED) + self.assertEqual(third.data["opened_periods"], 1) + self.assertEqual(third.data["closed_periods"], 1) + + organization_a = Organization.objects.get(mn_ogrn=1027600980990, mn_inn=7601000086) + periods = list( + RegistryMembershipPeriod.objects.filter( + registry=registry, + organization=organization_a, + ).order_by("started_at") + ) + + self.assertEqual(len(periods), 2) + self.assertEqual(periods[0].started_at, date(2026, 1, 1)) + self.assertEqual(periods[0].ended_at, date(2026, 2, 1)) + self.assertEqual(periods[1].started_at, date(2026, 3, 1)) + self.assertIsNone(periods[1].ended_at) + + def test_same_organization_can_be_in_multiple_registries(self): + registry_a = RegisterFactory(name="Росатом") + registry_b = RegisterFactory(name="Роскосмос") + + org_row = { + "pn_name": 'АО "Общая организация"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07506197", + } + + upload_a = self._post_upload( + registry=registry_a, + rows=[org_row], + actual_date_value=date(2026, 1, 1), + file_name="reg_a.xlsx", + ) + upload_b = self._post_upload( + registry=registry_b, + rows=[org_row], + actual_date_value=date(2026, 1, 1), + file_name="reg_b.xlsx", + ) + + self.assertEqual(upload_a.status_code, status.HTTP_201_CREATED) + self.assertEqual(upload_b.status_code, status.HTTP_201_CREATED) + + response_a = self.client.get( + reverse("api_v1:registers:registry-organizations-list", args=[registry_a.id]) + ) + response_b = self.client.get( + reverse("api_v1:registers:registry-organizations-list", args=[registry_b.id]) + ) + + self.assertEqual(response_a.status_code, status.HTTP_200_OK) + self.assertEqual(response_b.status_code, status.HTTP_200_OK) + + ids_a = {item["id"] for item in _extract_results(response_a.data)} + ids_b = {item["id"] for item in _extract_results(response_b.data)} + self.assertEqual(ids_a, ids_b) + self.assertEqual(len(ids_a), 1) + + def test_active_membership_period_is_unique_per_registry_and_organization(self): + registry = RegisterFactory(name="Уникальный период") + organization = OrganizationFactory() + upload = RegisterUploadFactory(registry=registry, actual_date=date(2026, 6, 1)) + + RegistryMembershipPeriodFactory( + registry=registry, + organization=organization, + started_at=date(2026, 6, 1), + started_by_upload=upload, + ended_at=None, + ) + + with self.assertRaises(IntegrityError): + RegistryMembershipPeriod.objects.create( + registry=registry, + organization=organization, + started_at=date(2026, 7, 1), + started_by_upload=upload, + ended_at=None, + ) + + def test_upload_without_kpp_column(self): + registry = RegisterFactory(name="Роскосмос") + response = self._post_upload( + registry=registry, + rows=[ + { + "pn_name": 'АО "Ярославский радиозавод"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "mn_okpo": "07506197", + } + ], + actual_date_value=date(2026, 4, 1), + with_kpp=False, + file_name="without_kpp.xlsx", + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + organization = Organization.objects.get(mn_ogrn=1027600980990, mn_inn=7601000086) + self.assertIsNone(organization.in_kpp) + + def test_upload_rejects_invalid_okpo(self): + registry = RegisterFactory(name="Реестр ошибки") + response = self._post_upload( + registry=registry, + rows=[ + { + "pn_name": 'АО "Невалидный ОКПО"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07A06197", + } + ], + actual_date_value=date(2026, 5, 1), + with_kpp=True, + file_name="invalid.xlsx", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(Organization.objects.count(), 0) + + def test_upload_requires_authentication(self): + registry = RegisterFactory(name="Закрытый реестр") + content = _build_register_excel_bytes( + [ + { + "pn_name": 'АО "Закрытый"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07506197", + } + ] + ) + upload = SimpleUploadedFile( + "auth.xlsx", + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + self.client.force_authenticate(user=None) + response = self.client.post( + reverse("api_v1:registers:register-upload"), + { + "registry": str(registry.id), + "actual_date": "2026-01-01", + "file": upload, + }, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) diff --git a/tests/apps/user/test_admin.py b/tests/apps/user/test_admin.py index 76d8414..0aed11d 100644 --- a/tests/apps/user/test_admin.py +++ b/tests/apps/user/test_admin.py @@ -2,13 +2,13 @@ from __future__ import annotations +from apps.user.admin import ProfileAdmin, UserAdmin +from apps.user.models import Profile, User from django.contrib.admin.sites import AdminSite from django.contrib.messages.storage.fallback import FallbackStorage from django.core.files.uploadedfile import SimpleUploadedFile from django.test import RequestFactory, TestCase -from apps.user.admin import ProfileAdmin, UserAdmin -from apps.user.models import Profile, User from tests.utils.fixtures import fake @@ -31,10 +31,16 @@ class UserAdminTest(TestCase): def test_badges(self): verified = User.objects.create_user( - email=fake.email(), username=fake.user_name(), password="pass", is_verified=True + email=fake.email(), + username=fake.user_name(), + password="pass", + is_verified=True, ) unverified = User.objects.create_user( - email=fake.email(), username=fake.user_name(), password="pass", is_verified=False + email=fake.email(), + username=fake.user_name(), + password="pass", + is_verified=False, ) self.assertIn("span", str(self.admin.is_verified_badge(verified))) self.assertIn("span", str(self.admin.is_verified_badge(unverified))) diff --git a/tests/apps/user/test_serializers.py b/tests/apps/user/test_serializers.py index e3ada24..86305e1 100644 --- a/tests/apps/user/test_serializers.py +++ b/tests/apps/user/test_serializers.py @@ -191,7 +191,7 @@ class LoginSerializerTest(TestCase): def setUp(self): self.login_data = { - "email": fake.email(), + "username": fake.user_name(), "password": fake.password(length=12, special_chars=False), } @@ -200,16 +200,16 @@ class LoginSerializerTest(TestCase): serializer = LoginSerializer(data=self.login_data) self.assertTrue(serializer.is_valid()) - def test_missing_email(self): - """Test validation fails without email""" + def test_missing_username(self): + """Test validation fails without username""" data = {"password": fake.password(length=12, special_chars=False)} serializer = LoginSerializer(data=data) self.assertFalse(serializer.is_valid()) - self.assertIn("email", serializer.errors) + self.assertIn("username", serializer.errors) def test_missing_password(self): """Test validation fails without password""" - data = {"email": fake.email()} + data = {"username": fake.user_name()} serializer = LoginSerializer(data=data) self.assertFalse(serializer.is_valid()) self.assertIn("password", serializer.errors) diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index 387545c..344641b 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -86,7 +86,7 @@ class LoginViewTest(APITestCase): self.password = fake.password(length=12, special_chars=False) self.user = UserFactory.create_user(password=self.password) - self.login_data = {"email": self.user.email, "password": self.password} + self.login_data = {"username": self.user.username, "password": self.password} def test_login_success(self): """Test successful login""" @@ -109,7 +109,7 @@ class LoginViewTest(APITestCase): def test_login_nonexistent_user(self): """Test login fails for nonexistent user""" data = { - "email": fake.unique.email(), + "username": fake.unique.user_name(), "password": fake.password(length=12, special_chars=False), } @@ -302,14 +302,18 @@ class TokenRefreshViewTest(APITestCase): response = self.client.post(self.refresh_url, data, format="json") self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) - self.assertIn("error", response.data) + self.assertTrue( + "errors" in response.data + or "detail" in response.data + or "code" in response.data + ) def test_refresh_token_missing(self): """Test token refresh fails without refresh token""" response = self.client.post(self.refresh_url, {}, format="json") self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn("error", response.data) + self.assertTrue("errors" in response.data or "detail" in response.data) class ApiJwtOnlyAuthenticationTest(APITestCase): diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 81d9f79..de0cc59 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,5 +1,5 @@ """Test utilities.""" -from .http_server import TestHTTPServer, Response +from .http_server import Response, TestHTTPServer __all__ = ["TestHTTPServer", "Response"] diff --git a/tests/utils/fixtures.py b/tests/utils/fixtures.py index 7d3c925..3bcbd6b 100644 --- a/tests/utils/fixtures.py +++ b/tests/utils/fixtures.py @@ -4,8 +4,8 @@ from __future__ import annotations import io import zipfile +from collections.abc import Iterable from dataclasses import dataclass -from typing import Iterable from faker import Faker from openpyxl import Workbook @@ -67,7 +67,9 @@ def _digits(length: int) -> str: return "".join(str(fake.random_int(0, 9)) for _ in range(length)) -def build_minpromtorg_certificates_excel(count: int = 5) -> tuple[bytes, list[CertificateRow]]: +def build_minpromtorg_certificates_excel( + count: int = 5, +) -> tuple[bytes, list[CertificateRow]]: wb = Workbook() ws = wb.active ws.append( @@ -158,18 +160,18 @@ def build_proverki_xml(count: int = 3) -> tuple[bytes, list[InspectionRow]]: rows.append(row) parts.append( "" + f'ERPID="{row.registration_number}" ' + f'INN="{row.inn}" ' + f'OGRN="{row.ogrn}" ' + f'ORG_NAME="{row.organisation_name}" ' + f'FRGU_ORG_NAME="{row.control_authority}" ' + f'ITYPE_NAME="{row.inspection_type}" ' + f'ICARRYOUT_TYPE_NAME="{row.inspection_form}" ' + f'START_DATE="{row.start_date}" ' + f'END_DATE="{row.end_date}" ' + f'STATUS="{row.status}" ' + f'FZ_NAME="{row.legal_basis}" ' + f'RESULT="{row.result}" />' ) parts.append("") diff --git a/tests/utils/http_server.py b/tests/utils/http_server.py index ba41655..eeb6b00 100644 --- a/tests/utils/http_server.py +++ b/tests/utils/http_server.py @@ -3,12 +3,11 @@ from __future__ import annotations import json +from collections.abc import Callable from dataclasses import dataclass, field from types import SimpleNamespace -from typing import Callable from urllib.parse import urlparse -import requests from requests.adapters import BaseAdapter from requests.models import Response as RequestsResponse @@ -123,7 +122,7 @@ class TestHTTPServer: def stop(self) -> None: self._started = False - def __enter__(self) -> "TestHTTPServer": + def __enter__(self) -> TestHTTPServer: self.start() return self