From 3d298ce352a20f9855cc72a5c9d6b1a008041da7 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 17 Mar 2026 12:56:48 +0100 Subject: [PATCH] feat: expand platform APIs, sources, and test coverage --- .gitignore | 1 + docs/adr/ADR-001: Platform Version Policy.md | 47 ++ .../ADR-002: Technology Stack Selection.md | 39 + ...003: Background Processing Architecture.md | 38 + ...DR-004: Data Ingestion and ETL Strategy.md | 34 + ...ADR-005: External Integrations Strategy.md | 30 + docs/adr/ADR-006: Configuration Strategy.md | 25 + docs/adr/ADR-007: Deployment Model.md | 24 + docs/adr/ADR-008: Testing Strategy.md | 27 + docs/adr/ADR-009: Observability.md | 24 + docs/adr/ADR-010: Project Structure.md | 30 + ...ADR-011: Idempotency and Retry Strategy.md | 128 +++ docs/adr/ADR-012: Data Consistency Model.md | 133 +++ .../adr/ADR-013: Parser Stability Strategy.md | 118 +++ ...oning and Backward Compatibility Policy.md | 81 ++ ...n Source of Truth and Secret Management.md | 87 ++ ...Tracking and Operational Recovery Model.md | 104 +++ docs/adr/ADR-INDEX.md | 20 + src/apps/backups/__init__.py | 1 - src/apps/backups/admin.py | 1 - src/apps/backups/apps.py | 1 - src/apps/backups/models.py | 1 - src/apps/backups/serializers.py | 1 - src/apps/backups/services.py | 66 +- src/apps/backups/urls.py | 1 - src/apps/backups/views.py | 14 +- src/apps/core/views.py | 9 +- src/apps/exchange/models.py | 60 +- src/apps/exchange/services.py | 24 +- src/apps/exchange/tasks.py | 4 +- src/apps/exchange/urls.py | 4 +- src/apps/exchange/views.py | 12 +- src/apps/parsers/admin.py | 82 ++ .../parsers/clients/minpromtorg/__init__.py | 10 +- .../parsers/clients/minpromtorg/products.py | 307 +++++++ .../parsers/clients/minpromtorg/schemas.py | 36 + .../0012_add_industrial_product_record.py | 176 ++++ src/apps/parsers/models.py | 87 ++ src/apps/parsers/serializers.py | 150 ++++ src/apps/parsers/services.py | 325 ++++++-- src/apps/parsers/source_cards.py | 772 ++++++++++++++++++ src/apps/parsers/tasks.py | 232 +++++- src/apps/parsers/urls.py | 23 + src/apps/parsers/views.py | 200 ++++- src/apps/registers/models.py | 3 +- src/apps/registers/serializers.py | 4 +- src/apps/registers/services.py | 8 +- src/apps/registers/views.py | 26 +- .../0005_create_default_admin_superuser.py | 3 +- .../0006_create_default_role_groups.py | 38 + src/apps/user/serializers.py | 106 ++- src/apps/user/services.py | 182 ++++- src/apps/user/urls.py | 11 + src/apps/user/views.py | 129 ++- src/core/api_v1_urls.py | 4 + src/core/celery.py | 5 + src/settings/production.py | 4 +- src/settings/test_postgres.py | 8 +- tests/apps/backups/__init__.py | 1 - tests/apps/backups/test_models.py | 17 + tests/apps/backups/test_services.py | 498 +++++++++++ tests/apps/backups/test_tasks.py | 122 +++ tests/apps/backups/test_views.py | 131 ++- tests/apps/core/test_celery_module.py | 61 ++ tests/apps/core/test_exception_handler.py | 6 + tests/apps/core/test_filters.py | 10 + tests/apps/core/test_management_commands.py | 3 +- tests/apps/core/test_mixins.py | 3 + tests/apps/core/test_pagination.py | 22 + tests/apps/core/test_permissions.py | 16 + tests/apps/core/test_signals.py | 8 +- tests/apps/core/test_startup_checks.py | 206 +++++ tests/apps/core/test_views.py | 11 +- tests/apps/exchange/test_models.py | 27 + tests/apps/exchange/test_serializers.py | 29 + tests/apps/exchange/test_service_units.py | 546 +++++++++++++ tests/apps/exchange/test_services.py | 64 ++ tests/apps/exchange/test_tasks.py | 127 +++ tests/apps/exchange/test_views.py | 2 + tests/apps/parsers/factories.py | 26 + tests/apps/parsers/test_clients.py | 174 +++- tests/apps/parsers/test_fns_upload.py | 200 ++++- tests/apps/parsers/test_models.py | 34 + .../apps/parsers/test_procurement_service.py | 20 +- tests/apps/parsers/test_service_helpers.py | 214 +++++ tests/apps/parsers/test_services.py | 198 ++++- .../apps/parsers/test_source_cards_service.py | 256 ++++++ tests/apps/parsers/test_source_cards_views.py | 219 +++++ tests/apps/parsers/test_sources_api_e2e.py | 225 +++++ tests/apps/parsers/test_tasks.py | 126 ++- tests/apps/parsers/test_views.py | 39 +- tests/apps/registers/factories.py | 8 +- tests/apps/registers/test_models.py | 32 + tests/apps/registers/test_serializers.py | 59 ++ tests/apps/registers/test_services.py | 341 ++++++++ tests/apps/registers/test_views.py | 64 +- tests/apps/user/test_admin.py | 14 +- tests/apps/user/test_serializers.py | 84 +- tests/apps/user/test_services.py | 76 ++ tests/apps/user/test_views.py | 169 ++++ tests/utils/fixtures.py | 71 ++ 101 files changed, 8387 insertions(+), 292 deletions(-) create mode 100644 docs/adr/ADR-001: Platform Version Policy.md create mode 100644 docs/adr/ADR-002: Technology Stack Selection.md create mode 100644 docs/adr/ADR-003: Background Processing Architecture.md create mode 100644 docs/adr/ADR-004: Data Ingestion and ETL Strategy.md create mode 100644 docs/adr/ADR-005: External Integrations Strategy.md create mode 100644 docs/adr/ADR-006: Configuration Strategy.md create mode 100644 docs/adr/ADR-007: Deployment Model.md create mode 100644 docs/adr/ADR-008: Testing Strategy.md create mode 100644 docs/adr/ADR-009: Observability.md create mode 100644 docs/adr/ADR-010: Project Structure.md create mode 100644 docs/adr/ADR-011: Idempotency and Retry Strategy.md create mode 100644 docs/adr/ADR-012: Data Consistency Model.md create mode 100644 docs/adr/ADR-013: Parser Stability Strategy.md create mode 100644 docs/adr/ADR-014: API Versioning and Backward Compatibility Policy.md create mode 100644 docs/adr/ADR-015: Configuration Source of Truth and Secret Management.md create mode 100644 docs/adr/ADR-016: Background Job Tracking and Operational Recovery Model.md create mode 100644 docs/adr/ADR-INDEX.md create mode 100644 src/apps/parsers/clients/minpromtorg/products.py create mode 100644 src/apps/parsers/migrations/0012_add_industrial_product_record.py create mode 100644 src/apps/parsers/source_cards.py create mode 100644 src/apps/user/migrations/0006_create_default_role_groups.py create mode 100644 tests/apps/backups/test_models.py create mode 100644 tests/apps/backups/test_services.py create mode 100644 tests/apps/backups/test_tasks.py create mode 100644 tests/apps/core/test_celery_module.py create mode 100644 tests/apps/core/test_pagination.py create mode 100644 tests/apps/core/test_startup_checks.py create mode 100644 tests/apps/exchange/test_models.py create mode 100644 tests/apps/exchange/test_serializers.py create mode 100644 tests/apps/exchange/test_service_units.py create mode 100644 tests/apps/exchange/test_tasks.py create mode 100644 tests/apps/parsers/test_service_helpers.py create mode 100644 tests/apps/parsers/test_source_cards_service.py create mode 100644 tests/apps/parsers/test_source_cards_views.py create mode 100644 tests/apps/parsers/test_sources_api_e2e.py create mode 100644 tests/apps/registers/test_models.py create mode 100644 tests/apps/registers/test_serializers.py create mode 100644 tests/apps/registers/test_services.py diff --git a/.gitignore b/.gitignore index ebc84b9..7ff4075 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ data/ .zed/ .env.prod tmp/ +_tmp/ diff --git a/docs/adr/ADR-001: Platform Version Policy.md b/docs/adr/ADR-001: Platform Version Policy.md new file mode 100644 index 0000000..697a818 --- /dev/null +++ b/docs/adr/ADR-001: Platform Version Policy.md @@ -0,0 +1,47 @@ +# ADR-001: Platform Version Policy + +## Status +Accepted + +## Context + +Проект разрабатывается и эксплуатируется в ограниченном (регулируемом) технологическом контуре. + +Доступные версии ПО определяются: +- внутренними репозиториями и зеркалами +- требованиями безопасности +- сертификацией +- инфраструктурными ограничениями + +Использование последних upstream-версий не является возможным или допустимым. + +## Decision + +Проект использует фиксированный (approved) стек версий: + +- Python 3.11.x +- Django 3.2.x +- PostgreSQL 15.x +- Redis 7.x +- Celery 5.3.x + +Политика обновлений: + +- PATCH-обновления внутри approved ветки — разрешены +- MINOR/MAJOR — только после отдельного согласования +- Источник правды — lock-файлы и Docker-образы + +## Consequences + +### Positive +- стабильность среды +- воспроизводимость окружения +- соответствие требованиям контура + +### Negative +- невозможность использовать новые фичи +- потенциальный техдолг (осознанный) + +## Alternatives + +- Использование latest upstream — отклонено (несовместимо с контуром) \ No newline at end of file diff --git a/docs/adr/ADR-002: Technology Stack Selection.md b/docs/adr/ADR-002: Technology Stack Selection.md new file mode 100644 index 0000000..8e1d5a2 --- /dev/null +++ b/docs/adr/ADR-002: Technology Stack Selection.md @@ -0,0 +1,39 @@ +# ADR-002: Technology Stack Selection + +## Status +Accepted + +## Context + +Необходим backend для: +- ETL обработки +- интеграции с внешними источниками +- фоновых задач +- администрирования данных + +## Decision + +Выбран стек: + +- Django — основной framework +- Django REST Framework — API +- Celery — асинхронные задачи +- PostgreSQL — основное хранилище +- Redis — брокер и кеш +- Docker Compose — оркестрация + +## Consequences + +### Positive +- зрелый стек +- высокая предсказуемость +- большой опыт эксплуатации + +### Negative +- монолитная архитектура +- ограниченная гибкость по сравнению с microservices + +## Alternatives + +- FastAPI — отклонён (меньше зрелости в админке и ORM экосистеме) +- Kubernetes — избыточен для текущего контура \ No newline at end of file diff --git a/docs/adr/ADR-003: Background Processing Architecture.md b/docs/adr/ADR-003: Background Processing Architecture.md new file mode 100644 index 0000000..51b1af5 --- /dev/null +++ b/docs/adr/ADR-003: Background Processing Architecture.md @@ -0,0 +1,38 @@ +# ADR-003: Background Processing Architecture + +## Status +Accepted + +## Context + +Обработка данных требует: +- асинхронности +- планирования задач +- устойчивости к сбоям + +## Decision + +Используется Celery: + +- worker — выполнение задач +- beat — планировщик +- Redis — broker/backend + +Типы задач: +- парсинг источников +- синхронизация данных +- обработка файлов + +## Consequences + +### Positive +- горизонтальное масштабирование +- разделение runtime и фоновых задач + +### Negative +- сложность дебага +- необходимость контроля idempotency + +## Alternatives + +- RQ / Dramatiq — отклонены (меньше зрелости) \ No newline at end of file diff --git a/docs/adr/ADR-004: Data Ingestion and ETL Strategy.md b/docs/adr/ADR-004: Data Ingestion and ETL Strategy.md new file mode 100644 index 0000000..f9027d6 --- /dev/null +++ b/docs/adr/ADR-004: Data Ingestion and ETL Strategy.md @@ -0,0 +1,34 @@ +# ADR-004: Data Ingestion and ETL Strategy + +## Status +Accepted + +## Context + +Система интегрируется с нестабильными внешними источниками: +- гос API +- HTML/JS порталы +- файлы (Excel/XML) + +## Decision + +Используется ETL-подход: + +- Extract — парсеры +- Transform — сервисный слой +- Load — PostgreSQL + +Особенности: +- инкрементальная загрузка +- потоковый парсинг больших файлов +- обработка ошибок + +## Consequences + +### Positive +- контроль над данными +- устойчивость к изменениям источников + +### Negative +- сложность поддержки парсеров +- необходимость ручного восстановления \ No newline at end of file diff --git a/docs/adr/ADR-005: External Integrations Strategy.md b/docs/adr/ADR-005: External Integrations Strategy.md new file mode 100644 index 0000000..9a0d1ac --- /dev/null +++ b/docs/adr/ADR-005: External Integrations Strategy.md @@ -0,0 +1,30 @@ +# ADR-005: External Integrations Strategy + +## Status +Accepted + +## Context + +Внешние источники: +- proverki.gov.ru +- zakupki.gov.ru +- ФНС +- Минпромторг + +## Decision + +Подход: + +- каждый источник — отдельный parser module +- Playwright используется для JS-порталов +- SOAP/API — через клиентов + +## Consequences + +### Positive +- изоляция интеграций +- возможность независимого развития + +### Negative +- высокая хрупкость +- зависимость от изменений внешних систем \ No newline at end of file diff --git a/docs/adr/ADR-006: Configuration Strategy.md b/docs/adr/ADR-006: Configuration Strategy.md new file mode 100644 index 0000000..988f857 --- /dev/null +++ b/docs/adr/ADR-006: Configuration Strategy.md @@ -0,0 +1,25 @@ +# ADR-006: Configuration and Environment Strategy + +## Status +Accepted + +## Context + +Необходимо разделение окружений: +- dev +- prod + +## Decision + +- .env.dev / .env.prod +- DJANGO_SETTINGS_MODULE +- startup checks (fail-fast) + +## Consequences + +### Positive +- предсказуемость запуска +- контроль конфигурации + +### Negative +- риск рассинхронизации env \ No newline at end of file diff --git a/docs/adr/ADR-007: Deployment Model.md b/docs/adr/ADR-007: Deployment Model.md new file mode 100644 index 0000000..4b5b5b6 --- /dev/null +++ b/docs/adr/ADR-007: Deployment Model.md @@ -0,0 +1,24 @@ +# ADR-007: Deployment Model + +## Status +Accepted + +## Context + +Ограниченный контур без Kubernetes. + +## Decision + +Используется Docker Compose: + +- dev — полный стек +- prod — только сервисы (без DB/Redis) + +## Consequences + +### Positive +- простота деплоя +- воспроизводимость + +### Negative +- ограниченная масштабируемость \ No newline at end of file diff --git a/docs/adr/ADR-008: Testing Strategy.md b/docs/adr/ADR-008: Testing Strategy.md new file mode 100644 index 0000000..1cfba5e --- /dev/null +++ b/docs/adr/ADR-008: Testing Strategy.md @@ -0,0 +1,27 @@ +# ADR-008: Testing Strategy + +## Status +Accepted + +## Context + +Нужно тестировать: +- бизнес-логику +- парсеры +- фоновые задачи + +## Decision + +- тесты вне src +- pytest +- отдельные скрипты запуска +- режим "production-like" + +## Consequences + +### Positive +- изоляция тестов +- удобство CI + +### Negative +- сложность настройки среды \ No newline at end of file diff --git a/docs/adr/ADR-009: Observability.md b/docs/adr/ADR-009: Observability.md new file mode 100644 index 0000000..fc344e0 --- /dev/null +++ b/docs/adr/ADR-009: Observability.md @@ -0,0 +1,24 @@ +# ADR-009: Observability and Health Checks + +## Status +Accepted + +## Context + +Необходим контроль состояния системы. + +## Decision + +- /health +- /health/live +- /health/ready +- логирование через Docker + +## Consequences + +### Positive +- готовность к оркестрации +- диагностика проблем + +### Negative +- ограниченная глубина мониторинга \ No newline at end of file diff --git a/docs/adr/ADR-010: Project Structure.md b/docs/adr/ADR-010: Project Structure.md new file mode 100644 index 0000000..47dee34 --- /dev/null +++ b/docs/adr/ADR-010: Project Structure.md @@ -0,0 +1,30 @@ +# ADR-010: Project Structure and Modularity + +## Status +Accepted + +## Context + +Проект должен быть: +- расширяемым +- читаемым + +## Decision + +Структура: + +- apps/ + - core + - user + - parsers +- core/ +- settings/ + +## Consequences + +### Positive +- модульность +- понятная навигация + +### Negative +- возможная связность между модулями \ No newline at end of file diff --git a/docs/adr/ADR-011: Idempotency and Retry Strategy.md b/docs/adr/ADR-011: Idempotency and Retry Strategy.md new file mode 100644 index 0000000..08c23e8 --- /dev/null +++ b/docs/adr/ADR-011: Idempotency and Retry Strategy.md @@ -0,0 +1,128 @@ +# ADR-011: Idempotency and Retry Strategy for Background Tasks + +## Status +Accepted + +## Context + +Система активно использует фоновые задачи Celery для: +- загрузки данных из внешних источников +- инкрементальной синхронизации +- обработки файлов +- периодических сканирований и парсинга + +Внешние источники и фоновые очереди не гарантируют: +- exactly-once delivery +- стабильность сети +- неизменность ответа источника +- отсутствие повторных запусков задач +- отсутствие ручных перезапусков оператором + +Также возможны следующие сценарии: +- worker завершился после частичной записи данных +- задача была запущена повторно по retry +- beat и ручной запуск вызвали одинаковую задачу почти одновременно +- внешняя система ответила ошибкой после того, как часть данных уже была получена +- оператор повторно инициировал синхронизацию за тот же период + +Для такого класса системы идемпотентность является не оптимизацией, а обязательным архитектурным требованием. + +## Decision + +Все фоновые задачи, изменяющие данные или взаимодействующие с нестабильными внешними источниками, должны проектироваться как idempotent-first. + +### Основные правила + +1. Повторный запуск одной и той же задачи не должен приводить к неконтролируемому дублированию данных. +2. Результат повторного выполнения должен быть либо: + - идентичен первому успешному выполнению, + - либо безопасно приводить систему к тому же целевому состоянию. +3. Retry рассматривается как нормальный сценарий эксплуатации, а не как исключение. + +### Идемпотентность обеспечивается за счет + +- уникальных ограничений на уровне БД для естественных бизнес-ключей +- upsert/update-or-create подходов там, где это возможно +- явной привязки загружаемых данных к периоду, источнику, типу выгрузки или внешнему идентификатору +- дедупликации на уровне сервисного слоя перед записью +- разделения этапов extract / transform / load +- фиксации статуса фоновой задачи и контекста её выполнения +- запрета на "append-only" запись без проверки уникальности для синхронизируемых сущностей + +### Retry policy + +Retry допускается только для временных ошибок: +- сетевые сбои +- временная недоступность внешнего источника +- rate limiting +- временные ошибки брокера или инфраструктуры +- временные проблемы с файловой системой или внешним сервисом + +Retry не должен безусловно выполняться для: +- ошибок валидации входных данных +- ошибок схемы/контракта источника +- систематических ошибок парсинга +- нарушений инвариантов модели +- ошибок конфигурации + +Для retry необходимо: +- использовать ограниченное число повторов +- использовать backoff +- логировать причину повтора +- сохранять контекст периода/источника/операции + +### Concurrency policy + +Для задач, работающих по одному и тому же периоду или источнику, должна применяться логическая защита от параллельного конкурентного запуска. + +Предпочтительный порядок контроля: +1. блокировка на уровне бизнес-контракта задачи +2. проверка существующего job-run статуса +3. защита уникальными ограничениями БД +4. безопасное повторное выполнение как fallback + +### Operational policy + +Задача считается корректной, если после: +- retry +- повторного ручного запуска +- запуска за уже обработанный период +- частичного падения и повторного восстановления + +данные остаются консистентными и не требуют ручной чистки в обычном сценарии эксплуатации. + +## Consequences + +### Positive + +- система устойчива к повторным запускам и временным отказам +- снижается риск дублирования данных +- упрощается эксплуатация и ручной re-run задач +- Celery retry становится безопасным штатным механизмом +- упрощается восстановление после сбоев worker-процессов + +### Negative + +- усложняется реализация сервисного слоя и моделей +- требуется дисциплина в проектировании бизнес-ключей +- часть задач становится медленнее из-за дополнительных проверок +- нужны дополнительные ограничения и индексы в БД +- необходимо явно проектировать поведение при partial success + +## Alternatives considered + +### 1. At-most-once execution +Отклонено, так как не соответствует реальной природе фоновой обработки и нестабильных внешних интеграций. + +### 2. Полагаться только на Celery retry без идемпотентности на уровне домена +Отклонено, так как это приводит к дублированию данных и хрупкому поведению при сбоях. + +### 3. Полная ручная очистка данных перед каждым повторным запуском +Отклонено, так как не масштабируется и опасно в эксплуатации. + +## Notes + +Следующими связанными решениями должны быть: +- политика дедупликации данных +- модель частичной загрузки и фиксации прогресса +- политика конкурентного запуска задач \ No newline at end of file diff --git a/docs/adr/ADR-012: Data Consistency Model.md b/docs/adr/ADR-012: Data Consistency Model.md new file mode 100644 index 0000000..8db616d --- /dev/null +++ b/docs/adr/ADR-012: Data Consistency Model.md @@ -0,0 +1,133 @@ +# ADR-012: Data Consistency and Partial Load Model + +## Status +Accepted + +## Context + +Система загружает данные из внешних государственных и смежных источников, которые могут: +- быть нестабильны +- отдавать большие объёмы данных +- завершаться по таймауту +- менять формат ответа +- содержать частично поврежденные записи +- быть доступны только частично на момент синхронизации + +Загрузка может выполняться: +- по периоду +- по файлам +- по внешним реестрам +- в инкрементальном режиме + +На практике возможно частичное успешное выполнение: +- часть записей уже сохранена, часть ещё нет +- файл обработан наполовину +- один источник за период обработался, второй — нет +- FZ-248 успешно загружен, а FZ-294 завершился ошибкой +- часть данных валидна, часть — нет + +Без явной модели частичной загрузки система становится трудно восстанавливаемой и операционно непрозрачной. + +## Decision + +Проект принимает модель controlled partial progress. + +Это означает: +- частично выполненная загрузка допустима +- частичный прогресс должен быть наблюдаем и восстанавливаем +- повторный запуск должен быть безопасен +- состояние загрузки должно быть определимо без ручного анализа базы + +### Основные правила + +1. Единицей согласованности является не "вся система целиком", а отдельная операция загрузки: + - период + - файл + - тип источника + - подтип данных + - конкретная синхронизационная задача + +2. Частичная загрузка допустима только если: + - данные записываются идемпотентно + - можно безопасно повторить обработку + - есть способ определить статус операции + +3. Полный rollback всей загрузки не является обязательным требованием. + Вместо этого используется повторяемая и безопасная модель дозагрузки/перезапуска. + +### Модель фиксации состояния + +Для каждой операции загрузки система должна уметь определить: +- что именно загружалось +- за какой период или из какого файла +- когда началась обработка +- завершилась ли она успешно +- была ли частичная ошибка +- можно ли безопасно повторить запуск + +Предпочтительная гранулярность статусов: +- pending +- running +- partial +- failed +- completed + +### Правила работы с частичными данными + +- уже сохраненные валидные данные не удаляются автоматически только из-за того, что операция завершилась частично +- повторный запуск обязан корректно переиспользовать уже записанные сущности +- повреждённые или невалидные записи должны логироваться отдельно +- ошибка части набора данных не должна молча скрываться как общий успех + +### Периодическая синхронизация + +Для периодических источников (например, месячные загрузки): +- период считается завершённым только после явной успешной фиксации +- отсутствие новых записей само по себе не всегда считается ошибкой +- для "пустых" периодов допустима отдельная бизнес-логика подтверждения отсутствия данных + +### Работа с файлами + +Для file-based ingestion: +- файл должен иметь трассируемый жизненный цикл +- необходимо различать: + - файл получен + - файл распознан + - файл обработан успешно + - файл обработан частично + - файл отклонён +- перемещение файла в processed или failed является частью операционного контракта + +## Consequences + +### Positive + +- система сохраняет управляемость при partial failures +- упрощается восстановление после ошибок +- уменьшается потребность в ручной чистке данных +- повышается наблюдаемость загрузок и синхронизаций +- упрощается повторный запуск задач по периоду или файлу + +### Negative + +- модель состояний усложняется +- требуется более аккуратный job tracking +- появляется необходимость различать "partial" и "failed" +- возрастает ответственность сервисного слоя за корректную фиксацию статусов + +## Alternatives considered + +### 1. Полная транзакционность всей синхронизации +Отклонено, так как для больших и внешних ETL-процессов это дорого, хрупко и часто нереализуемо. + +### 2. Игнорировать частичный прогресс и считать любую ошибку полным провалом +Отклонено, так как приводит к потере уже обработанного полезного результата и затрудняет recovery. + +### 3. Append-only ingestion без статусов операции +Отклонено, так как делает систему непрозрачной и плохо сопровождаемой. + +## Notes + +Это решение напрямую зависит от: +- ADR-011 Idempotency and Retry Strategy +- ADR-013 Parser Stability and Source Change Detection \ No newline at end of file diff --git a/docs/adr/ADR-013: Parser Stability Strategy.md b/docs/adr/ADR-013: Parser Stability Strategy.md new file mode 100644 index 0000000..6e9f8ae --- /dev/null +++ b/docs/adr/ADR-013: Parser Stability Strategy.md @@ -0,0 +1,118 @@ +# ADR-013: Parser Stability and Source Change Detection for External Sources + +## Status +Accepted + +## Context + +Система зависит от внешних источников данных, которые находятся вне зоны контроля команды: +- государственные порталы +- HTML-страницы +- JS-heavy интерфейсы +- файловые выгрузки +- SOAP/API интеграции +- Excel/XML источники + +Эти источники подвержены изменениям: +- меняется DOM-структура +- меняются URL и маршруты +- меняются названия полей +- меняется формат выгрузки +- появляются новые обязательные поля +- меняются ограничения доступа +- изменяется поведение клиентского JavaScript + +Для системы такого типа изменение внешнего источника является не исключением, а ожидаемым операционным событием. + +## Decision + +Проект принимает стратегию parser instability as expected. + +Это означает: +- каждый внешний источник считается потенциально нестабильным +- отказ парсера по причине изменения источника рассматривается как ожидаемый класс инцидента +- архитектура должна облегчать локализацию, диагностику и восстановление интеграции + +### Архитектурные правила + +1. Каждый источник должен быть изолирован в собственном parser/client/service слое. +2. Логика получения данных из источника не должна быть размазана по нескольким доменам без явной границы. +3. Изменение одного источника не должно ломать остальные интеграции. +4. Парсер должен быть максимально отделён от логики сохранения данных. +5. Нормализация и запись в БД не должны зависеть от деталей DOM/HTML/XML конкретного источника. + +### Change detection policy + +Система должна позволять определить, что проблема вызвана именно изменением внешнего источника, а не внутренней ошибкой приложения. + +Минимально необходимы: +- явное логирование этапа сбоя +- логирование URL, периода, типа выгрузки или источника +- различение сетевых ошибок, ошибок структуры ответа и ошибок сохранения +- сохранение диагностического контекста для повторного анализа + +### Playwright policy + +Playwright допускается для источников, где: +- критическая логика формируется на клиенте +- прямой HTTP scraping недостаточен +- требуется JS rendering или пользовательская навигация + +При этом Playwright считается более хрупкой интеграционной точкой по сравнению с HTTP/API клиентами. + +Для Playwright-based интеграций принимаются дополнительные ограничения: +- сценарий навигации должен быть максимально коротким +- DOM-селекторы должны быть минимально хрупкими +- логика парсинга должна быть отделена от логики браузерной навигации +- fallback или быстрая диагностика деградации обязательны + +### Resilience policy + +При изменении внешнего источника система должна: +- завершать конкретную задачу контролируемой ошибкой +- не повреждать уже сохранённые данные +- не маскировать сбой как успешную синхронизацию +- сохранять возможность повторного запуска после исправления + +### Ownership policy + +Каждая интеграция с внешним источником должна иметь: +- понятную точку входа +- понятный набор задач +- понятный формат выходных данных +- наблюдаемый статус выполнения + +## Consequences + +### Positive + +- уменьшается связность между интеграциями +- упрощается ремонт конкретного источника +- проще локализовать причину отказа +- снижается риск каскадного повреждения всей системы +- повышается операционная предсказуемость + +### Negative + +- возрастает количество прикладного кода и адаптеров +- приходится поддерживать отдельные контракты на источник +- растут требования к логированию и smoke-проверкам +- интеграции невозможно считать "написал один раз и забыл" + +## Alternatives considered + +### 1. Общий универсальный парсер для разных источников +Отклонено, так как источники сильно отличаются по протоколу, структуре и стабильности. + +### 2. Встраивание логики источников прямо в задачи или view-слой +Отклонено, так как это увеличивает связность и ухудшает сопровождаемость. + +### 3. Опора только на "ручное замечание", что парсер сломался +Отклонено, так как это не соответствует требованиям эксплуатационной зрелости. + +## Notes + +Следующим развитием данного решения должны быть: +- smoke checks для критичных источников +- классификация типов ошибок интеграции +- runbook для восстановления парсеров после изменения внешних порталов \ No newline at end of file diff --git a/docs/adr/ADR-014: API Versioning and Backward Compatibility Policy.md b/docs/adr/ADR-014: API Versioning and Backward Compatibility Policy.md new file mode 100644 index 0000000..ab996be --- /dev/null +++ b/docs/adr/ADR-014: API Versioning and Backward Compatibility Policy.md @@ -0,0 +1,81 @@ +# ADR-014: API Versioning and Backward Compatibility Policy + +## Status +Accepted + +## Context + +Система предоставляет API для: +- внутренних сервисов +- админ-панели +- потенциальных внешних интеграций + +API развивается со временем: +- добавляются новые поля +- меняется структура ответов +- появляются новые эндпоинты +- изменяется бизнес-логика + +Без политики версионирования возможны: +- поломка клиентов +- неявные регрессии +- невозможность безопасного развития API + +## Decision + +Используется versioned API через URL: + +- /api/v1/ + +Версия API фиксируется в URL и не изменяется неявно. + +### Правила изменения API + +#### Разрешено (без изменения версии) + +- добавление новых эндпоинтов +- добавление новых необязательных полей +- расширение enum-значений (если не ломает клиентов) +- улучшение производительности без изменения контракта + +#### Запрещено без новой версии + +- удаление полей +- изменение типа поля +- изменение обязательности поля +- изменение структуры ответа +- изменение семантики существующего поля + +### Deprecation policy + +- устаревшие эндпоинты помечаются как deprecated +- поддерживаются ограниченное время +- удаляются только после явного перехода клиентов + +### Backward compatibility + +Система гарантирует: +- сохранение контракта внутри одной версии API +- предсказуемое поведение для существующих клиентов + +## Consequences + +### Positive + +- безопасное развитие API +- предсказуемость для клиентов +- снижение риска регрессий + +### Negative + +- необходимость поддержки старых версий +- усложнение документации +- возможный рост технического долга + +## Alternatives considered + +### 1. Версионирование через заголовки +Отклонено — сложнее отлаживать и документировать. + +### 2. Отсутствие версионирования +Отклонено — высокий риск поломки клиентов. \ No newline at end of file diff --git a/docs/adr/ADR-015: Configuration Source of Truth and Secret Management.md b/docs/adr/ADR-015: Configuration Source of Truth and Secret Management.md new file mode 100644 index 0000000..e6f9ad3 --- /dev/null +++ b/docs/adr/ADR-015: Configuration Source of Truth and Secret Management.md @@ -0,0 +1,87 @@ +# ADR-015: Configuration Source of Truth and Secret Management + +## Status +Accepted + +## Context + +Система использует множество конфигураций: +- база данных +- Redis +- Celery +- внешние API (FNS, закупки и т.д.) +- Django settings + +Ошибки конфигурации приводят к: +- невозможности запуска +- некорректной работе +- утечкам секретов + +Также важно: +- разделение dev/prod +- воспроизводимость окружения +- безопасность секретов + +## Decision + +Используется environment-based configuration. + +### Источники конфигурации + +1. `.env.dev` — локальная разработка +2. `.env.prod` — production +3. `.env.prod.example` — контракт обязательных переменных +4. Docker Compose — точка применения конфигурации +5. `settings.*` — логика конфигурации + +### Source of Truth + +- обязательные переменные определяются через `.env.*` +- структура и использование переменных — в `settings` +- значения по умолчанию должны быть минимальны + +### Secret policy + +- секреты НЕ хранятся в репозитории +- `.env.*` файлы не коммитятся (кроме example) +- реальные значения задаются через: + - env-файлы + - секреты инфраструктуры + - CI/CD переменные + +### Fail-fast policy + +При старте приложения: +- проверяется доступность DB +- проверяется Redis +- проверяются критические переменные + +При ошибке: +- приложение не стартует + +### Configuration constraints + +- запрещено хардкодить значения в коде +- запрещено использовать разные имена переменных для одного и того же +- запрещено использовать устаревшие пути (например config.settings.*) + +## Consequences + +### Positive + +- воспроизводимость окружения +- безопасность +- предсказуемость запуска + +### Negative + +- необходимость поддерживать env-файлы +- риск рассинхронизации example и реальных env + +## Alternatives considered + +### 1. Конфигурация через код +Отклонено — небезопасно и негибко. + +### 2. Хранение секретов в репозитории +Отклонено — нарушает требования безопасности. diff --git a/docs/adr/ADR-016: Background Job Tracking and Operational Recovery Model.md b/docs/adr/ADR-016: Background Job Tracking and Operational Recovery Model.md new file mode 100644 index 0000000..1b69157 --- /dev/null +++ b/docs/adr/ADR-016: Background Job Tracking and Operational Recovery Model.md @@ -0,0 +1,104 @@ +# ADR-016: Background Job Tracking and Operational Recovery Model + +## Status +Accepted + +## Context + +Система использует фоновые задачи для: +- парсинга данных +- синхронизации источников +- обработки файлов + +Возможны сценарии: +- задача упала +- задача зависла +- задача выполнилась частично +- задача была запущена повторно +- оператору нужно вручную перезапустить процесс + +Без модели трекинга задач: +- невозможно понять состояние системы +- невозможно безопасно повторить операцию +- сложно диагностировать ошибки + +## Decision + +Вводится явная модель отслеживания фоновых задач (BackgroundJob). + +### Основные свойства задачи + +Каждая задача должна иметь: + +- тип (parser / sync / file processing) +- источник данных +- период или файл +- статус +- время запуска +- время завершения +- результат (успех / ошибка / частично) +- контекст выполнения + +### Статусы задач + +Минимальный набор: + +- pending +- running +- completed +- failed +- partial + +### Правила выполнения + +1. Каждая бизнес-операция должна быть трассируема через job. +2. Повторный запуск не должен создавать конфликтов (см. ADR-011). +3. Статус задачи должен отражать реальное состояние, а не "успешно/не успешно". +4. Частичный успех должен фиксироваться явно. + +### Recovery model + +Система должна позволять: + +- повторный запуск задачи за тот же период +- повторную обработку файла +- безопасный re-run без очистки БД +- диагностику причины ошибки через лог и статус + +### Manual operations + +Администратор должен иметь возможность: + +- увидеть список задач +- понять их состояние +- повторно запустить задачу +- увидеть ошибки + +### Integration with Celery + +- Celery отвечает за выполнение +- BackgroundJob — за бизнес-трекинг +- ID задачи Celery связывается с доменной задачей + +## Consequences + +### Positive + +- прозрачность фоновых процессов +- управляемость ETL +- возможность безопасного восстановления +- упрощение эксплуатации + +### Negative + +- дополнительный слой логики +- необходимость синхронизации статусов +- увеличение количества записей в БД + +## Alternatives considered + +### 1. Полагаться только на Celery task state +Отклонено — недостаточно для бизнес-анализа и recovery. + +### 2. Отсутствие трекинга задач +Отклонено — делает систему неуправляемой. diff --git a/docs/adr/ADR-INDEX.md b/docs/adr/ADR-INDEX.md new file mode 100644 index 0000000..9de6723 --- /dev/null +++ b/docs/adr/ADR-INDEX.md @@ -0,0 +1,20 @@ +# Architecture Decision Records (ADR) + +## Index + +- ADR-001: Platform Version Policy +- ADR-002: Technology Stack Selection +- ADR-003: Background Processing Architecture (Celery) +- ADR-004: Data Ingestion and ETL Strategy +- ADR-005: External Integrations Strategy +- ADR-006: Configuration and Environment Management +- ADR-007: Deployment Model (Docker Compose) +- ADR-008: Testing Strategy +- ADR-009: Observability and Health Checks +- ADR-010: Project Structure and Modularity + +## Future ADRs (Proposed) + +- ADR-011: Idempotency and Retry Strategy +- ADR-012: Data Deduplication and Consistency Model +- ADR-013: Parser Stability and Change Detection \ No newline at end of file diff --git a/src/apps/backups/__init__.py b/src/apps/backups/__init__.py index a1f4418..0362196 100644 --- a/src/apps/backups/__init__.py +++ b/src/apps/backups/__init__.py @@ -1,2 +1 @@ """Приложение экспорта защищённых резервных архивов.""" - diff --git a/src/apps/backups/admin.py b/src/apps/backups/admin.py index 1e19498..6092b9b 100644 --- a/src/apps/backups/admin.py +++ b/src/apps/backups/admin.py @@ -36,4 +36,3 @@ class BackupExportJobAdmin(admin.ModelAdmin): "updated_at", ] ordering = ["-actual_date", "-created_at"] - diff --git a/src/apps/backups/apps.py b/src/apps/backups/apps.py index fb4ae92..06aefec 100644 --- a/src/apps/backups/apps.py +++ b/src/apps/backups/apps.py @@ -10,4 +10,3 @@ class BackupsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.backups" verbose_name = _("Резервные копии") - diff --git a/src/apps/backups/models.py b/src/apps/backups/models.py index 4d8d649..0367356 100644 --- a/src/apps/backups/models.py +++ b/src/apps/backups/models.py @@ -92,4 +92,3 @@ class BackupExportJob(TimestampMixin, models.Model): def __str__(self) -> str: return f"Backup {self.actual_date} [{self.status}]" - diff --git a/src/apps/backups/serializers.py b/src/apps/backups/serializers.py index fa259fe..2ea38f5 100644 --- a/src/apps/backups/serializers.py +++ b/src/apps/backups/serializers.py @@ -13,4 +13,3 @@ class BackupExportRequestSerializer(serializers.Serializer): "Если не передана, используется текущая дата." ), ) - diff --git a/src/apps/backups/services.py b/src/apps/backups/services.py index d4f9c53..5345111 100644 --- a/src/apps/backups/services.py +++ b/src/apps/backups/services.py @@ -7,6 +7,7 @@ import hashlib import json import os import struct +import uuid import zlib from collections.abc import Iterable from contextlib import suppress @@ -144,7 +145,9 @@ class BackupExportService: started_at__lte=actual_date, ).filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date)) - register_ids = list(active_periods.values_list("registry_id", flat=True).distinct()) + register_ids = list( + active_periods.values_list("registry_id", flat=True).distinct() + ) upload_ids = list( active_periods.values_list("started_by_upload_id", flat=True).distinct() ) @@ -155,9 +158,13 @@ class BackupExportService: report_ids = list(reports_qs.values_list("id", flat=True)) export_map: dict[type[Model], Iterable] = { - Organization: Organization.objects.filter(id__in=active_org_ids).order_by("id"), + Organization: Organization.objects.filter(id__in=active_org_ids).order_by( + "id" + ), Register: Register.objects.filter(id__in=register_ids).order_by("name"), - RegisterUpload: RegisterUpload.objects.filter(id__in=upload_ids).order_by("id"), + RegisterUpload: RegisterUpload.objects.filter(id__in=upload_ids).order_by( + "id" + ), RegistryMembershipPeriod: active_periods.order_by( "registry_id", "organization_id", @@ -216,7 +223,9 @@ class BackupExportService: } if field.is_relation and field.related_model is not None: - on_delete_handler = getattr(field.remote_field.on_delete, "__name__", "unknown") + on_delete_handler = getattr( + field.remote_field.on_delete, "__name__", "unknown" + ) field_meta["related_model"] = field.related_model._meta.label field_meta["on_delete"] = on_delete_handler @@ -249,15 +258,14 @@ class BackupExportService: } @classmethod - def _serialize_queryset(cls, *, model: type[Model], queryset: Iterable) -> list[dict]: + def _serialize_queryset( + cls, *, model: type[Model], queryset: Iterable + ) -> list[dict]: field_names = [field.attname for field in model._meta.local_fields] serialized = [] for row in queryset.values(*field_names).iterator(chunk_size=1000): serialized.append( - { - key: cls._normalize_value(value) - for key, value in row.items() - } + {key: cls._normalize_value(value) for key, value in row.items()} ) return serialized @@ -320,7 +328,9 @@ class BackupExportService: return decoded_key @classmethod - def _build_bin_container(cls, *, encrypted_payload: bytes, header_payload: dict) -> bytes: + def _build_bin_container( + cls, *, encrypted_payload: bytes, header_payload: dict + ) -> bytes: header = { "format": "mostovik-backup-bin", "version": cls.BIN_FORMAT_VERSION, @@ -375,7 +385,9 @@ class BackupExportJobService: requested_by_id: int | None, ) -> BackupRequestResult: job = cls._get_job_for_update(actual_date) - existing_job_result = cls._result_for_existing_job(actual_date=actual_date, job=job) + existing_job_result = cls._result_for_existing_job( + actual_date=actual_date, job=job + ) if existing_job_result is not None: return existing_job_result @@ -408,17 +420,27 @@ class BackupExportJobService: status=BackupExportJob.Status.PENDING, ) - from apps.backups.tasks import generate_backup_for_date - - task = generate_backup_for_date.delay(job_id=new_job.id) - new_job.task_id = task.id or "" + task_id = str(uuid.uuid4()) + new_job.task_id = task_id new_job.save(update_fields=["task_id", "updated_at"]) + transaction.on_commit( + lambda: cls._enqueue_backup_task(job_id=new_job.id, task_id=task_id) + ) return BackupRequestResult( action="started", message="Формирование бэкапа запущено.", actual_date=actual_date, - task_id=new_job.task_id, + task_id=task_id, + ) + + @staticmethod + def _enqueue_backup_task(*, job_id: int, task_id: str) -> None: + from apps.backups.tasks import generate_backup_for_date + + generate_backup_for_date.apply_async( + kwargs={"job_id": job_id}, + task_id=task_id, ) @classmethod @@ -431,7 +453,10 @@ class BackupExportJobService: if job is None: return None - if job.status in (BackupExportJob.Status.PENDING, BackupExportJob.Status.STARTED): + if job.status in ( + BackupExportJob.Status.PENDING, + BackupExportJob.Status.STARTED, + ): return BackupRequestResult( action="wait", message="Бэкап формируется, пожалуйста подождите.", @@ -461,7 +486,9 @@ class BackupExportJobService: if not cls._archive_exists(job): job.delete() - raise BackupExportError("Файл бэкапа отсутствует, запустите формирование снова") + raise BackupExportError( + "Файл бэкапа отсутствует, запустите формирование снова" + ) archive_path = Path(job.archive_path) archive_bytes = archive_path.read_bytes() @@ -475,7 +502,8 @@ class BackupExportJobService: archive_filename=archive_filename, bin_filename="", checksum_filename=job.checksum_filename, - checksum_sha256=job.checksum_sha256 or hashlib.sha256(archive_bytes).hexdigest(), + checksum_sha256=job.checksum_sha256 + or hashlib.sha256(archive_bytes).hexdigest(), organizations_count=job.organizations_count, actual_date=job.actual_date, ) diff --git a/src/apps/backups/urls.py b/src/apps/backups/urls.py index daad737..891c7de 100644 --- a/src/apps/backups/urls.py +++ b/src/apps/backups/urls.py @@ -10,4 +10,3 @@ backups_urlpatterns = [ ] urlpatterns = [] - diff --git a/src/apps/backups/views.py b/src/apps/backups/views.py index 5f79588..3ec146a 100644 --- a/src/apps/backups/views.py +++ b/src/apps/backups/views.py @@ -64,12 +64,16 @@ class BackupExportView(APIView): def post(self, request): serializer = BackupExportRequestSerializer(data=request.data) serializer.is_valid(raise_exception=True) - actual_date = serializer.validated_data.get("actual_date") or timezone.localdate() + actual_date = ( + serializer.validated_data.get("actual_date") or timezone.localdate() + ) try: result = BackupExportJobService.check_or_start_job( actual_date=actual_date, - requested_by_id=request.user.id if request.user.is_authenticated else None, + requested_by_id=request.user.id + if request.user.is_authenticated + else None, ) except BackupExportError as exc: raise ValidationError({"backup": str(exc)}) from exc @@ -94,9 +98,9 @@ class BackupExportView(APIView): response = HttpResponse(artifact.archive_bytes, content_type="application/zip") response.status_code = status.HTTP_200_OK - response["Content-Disposition"] = ( - f'attachment; filename="{artifact.archive_filename}"' - ) + response[ + "Content-Disposition" + ] = f'attachment; filename="{artifact.archive_filename}"' response["X-Backup-SHA256"] = artifact.checksum_sha256 response["X-Backup-Checksum-File"] = artifact.checksum_filename response["X-Backup-Organizations"] = str(artifact.organizations_count) diff --git a/src/apps/core/views.py b/src/apps/core/views.py index 5c2a521..1842f6e 100644 --- a/src/apps/core/views.py +++ b/src/apps/core/views.py @@ -232,8 +232,9 @@ class BackgroundJobStatusView(APIView): job = BackgroundJobService.get_by_task_id(task_id) - # Проверка доступа: только владелец или админ - if job.user_id and job.user_id != request.user.id and not request.user.is_staff: + # Проверка доступа: только владелец или админ. + # Задачи без владельца считаем системными и не показываем обычным пользователям. + if not request.user.is_staff and job.user_id != request.user.id: return Response( {"detail": "Нет доступа к этой задаче"}, status=status.HTTP_403_FORBIDDEN, @@ -275,7 +276,9 @@ class BackgroundJobListView(APIView): try: limit = int(limit_raw) except (TypeError, ValueError) as exc: - raise ValidationError({"limit": "Параметр limit должен быть целым числом"}) from exc + raise ValidationError( + {"limit": "Параметр limit должен быть целым числом"} + ) from exc if limit < 1: raise ValidationError({"limit": "Параметр limit должен быть больше 0"}) limit = min(limit, 100) diff --git a/src/apps/exchange/models.py b/src/apps/exchange/models.py index ed731a0..6820464 100644 --- a/src/apps/exchange/models.py +++ b/src/apps/exchange/models.py @@ -1,6 +1,11 @@ """Модели приложения обмена данными.""" +import base64 +import hashlib + from apps.core.mixins import TimestampMixin +from cryptography.fernet import Fernet, InvalidToken +from django.conf import settings from django.db import models from django.db.models import Q from django.utils.translation import gettext_lazy as _ @@ -9,10 +14,15 @@ from django.utils.translation import gettext_lazy as _ class ExchangeConnection(TimestampMixin, models.Model): """Подключение к целевой БД для обмена данными.""" + PASSWORD_PREFIX = "enc:v1:" # noqa: S105 + server = models.CharField(_("сервер"), max_length=255) port = models.PositiveIntegerField(_("порт"), default=5432) username = models.CharField(_("пользователь"), max_length=255) - password = models.TextField(_("пароль"), help_text=_("Хранится в открытом виде")) + password = models.TextField( + _("пароль"), + help_text=_("Хранится в зашифрованном виде"), + ) database_name = models.CharField(_("имя БД"), max_length=255) schema_name = models.CharField(_("имя схемы"), max_length=255, default="public") is_active = models.BooleanField(_("активное"), default=False, db_index=True) @@ -41,3 +51,51 @@ class ExchangeConnection(TimestampMixin, models.Model): f"{self.username}@{self.server}:{self.port}/{self.database_name}" f"[{self.schema_name}]" ) + + @classmethod + def _get_cipher(cls) -> Fernet: + secret_material = ( + getattr(settings, "EXCHANGE_CREDENTIALS_ENCRYPTION_KEY", "") + or settings.SECRET_KEY + ) + digest = hashlib.sha256(secret_material.encode("utf-8")).digest() + return Fernet(base64.urlsafe_b64encode(digest)) + + @classmethod + def is_password_encrypted(cls, value: str) -> bool: + return bool(value) and value.startswith(cls.PASSWORD_PREFIX) + + @classmethod + def encrypt_password(cls, raw_password: str) -> str: + encrypted = ( + cls._get_cipher().encrypt(raw_password.encode("utf-8")).decode("ascii") + ) + return f"{cls.PASSWORD_PREFIX}{encrypted}" + + @classmethod + def decrypt_password(cls, stored_password: str) -> str: + if not cls.is_password_encrypted(stored_password): + return stored_password + + token = stored_password[len(cls.PASSWORD_PREFIX) :].encode("ascii") + try: + return cls._get_cipher().decrypt(token).decode("utf-8") + except InvalidToken as exc: + raise ValueError( + "Не удалось расшифровать пароль exchange connection" + ) from exc + + def get_decrypted_password(self) -> str: + return self.decrypt_password(self.password) + + def save(self, *args, **kwargs): + password_was_encrypted = False + if self.password and not self.is_password_encrypted(self.password): + self.password = self.encrypt_password(self.password) + password_was_encrypted = True + + update_fields = kwargs.get("update_fields") + if password_was_encrypted and update_fields is not None: + kwargs["update_fields"] = list(set(update_fields) | {"password"}) + + super().save(*args, **kwargs) diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py index 1d6ad50..469ab25 100644 --- a/src/apps/exchange/services.py +++ b/src/apps/exchange/services.py @@ -79,7 +79,9 @@ class ExchangeConnectionService: cursor.execute("SELECT 1") except Exception as exc: # noqa: BLE001 cls._mark_connection_error(connection, str(exc)) - raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc + raise ExchangeServiceError( + f"Ошибка подключения к целевой БД: {exc}" + ) from exc return alias @@ -145,7 +147,9 @@ class ExchangeConnectionService: connections[alias].ensure_connection() except Exception as exc: # noqa: BLE001 cls._mark_connection_error(connection, str(exc)) - raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc + raise ExchangeServiceError( + f"Ошибка подключения к целевой БД: {exc}" + ) from exc cls.validate_target_structure( connection=connection, @@ -187,7 +191,7 @@ class ExchangeConnectionService: "ENGINE": "django.db.backends.postgresql", "NAME": connection.database_name, "USER": connection.username, - "PASSWORD": connection.password, + "PASSWORD": connection.get_decrypted_password(), "HOST": connection.server, "PORT": connection.port, "OPTIONS": { @@ -348,7 +352,10 @@ class ExchangeConnectionService: @classmethod def _requires_registry_organizations(cls, models_to_copy: list) -> bool: return any( - any(field.name == "registry_organization" for field in model._meta.local_fields) + any( + field.name == "registry_organization" + for field in model._meta.local_fields + ) for model in models_to_copy ) @@ -377,7 +384,10 @@ class ExchangeConnectionService: pk_name = model._meta.pk.attname for source_obj in queryset.iterator(chunk_size=chunk_size): - row_data = {field_name: getattr(source_obj, field_name) for field_name in field_names} + row_data = { + field_name: getattr(source_obj, field_name) + for field_name in field_names + } batch.append(model(**row_data)) if len(batch) >= chunk_size: @@ -441,7 +451,9 @@ class ExchangeConnectionService: return len(existing_after - existing_before) @classmethod - def _mark_connection_error(cls, connection: ExchangeConnection, error_message: str) -> None: + def _mark_connection_error( + cls, connection: ExchangeConnection, error_message: str + ) -> None: connection.last_checked_at = timezone.now() connection.last_error = error_message connection.save(update_fields=["last_checked_at", "last_error", "updated_at"]) diff --git a/src/apps/exchange/tasks.py b/src/apps/exchange/tasks.py index fff9684..880a27d 100644 --- a/src/apps/exchange/tasks.py +++ b/src/apps/exchange/tasks.py @@ -36,7 +36,9 @@ def copy_parsers_data_async( }, ) - connection = ExchangeConnection.objects.filter(id=connection_id, is_active=True).first() + connection = ExchangeConnection.objects.filter( + id=connection_id, is_active=True + ).first() if connection is None: background_job.fail(error="Активное подключение не найдено") raise ValueError(f"Active exchange connection not found: {connection_id}") diff --git a/src/apps/exchange/urls.py b/src/apps/exchange/urls.py index 3965581..ff493c1 100644 --- a/src/apps/exchange/urls.py +++ b/src/apps/exchange/urls.py @@ -6,7 +6,9 @@ from django.urls import path app_name = "exchange" exchange_urlpatterns = [ - path("connections/", ExchangeConnectionListCreateView.as_view(), name="connections"), + path( + "connections/", ExchangeConnectionListCreateView.as_view(), name="connections" + ), path("copy/", ExchangeCopyDataView.as_view(), name="copy"), ] diff --git a/src/apps/exchange/views.py b/src/apps/exchange/views.py index e8c2b6b..60d4659 100644 --- a/src/apps/exchange/views.py +++ b/src/apps/exchange/views.py @@ -42,7 +42,9 @@ class ExchangeConnectionListCreateView(APIView): }, ) def get(self, request): - queryset = ExchangeConnection.objects.all().order_by("-is_active", "-created_at") + queryset = ExchangeConnection.objects.all().order_by( + "-is_active", "-created_at" + ) serializer = ExchangeConnectionSerializer(queryset, many=True) return api_response(serializer.data, status_code=status.HTTP_200_OK) @@ -122,7 +124,9 @@ class ExchangeCopyDataView(APIView): task = copy_parsers_data_async.delay( connection_id=active_connection.id, payload=serializer.validated_data, - requested_by_id=request.user.id if request.user.is_authenticated else None, + requested_by_id=request.user.id + if request.user.is_authenticated + else None, ) # Предсоздаём запись для мгновенного отслеживания в /api/v1/jobs/{task_id}/ @@ -151,7 +155,9 @@ class ExchangeCopyDataView(APIView): "task_id": task.id, "connection_id": active_connection.id, "mode": serializer.validated_data["mode"], - "truncate_before_copy": serializer.validated_data["truncate_before_copy"], + "truncate_before_copy": serializer.validated_data[ + "truncate_before_copy" + ], }, status_code=status.HTTP_202_ACCEPTED, ) diff --git a/src/apps/parsers/admin.py b/src/apps/parsers/admin.py index a41ff0a..7be01b3 100644 --- a/src/apps/parsers/admin.py +++ b/src/apps/parsers/admin.py @@ -6,6 +6,7 @@ from apps.parsers.models import ( FinancialReport, FinancialReportLine, IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -275,6 +276,87 @@ class ManufacturerRecordAdmin(admin.ModelAdmin): return False +@admin.register(IndustrialProductRecord) +class IndustrialProductRecordAdmin(admin.ModelAdmin): + """Admin для реестра промышленной продукции.""" + + list_display = [ + "registry_number", + "product_name_short", + "organisation_name_short", + "inn", + "ogrn", + "load_batch", + "created_at", + ] + list_filter = ["load_batch", "created_at"] + search_fields = [ + "registry_number", + "product_name", + "full_organisation_name", + "inn", + "ogrn", + "okpd2_code", + "tnved_code", + ] + readonly_fields = ["created_at", "updated_at", "load_batch"] + ordering = ["-created_at"] + list_per_page = 100 + date_hierarchy = "created_at" + + fieldsets = ( + ( + "Запись реестра", + { + "fields": ( + "registry_number", + "product_name", + "product_model", + "regulatory_document", + ) + }, + ), + ( + "Организация", + {"fields": ("full_organisation_name", "inn", "ogrn")}, + ), + ( + "Классификаторы", + {"fields": ("okpd2_code", "tnved_code")}, + ), + ( + "Системное", + { + "fields": ("load_batch", "created_at", "updated_at"), + "classes": ("collapse",), + }, + ), + ) + + def product_name_short(self, obj): + """Сокращённое название продукции.""" + name = obj.product_name or "" + return name[:60] + "..." if len(name) > 60 else name + + def organisation_name_short(self, obj): + """Сокращённое название организации.""" + name = obj.full_organisation_name or "" + return name[:60] + "..." if len(name) > 60 else name + + product_name_short.short_description = "Продукция" + product_name_short.admin_order_field = "product_name" + organisation_name_short.short_description = "Организация" + organisation_name_short.admin_order_field = "full_organisation_name" + + def has_add_permission(self, request): + """Запретить создание записей вручную.""" + return False + + def has_change_permission(self, request, obj=None): + """Запретить редактирование записей.""" + return False + + @admin.register(InspectionRecord) class InspectionRecordAdmin(admin.ModelAdmin): """Admin для проверок из Единого реестра проверок.""" diff --git a/src/apps/parsers/clients/minpromtorg/__init__.py b/src/apps/parsers/clients/minpromtorg/__init__.py index 5fba646..9456f1e 100644 --- a/src/apps/parsers/clients/minpromtorg/__init__.py +++ b/src/apps/parsers/clients/minpromtorg/__init__.py @@ -4,15 +4,23 @@ Источники: - IndustrialProductionClient: сертификаты промышленного производства - ManufacturesClient: реестр производителей +- IndustrialProductsClient: реестр промышленной продукции """ from apps.parsers.clients.minpromtorg.industrial import IndustrialProductionClient from apps.parsers.clients.minpromtorg.manufactures import ManufacturesClient -from apps.parsers.clients.minpromtorg.schemas import IndustrialCertificate, Manufacturer +from apps.parsers.clients.minpromtorg.products import IndustrialProductsClient +from apps.parsers.clients.minpromtorg.schemas import ( + IndustrialCertificate, + IndustrialProduct, + Manufacturer, +) __all__ = [ "IndustrialProductionClient", + "IndustrialProductsClient", "ManufacturesClient", "IndustrialCertificate", + "IndustrialProduct", "Manufacturer", ] diff --git a/src/apps/parsers/clients/minpromtorg/products.py b/src/apps/parsers/clients/minpromtorg/products.py new file mode 100644 index 0000000..62c6acb --- /dev/null +++ b/src/apps/parsers/clients/minpromtorg/products.py @@ -0,0 +1,307 @@ +""" +Клиент для парсинга реестра промышленной продукции Минпромторга. + +Источник: Минпромторг, реестр промышленной продукции РФ. +""" + +import logging +import re +from dataclasses import dataclass, field +from datetime import datetime +from io import BytesIO + +from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError +from apps.parsers.clients.minpromtorg.schemas import IndustrialProduct +from openpyxl import load_workbook +from requests.adapters import BaseAdapter + +logger = logging.getLogger(__name__) + +DEFAULT_HOST = "minpromtorg.gov.ru" +DEFAULT_API_PATH = "/api/kss-document-preview" +DEFAULT_DOC_TYPE = "668d4f2a-966a-4b65-9fb9-2f1ad19a3d1f" +DEFAULT_QUERY = ( + "Реестр промышленной продукции, произведенной на территории " + "Российской Федерации" +) +DATE_PATTERN = re.compile(r"(\d{8})") + +HEADER_ALIASES = { + "full_organisation_name": { + "полное наименование организации", + "наименование организации", + "организация", + "full_organisation_name", + }, + "ogrn": {"огрн", "ogrn"}, + "inn": {"инн", "inn"}, + "registry_number": { + "регистрационный номер записи", + "регистрационный номер", + "реестровый номер", + "номер реестровой записи", + "registry_number", + }, + "product_name": { + "наименование продукции", + "наименование промышленной продукции", + "продукция", + "product_name", + }, + "product_model": { + "модель или модификация", + "модельмодификация", + "модель или модификация продукции", + "product_model", + }, + "okpd2_code": { + "код по окпд2", + "окпд2", + "okpd2_code", + }, + "tnved_code": { + "код по тн вэд", + "тн вэд", + "тнвэд", + "tnved_code", + }, + "regulatory_document": { + "наименование нормативного документа", + "нормативный документ", + "regulatory_document", + }, +} + +REQUIRED_HEADERS = { + "full_organisation_name", + "inn", + "ogrn", + "registry_number", + "product_name", +} + + +class IndustrialProductsClientError(HTTPClientError): + """Ошибка клиента реестра промышленной продукции.""" + + pass + + +def _normalize_header(value) -> str: + text = str(value or "").strip().lower() + text = text.replace("\n", " ") + text = re.sub(r"\s+", " ", text) + return re.sub(r"[^a-zа-я0-9]+", "", text) + + +@dataclass +class IndustrialProductsClient: + """ + Клиент для получения реестра промышленной продукции Минпромторга. + + Разбирает Excel по заголовкам, а не по жестким индексам колонок, чтобы + выдерживать небольшие изменения структуры выгрузки. + """ + + proxies: list[str] | None = None + host: str = DEFAULT_HOST + scheme: str = "https" + api_path: str = DEFAULT_API_PATH + doc_type: str = DEFAULT_DOC_TYPE + query: str = DEFAULT_QUERY + timeout: int = 120 + http_adapter: BaseAdapter | None = None + _http_client: BaseHTTPClient | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + self._http_client = None + + @property + def http_client(self) -> BaseHTTPClient: + if self._http_client is None: + self._http_client = BaseHTTPClient( + base_url=f"{self.scheme}://{self.host}", + proxies=self.proxies, + timeout=self.timeout, + adapter=self.http_adapter, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "Chrome/120.0.0.0" + ), + "Accept": "application/json", + }, + ) + return self._http_client + + def fetch_products(self) -> list[IndustrialProduct]: + """Получить список записей из реестра промышленной продукции.""" + logger.info("Fetching industrial products registry") + + try: + files_data = self._fetch_files_list() + file_url = self._get_latest_file_url(files_data) + if not file_url: + logger.warning("No files found for industrial products registry") + return [] + + products = self._download_and_parse(file_url) + logger.info("Fetched %d industrial products", len(products)) + return products + except HTTPClientError: + raise + except Exception as exc: + logger.error("Error fetching industrial products: %s", exc) + raise IndustrialProductsClientError( + f"Failed to fetch industrial products: {exc}" + ) from exc + + def _fetch_files_list(self) -> list[dict]: + params = { + "types[]": self.doc_type, + "fragment": self.query, + } + data = self.http_client.get_json(self.api_path, params=params) + + for item in data.get("data", []): + if self.query in item.get("name", ""): + return item.get("files", []) + + return [] + + def _get_latest_file_url(self, files_data: list[dict]) -> str | None: + if not files_data: + return None + + latest_file = None + latest_date = None + fallback_file = None + + for file_info in files_data: + name = file_info.get("name", "") + if fallback_file is None and name.lower().endswith( + (".xlsx", ".xlsm", ".xls") + ): + fallback_file = file_info + + match = DATE_PATTERN.search(name) + if not match: + continue + + try: + file_date = datetime.strptime(match.group(1), "%Y%m%d") + except ValueError: + continue + + if latest_date is None or file_date > latest_date: + latest_date = file_date + latest_file = file_info + + selected = latest_file or fallback_file or files_data[-1] + url = selected.get("url", "") + if url and not url.startswith("http"): + return f"{self.scheme}://{self.host}{url}" + return url + + def _download_and_parse(self, file_url: str) -> list[IndustrialProduct]: + logger.info("Downloading Excel file: %s", file_url) + + content = self.http_client.download_file(file_url) + workbook = load_workbook(filename=BytesIO(content), data_only=True) + worksheet = workbook.active + + header_row_index, header_map = self._detect_headers(worksheet) + products: list[IndustrialProduct] = [] + + for row in worksheet.iter_rows( + min_row=header_row_index + 1, + values_only=True, + ): + product = self._parse_row(row, header_map) + if product: + products.append(product) + + workbook.close() + return products + + def _detect_headers(self, worksheet) -> tuple[int, dict[str, int]]: + best_row = 1 + best_map: dict[str, int] = {} + + for row_index in range(1, min(worksheet.max_row, 10) + 1): + row = next( + worksheet.iter_rows( + min_row=row_index, + max_row=row_index, + values_only=True, + ) + ) + header_map = self._build_header_map(row) + if len(header_map) > len(best_map): + best_row = row_index + best_map = header_map + + if REQUIRED_HEADERS.issubset(header_map): + return row_index, header_map + + missing = sorted(REQUIRED_HEADERS - set(best_map)) + raise IndustrialProductsClientError( + "Missing expected headers in industrial products file: " + + ", ".join(missing) + ) + + def _build_header_map(self, row: tuple) -> dict[str, int]: + header_map: dict[str, int] = {} + + for index, value in enumerate(row): + normalized = _normalize_header(value) + if not normalized: + continue + + for field_name, aliases in HEADER_ALIASES.items(): + if normalized in {_normalize_header(alias) for alias in aliases}: + header_map[field_name] = index + break + + return header_map + + def _parse_row( + self, + row: tuple, + header_map: dict[str, int], + ) -> IndustrialProduct | None: + def value_for(field_name: str) -> str: + index = header_map.get(field_name) + if index is None or index >= len(row): + return "" + return str(row[index] or "").strip() + + product = IndustrialProduct( + full_organisation_name=value_for("full_organisation_name"), + inn=value_for("inn"), + ogrn=value_for("ogrn"), + registry_number=value_for("registry_number"), + product_name=value_for("product_name"), + product_model=value_for("product_model"), + okpd2_code=value_for("okpd2_code"), + tnved_code=value_for("tnved_code"), + regulatory_document=value_for("regulatory_document"), + ) + + if not any(product.__dict__.values()): + return None + if not product.registry_number or not product.product_name: + logger.warning("Skipping product row without registry number or name: %s", row) + return None + return product + + def close(self) -> None: + if self._http_client is not None: + self._http_client.close() + self._http_client = None + + def __enter__(self) -> "IndustrialProductsClient": + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() diff --git a/src/apps/parsers/clients/minpromtorg/schemas.py b/src/apps/parsers/clients/minpromtorg/schemas.py index 3c85ecd..1ea7df9 100644 --- a/src/apps/parsers/clients/minpromtorg/schemas.py +++ b/src/apps/parsers/clients/minpromtorg/schemas.py @@ -57,3 +57,39 @@ class Manufacturer: address: str """Адрес организации.""" + + +@dataclass(frozen=True) +class IndustrialProduct: + """ + Промышленная продукция из реестра Минпромторга. + + Источник: Минпромторг, реестр промышленной продукции РФ. + """ + + full_organisation_name: str + """Полное наименование организации.""" + + inn: str + """ИНН организации.""" + + ogrn: str + """ОГРН организации.""" + + registry_number: str + """Регистрационный номер записи.""" + + product_name: str + """Наименование продукции.""" + + product_model: str + """Модель или модификация.""" + + okpd2_code: str + """Код по ОКПД2.""" + + tnved_code: str + """Код по ТН ВЭД.""" + + regulatory_document: str + """Нормативный документ.""" diff --git a/src/apps/parsers/migrations/0012_add_industrial_product_record.py b/src/apps/parsers/migrations/0012_add_industrial_product_record.py new file mode 100644 index 0000000..5e88116 --- /dev/null +++ b/src/apps/parsers/migrations/0012_add_industrial_product_record.py @@ -0,0 +1,176 @@ +# Generated by Django 3.2.25 on 2026-03-17 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registers", "0003_add_unique_active_membership_period"), + ("parsers", "0011_add_normalized_date_and_amount_fields"), + ] + + operations = [ + migrations.AlterField( + model_name="parserloadlog", + name="source", + field=models.CharField( + choices=[ + ("industrial", "Промышленное производство"), + ("industrial_products", "Реестр промышленной продукции"), + ("manufactures", "Реестр производителей"), + ("inspections", "Единый реестр проверок"), + ("procurements", "Государственные закупки"), + ("fns_reports", "Бухгалтерская отчетность ФНС"), + ], + db_index=True, + help_text="Источник данных", + max_length=50, + verbose_name="источник", + ), + ), + migrations.CreateModel( + name="IndustrialProductRecord", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Дата и время создания записи", + verbose_name="создано", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="Дата и время последнего обновления", + verbose_name="обновлено", + ), + ), + ( + "load_batch", + models.PositiveIntegerField( + db_index=True, + help_text="Идентификатор пакета загрузки", + verbose_name="ID пакета загрузки", + ), + ), + ( + "full_organisation_name", + models.TextField( + help_text="Полное наименование организации", + verbose_name="полное наименование организации", + ), + ), + ( + "ogrn", + models.CharField( + db_index=True, + help_text="ОГРН организации", + max_length=15, + verbose_name="ОГРН", + ), + ), + ( + "inn", + models.CharField( + db_index=True, + help_text="ИНН организации", + max_length=12, + verbose_name="ИНН", + ), + ), + ( + "registry_number", + models.CharField( + db_index=True, + help_text="Регистрационный номер записи", + max_length=100, + verbose_name="регистрационный номер", + ), + ), + ( + "product_name", + models.TextField( + help_text="Наименование продукции", + verbose_name="наименование продукции", + ), + ), + ( + "product_model", + models.CharField( + blank=True, + help_text="Модель или модификация продукции", + max_length=255, + verbose_name="модель или модификация", + ), + ), + ( + "okpd2_code", + models.CharField( + blank=True, + help_text="Код по ОКПД2", + max_length=20, + verbose_name="код ОКПД2", + ), + ), + ( + "tnved_code", + models.CharField( + blank=True, + help_text="Код по ТН ВЭД", + max_length=20, + verbose_name="код ТН ВЭД", + ), + ), + ( + "regulatory_document", + models.CharField( + blank=True, + help_text="Наименование нормативного документа", + max_length=500, + verbose_name="нормативный документ", + ), + ), + ( + "registry_organization", + models.ForeignKey( + blank=True, + help_text="Связь с организацией из приложения реестров", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="industrial_product_records", + to="registers.organization", + verbose_name="организация из реестров", + ), + ), + ], + options={ + "verbose_name": "промышленная продукция", + "verbose_name_plural": "промышленная продукция", + "db_table": "parsers_industrial_product", + "ordering": ["-created_at"], + "indexes": [ + models.Index( + fields=["load_batch", "inn"], + name="parsers_indprod_load_ba_98b870_idx", + ), + models.Index( + fields=["inn", "registry_number"], + name="parsers_indprod_inn_eb2ad3_idx", + ), + ], + }, + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index fd1991c..81181f7 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -18,6 +18,7 @@ class ParserLoadLog(TimestampMixin, models.Model): class Source(models.TextChoices): INDUSTRIAL = "industrial", _("Промышленное производство") + INDUSTRIAL_PRODUCTS = "industrial_products", _("Реестр промышленной продукции") MANUFACTURES = "manufactures", _("Реестр производителей") INSPECTIONS = "inspections", _("Единый реестр проверок") PROCUREMENTS = "procurements", _("Государственные закупки") @@ -228,6 +229,92 @@ class ManufacturerRecord(TimestampMixin, models.Model): return f"{self.inn} - {self.full_legal_name[:50]}" +class IndustrialProductRecord(TimestampMixin, models.Model): + """ + Промышленная продукция из реестра Минпромторга. + + Данные загружаются из Минпромторга. + """ + + load_batch = models.PositiveIntegerField( + _("ID пакета загрузки"), + db_index=True, + help_text=_("Идентификатор пакета загрузки"), + ) + full_organisation_name = models.TextField( + _("полное наименование организации"), + help_text=_("Полное наименование организации"), + ) + ogrn = models.CharField( + _("ОГРН"), + max_length=15, + db_index=True, + help_text=_("ОГРН организации"), + ) + inn = models.CharField( + _("ИНН"), + max_length=12, + db_index=True, + help_text=_("ИНН организации"), + ) + registry_number = models.CharField( + _("регистрационный номер"), + max_length=100, + db_index=True, + help_text=_("Регистрационный номер записи"), + ) + product_name = models.TextField( + _("наименование продукции"), + help_text=_("Наименование продукции"), + ) + product_model = models.CharField( + _("модель или модификация"), + max_length=255, + blank=True, + help_text=_("Модель или модификация продукции"), + ) + okpd2_code = models.CharField( + _("код ОКПД2"), + max_length=20, + blank=True, + help_text=_("Код по ОКПД2"), + ) + tnved_code = models.CharField( + _("код ТН ВЭД"), + max_length=20, + blank=True, + help_text=_("Код по ТН ВЭД"), + ) + regulatory_document = models.CharField( + _("нормативный документ"), + max_length=500, + blank=True, + help_text=_("Наименование нормативного документа"), + ) + registry_organization = models.ForeignKey( + "registers.Organization", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="industrial_product_records", + verbose_name=_("организация из реестров"), + help_text=_("Связь с организацией из приложения реестров"), + ) + + class Meta: + db_table = "parsers_industrial_product" + verbose_name = _("промышленная продукция") + verbose_name_plural = _("промышленная продукция") + ordering = ["-created_at"] + indexes = [ + models.Index(fields=["load_batch", "inn"]), + models.Index(fields=["inn", "registry_number"]), + ] + + def __str__(self) -> str: + return f"{self.registry_number} - {self.product_name[:50]}" + + class Proxy(TimestampMixin, models.Model): """ Прокси-сервер для парсеров. diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index 70abb19..5bce10c 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -8,6 +8,7 @@ from apps.parsers.models import ( FinancialReport, FinancialReportLine, IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -77,6 +78,34 @@ class ManufacturerSerializer(serializers.ModelSerializer): read_only_fields = fields +class IndustrialProductSerializer(serializers.ModelSerializer): + """ + Промышленная продукция из реестра Минпромторга. + + Данные загружаются из Минпромторга. + """ + + class Meta: + model = IndustrialProductRecord + fields = [ + "id", + "load_batch", + "full_organisation_name", + "ogrn", + "inn", + "registry_number", + "product_name", + "product_model", + "okpd2_code", + "tnved_code", + "regulatory_document", + "registry_organization", + "created_at", + "updated_at", + ] + read_only_fields = fields + + # ============================================================================= # Единый реестр проверок (proverki.gov.ru) # ============================================================================= @@ -304,3 +333,124 @@ class ProxySerializer(serializers.ModelSerializer): "updated_at", ] read_only_fields = fields + + +class SourceCardRefreshParamSerializer(serializers.Serializer): + """Описание параметра ручного обновления карточки источника.""" + + name = serializers.CharField(read_only=True) + label = serializers.CharField(read_only=True) + description = serializers.CharField(read_only=True) + required = serializers.BooleanField(read_only=True) + type = serializers.CharField(read_only=True) + default = serializers.JSONField(read_only=True) + + +class SourceCardLoadSerializer(serializers.Serializer): + """Сводка по последней загрузке источника.""" + + batch_id = serializers.IntegerField(read_only=True) + source = serializers.CharField(read_only=True) + source_display = serializers.CharField(read_only=True) + records_count = serializers.IntegerField(read_only=True) + status = serializers.CharField(read_only=True) + error_message = serializers.CharField(read_only=True) + created_at = serializers.DateTimeField(read_only=True) + updated_at = serializers.DateTimeField(read_only=True) + + +class SourceCardTaskSerializer(serializers.Serializer): + """Краткая информация по фоновой задаче карточки.""" + + task_id = serializers.CharField(read_only=True) + task_name = serializers.CharField(read_only=True) + status = serializers.CharField(read_only=True) + progress = serializers.IntegerField(read_only=True) + progress_message = serializers.CharField(read_only=True) + started_at = serializers.DateTimeField(read_only=True) + created_at = serializers.DateTimeField(read_only=True) + meta = serializers.JSONField(read_only=True) + + +class SourceCardItemSerializer(serializers.Serializer): + """Подисточник внутри агрегированной карточки.""" + + code = serializers.CharField(read_only=True) + title = serializers.CharField(read_only=True) + description = serializers.CharField(read_only=True) + parser_source = serializers.CharField(read_only=True, allow_null=True) + parser_source_display = serializers.CharField(read_only=True, allow_null=True) + records_count = serializers.IntegerField(read_only=True) + organizations_count = serializers.IntegerField(read_only=True) + last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True) + latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True) + latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True) + + +class SourceCardSerializer(serializers.Serializer): + """Serializer for frontend source cards list.""" + + slug = serializers.CharField(read_only=True) + title = serializers.CharField(read_only=True) + description = serializers.CharField(read_only=True) + order = serializers.IntegerField(read_only=True) + is_available = serializers.BooleanField(read_only=True) + status = serializers.CharField(read_only=True) + status_label = serializers.CharField(read_only=True) + progress = serializers.IntegerField(read_only=True) + records_count = serializers.IntegerField(read_only=True) + organizations_count = serializers.IntegerField(read_only=True) + last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True) + next_update_at = serializers.DateTimeField(read_only=True, allow_null=True) + error_message = serializers.CharField(read_only=True) + task_names = serializers.ListField(child=serializers.CharField(), read_only=True) + refresh_requires_params = serializers.BooleanField(read_only=True) + refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True) + + +class SourceTaskStatusSerializer(serializers.Serializer): + """Табличная строка статуса источника для экрана задач парсинга.""" + + row_number = serializers.IntegerField(read_only=True) + slug = serializers.CharField(read_only=True) + source = serializers.CharField(read_only=True) + status = serializers.CharField(read_only=True) + status_label = serializers.CharField(read_only=True) + actualized_at = serializers.DateTimeField(read_only=True, allow_null=True) + next_update_at = serializers.DateTimeField(read_only=True, allow_null=True) + records_count = serializers.IntegerField(read_only=True) + organizations_count = serializers.IntegerField(read_only=True) + progress = serializers.IntegerField(read_only=True) + error_message = serializers.CharField(read_only=True) + active_tasks = SourceCardTaskSerializer(many=True, read_only=True) + + +class SourceCardDetailSerializer(SourceCardSerializer): + """Detailed serializer for a single frontend source card.""" + + active_tasks = SourceCardTaskSerializer(many=True, read_only=True) + source_items = SourceCardItemSerializer(many=True, read_only=True) + latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True) + latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True) + + +class SourceCardRefreshRequestSerializer(serializers.Serializer): + """Request body for manual card refresh.""" + + params = serializers.DictField(required=False) + + +class SourceCardRefreshTaskSerializer(serializers.Serializer): + """Queued task info after refresh start.""" + + task_id = serializers.CharField(read_only=True) + task_name = serializers.CharField(read_only=True) + + +class SourceCardRefreshResponseSerializer(serializers.Serializer): + """Response serializer for manual card refresh.""" + + source_card = serializers.CharField(read_only=True) + status = serializers.CharField(read_only=True) + requested_at = serializers.DateTimeField(read_only=True) + tasks = SourceCardRefreshTaskSerializer(many=True, read_only=True) diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index f584988..6f689b5 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -13,13 +13,18 @@ from datetime import date, datetime from decimal import Decimal, InvalidOperation from apps.core.services import BaseService, BulkOperationsMixin -from apps.parsers.clients.minpromtorg.schemas import IndustrialCertificate, Manufacturer +from apps.parsers.clients.minpromtorg.schemas import ( + IndustrialCertificate, + IndustrialProduct, + Manufacturer, +) from apps.parsers.clients.proverki.schemas import Inspection from apps.parsers.clients.zakupki.schemas import Procurement from apps.parsers.models import ( FinancialReport, FinancialReportLine, IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -467,31 +472,40 @@ class ManufacturerService(BulkOperationsMixin, BaseService[ManufacturerRecord]): [(manufacturer.inn, manufacturer.ogrn) for manufacturer in manufacturers] ) - instances = [ - cls.model( - load_batch=batch_id, - full_legal_name=m.full_legal_name, - inn=m.inn, - ogrn=m.ogrn, - address=m.address, - registry_organization_id=RegistryOrganizationResolver.resolve_organization_id( + items = [ + { + "load_batch": batch_id, + "full_legal_name": m.full_legal_name, + "inn": m.inn, + "ogrn": m.ogrn, + "address": m.address, + "registry_organization_id": RegistryOrganizationResolver.resolve_organization_id( lookup=registry_lookup, inn=m.inn, ogrn=m.ogrn, ), - ) + } for m in manufacturers ] - before_count = cls.model.objects.filter(load_batch=batch_id).count() - cls.bulk_create_chunked( - instances, - chunk_size=chunk_size, - ignore_conflicts=True, # Skip duplicates by INN + created_count, updated_count = cls.bulk_update_or_create( + items, + unique_fields=["inn"], + update_fields=[ + "load_batch", + "full_legal_name", + "ogrn", + "address", + "registry_organization_id", + ], + ) + saved_count = created_count + updated_count + logger.info( + "Saved %d manufacturers (created=%d, updated=%d)", + saved_count, + created_count, + updated_count, ) - after_count = cls.model.objects.filter(load_batch=batch_id).count() - saved_count = max(0, after_count - before_count) - logger.info("Saved %d manufacturers", saved_count) return saved_count @@ -515,6 +529,105 @@ class ManufacturerService(BulkOperationsMixin, BaseService[ManufacturerRecord]): return cls.filter(ogrn=ogrn) +class IndustrialProductService( + BulkOperationsMixin, BaseService[IndustrialProductRecord] +): + """ + Сервис для управления реестром промышленной продукции. + + Отвечает за: + - Массовое сохранение продукции из парсера + - Поиск продукции по ИНН/ОГРН/регистрационному номеру + """ + + model = IndustrialProductRecord + + @classmethod + @transaction.atomic + def save_products( + cls, + products: list[IndustrialProduct], + batch_id: int, + ) -> int: + """Сохранить список промышленной продукции из парсера.""" + if not products: + logger.warning("No industrial products to save") + return 0 + + logger.info( + "Saving %d industrial products (batch_id=%d)", len(products), batch_id + ) + + registry_lookup = RegistryOrganizationResolver.build_lookup( + [(product.inn, product.ogrn) for product in products] + ) + + items = [ + { + "load_batch": batch_id, + "full_organisation_name": product.full_organisation_name, + "ogrn": product.ogrn, + "inn": product.inn, + "registry_number": product.registry_number, + "product_name": product.product_name, + "product_model": product.product_model, + "okpd2_code": product.okpd2_code, + "tnved_code": product.tnved_code, + "regulatory_document": product.regulatory_document, + "registry_organization_id": RegistryOrganizationResolver.resolve_organization_id( + lookup=registry_lookup, + inn=product.inn, + ogrn=product.ogrn, + ), + } + for product in products + if product.registry_number + ] + + created_count, updated_count = cls.bulk_update_or_create( + items, + unique_fields=["registry_number"], + update_fields=[ + "load_batch", + "full_organisation_name", + "ogrn", + "inn", + "product_name", + "product_model", + "okpd2_code", + "tnved_code", + "regulatory_document", + "registry_organization_id", + ], + ) + saved_count = created_count + updated_count + logger.info( + "Saved %d industrial products (created=%d, updated=%d)", + saved_count, + created_count, + updated_count, + ) + return saved_count + + @classmethod + def find_by_inn(cls, inn: str, batch_id: int | None = None): + """Найти продукцию по ИНН.""" + qs = cls.filter(inn=inn) + if batch_id: + qs = qs.filter(load_batch=batch_id) + return qs + + @classmethod + def find_by_ogrn(cls, ogrn: str): + """Найти продукцию по ОГРН.""" + return cls.filter(ogrn=ogrn) + + @classmethod + def find_by_registry_number(cls, registry_number: str): + """Найти продукцию по регистрационному номеру.""" + return cls.filter(registry_number=registry_number) + + class ProxyService(BaseService[Proxy]): """ Сервис для управления прокси-серверами. @@ -683,44 +796,66 @@ class InspectionService(BulkOperationsMixin, BaseService[InspectionRecord]): [(inspection.inn, inspection.ogrn) for inspection in inspections] ) - instances = [ - cls.model( - load_batch=batch_id, - registration_number=insp.registration_number, - inn=insp.inn, - ogrn=insp.ogrn, - organisation_name=insp.organisation_name, - control_authority=insp.control_authority, - inspection_type=insp.inspection_type, - inspection_form=insp.inspection_form, - start_date=insp.start_date, - start_date_normalized=normalize_to_date(insp.start_date), - end_date=insp.end_date, - end_date_normalized=normalize_to_date(insp.end_date), - status=insp.status, - legal_basis=insp.legal_basis, - result=insp.result, - is_federal_law_248=is_federal_law_248, - data_year=data_year, - data_month=data_month, - registry_organization_id=RegistryOrganizationResolver.resolve_organization_id( + items = [ + { + "load_batch": batch_id, + "registration_number": insp.registration_number, + "inn": insp.inn, + "ogrn": insp.ogrn, + "organisation_name": insp.organisation_name, + "control_authority": insp.control_authority, + "inspection_type": insp.inspection_type, + "inspection_form": insp.inspection_form, + "start_date": insp.start_date, + "start_date_normalized": normalize_to_date(insp.start_date), + "end_date": insp.end_date, + "end_date_normalized": normalize_to_date(insp.end_date), + "status": insp.status, + "legal_basis": insp.legal_basis, + "result": insp.result, + "is_federal_law_248": is_federal_law_248, + "data_year": data_year, + "data_month": data_month, + "registry_organization_id": RegistryOrganizationResolver.resolve_organization_id( lookup=registry_lookup, inn=insp.inn, ogrn=insp.ogrn, ), - ) + } for insp in inspections ] - before_count = cls.model.objects.filter(load_batch=batch_id).count() - cls.bulk_create_chunked( - instances, - chunk_size=chunk_size, - ignore_conflicts=True, # Skip duplicates by registration_number + created_count, updated_count = cls.bulk_update_or_create( + items, + unique_fields=["registration_number"], + update_fields=[ + "load_batch", + "inn", + "ogrn", + "organisation_name", + "control_authority", + "inspection_type", + "inspection_form", + "start_date", + "start_date_normalized", + "end_date", + "end_date_normalized", + "status", + "legal_basis", + "result", + "is_federal_law_248", + "data_year", + "data_month", + "registry_organization_id", + ], + ) + saved_count = created_count + updated_count + logger.info( + "Saved %d inspections (created=%d, updated=%d)", + saved_count, + created_count, + updated_count, ) - after_count = cls.model.objects.filter(load_batch=batch_id).count() - saved_count = max(0, after_count - before_count) - logger.info("Saved %d inspections", saved_count) return saved_count @@ -867,48 +1002,74 @@ class ProcurementService(BulkOperationsMixin, BaseService[ProcurementRecord]): ] ) - instances = [ - cls.model( - load_batch=batch_id, - purchase_number=proc.purchase_number, - purchase_name=proc.purchase_name, - customer_inn=proc.customer_inn, - customer_kpp=proc.customer_kpp, - customer_ogrn=proc.customer_ogrn, - customer_name=proc.customer_name, - max_price=proc.max_price, - max_price_amount=normalize_to_decimal(proc.max_price), - currency_code=proc.currency_code, - placement_method=proc.placement_method, - publish_date=proc.publish_date, - publish_date_normalized=normalize_to_date(proc.publish_date), - end_date=proc.end_date, - end_date_normalized=normalize_to_date(proc.end_date), - status=proc.status, - law_type=proc.law_type, - purchase_object_info=proc.purchase_object_info, - href=proc.href, - region_code=region_code or "", - data_year=data_year, - data_month=data_month, - registry_organization_id=RegistryOrganizationResolver.resolve_organization_id( + items = [ + { + "load_batch": batch_id, + "purchase_number": proc.purchase_number, + "purchase_name": proc.purchase_name, + "customer_inn": proc.customer_inn, + "customer_kpp": proc.customer_kpp, + "customer_ogrn": proc.customer_ogrn, + "customer_name": proc.customer_name, + "max_price": proc.max_price, + "max_price_amount": normalize_to_decimal(proc.max_price), + "currency_code": proc.currency_code, + "placement_method": proc.placement_method, + "publish_date": proc.publish_date, + "publish_date_normalized": normalize_to_date(proc.publish_date), + "end_date": proc.end_date, + "end_date_normalized": normalize_to_date(proc.end_date), + "status": proc.status, + "law_type": proc.law_type, + "purchase_object_info": proc.purchase_object_info, + "href": proc.href, + "region_code": region_code or "", + "data_year": data_year, + "data_month": data_month, + "registry_organization_id": RegistryOrganizationResolver.resolve_organization_id( lookup=registry_lookup, inn=proc.customer_inn, ogrn=proc.customer_ogrn, ), - ) + } for proc in procurements ] - before_count = cls.model.objects.filter(load_batch=batch_id).count() - cls.bulk_create_chunked( - instances, - chunk_size=chunk_size, - ignore_conflicts=True, # Skip duplicates by purchase_number + created_count, updated_count = cls.bulk_update_or_create( + items, + unique_fields=["purchase_number"], + update_fields=[ + "load_batch", + "purchase_name", + "customer_inn", + "customer_kpp", + "customer_ogrn", + "customer_name", + "max_price", + "max_price_amount", + "currency_code", + "placement_method", + "publish_date", + "publish_date_normalized", + "end_date", + "end_date_normalized", + "status", + "law_type", + "purchase_object_info", + "href", + "region_code", + "data_year", + "data_month", + "registry_organization_id", + ], + ) + saved_count = created_count + updated_count + logger.info( + "Saved %d procurements (created=%d, updated=%d)", + saved_count, + created_count, + updated_count, ) - after_count = cls.model.objects.filter(load_batch=batch_id).count() - saved_count = max(0, after_count - before_count) - logger.info("Saved %d procurements", saved_count) return saved_count diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py new file mode 100644 index 0000000..604756e --- /dev/null +++ b/src/apps/parsers/source_cards.py @@ -0,0 +1,772 @@ +"""Frontend-oriented source cards service.""" + +from __future__ import annotations + +import uuid +from contextlib import suppress +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +from apps.core.models import JobStatus +from apps.core.services import BackgroundJobService +from apps.parsers.models import ( + FinancialReport, + FinancialReportLine, + IndustrialCertificateRecord, + IndustrialProductRecord, + InspectionRecord, + ManufacturerRecord, + ParserLoadLog, + ProcurementRecord, +) +from django.db.models import Max +from django.http import Http404 +from django.utils import timezone +from rest_framework.exceptions import ValidationError + + +SUCCESSFUL_LOAD_STATUSES = {"success", "skipped"} +ACTIVE_JOB_STATUSES = [JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY] + + +@dataclass(frozen=True) +class RefreshParamDefinition: + """Schema for refresh params expected by a source card.""" + + name: str + label: str + description: str + required: bool = False + param_type: str = "string" + default: Any = None + + +@dataclass(frozen=True) +class SourceItemDefinition: + """Single backend source used inside a frontend card.""" + + code: str + title: str + description: str + parser_source: str | None = None + + +@dataclass(frozen=True) +class SourceCardDefinition: + """Frontend card definition.""" + + slug: str + title: str + description: str + order: int + task_names: tuple[str, ...] + source_items: tuple[SourceItemDefinition, ...] + refresh_params: tuple[RefreshParamDefinition, ...] = () + refresh_interval: timedelta | None = None + is_available: bool = True + + +SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( + SourceCardDefinition( + slug="financial-indicators", + title="Финансово-экономические показатели", + description="Бухгалтерская отчетность ФНС и рассчитанные показатели по ней.", + order=10, + task_names=( + "apps.parsers.tasks.scan_fns_directory", + "apps.parsers.tasks.process_fns_file", + "apps.parsers.tasks.process_fns_files_batch", + ), + source_items=( + SourceItemDefinition( + code="fns_reports", + title="Бухгалтерская отчетность ФНС", + description=( + "Excel-файлы бухгалтерской отчетности, которые загружаются " + "в систему и раскладываются на строки показателей." + ), + parser_source=ParserLoadLog.Source.FNS_REPORTS, + ), + ), + refresh_interval=timedelta(minutes=5), + ), + SourceCardDefinition( + slug="public-procurements", + title="Государственные закупки по 44-ФЗ и 223-ФЗ", + description="Данные ЕИС закупок по тендерам и заказчикам.", + order=20, + task_names=( + "apps.parsers.tasks.parse_procurements", + "apps.parsers.tasks.sync_procurements", + ), + source_items=( + SourceItemDefinition( + code="procurements", + title="Единая информационная система закупок", + description=( + "Закупки и связанные данные из ЕИС по 44-ФЗ и 223-ФЗ." + ), + parser_source=ParserLoadLog.Source.PROCUREMENTS, + ), + ), + refresh_params=( + RefreshParamDefinition( + name="region_code", + label="Код региона", + description="Код региона ЕИС, например 77 для Москвы.", + required=True, + ), + RefreshParamDefinition( + name="law_type", + label="Тип закона", + description="Тип закупок: 44 или 223. По умолчанию 44.", + default="44", + ), + RefreshParamDefinition( + name="current_year", + label="Текущий год", + description="Опциональная верхняя граница синхронизации по году.", + param_type="integer", + ), + RefreshParamDefinition( + name="current_month", + label="Текущий месяц", + description="Опциональная верхняя граница синхронизации по месяцу.", + param_type="integer", + ), + ), + ), + SourceCardDefinition( + slug="manufacturers-and-products", + title="Производители и продукция России", + description=( + "Данные Минпромторга о сертификатах промышленной продукции " + "и реестрах производителей и продукции." + ), + order=30, + task_names=( + "apps.parsers.tasks.parse_industrial_production", + "apps.parsers.tasks.parse_industrial_products", + "apps.parsers.tasks.parse_manufactures", + ), + source_items=( + SourceItemDefinition( + code="industrial", + title="Сертификаты промышленного производства", + description=( + "Заключения и сертификаты промышленного производства " + "из источников Минпромторга." + ), + parser_source=ParserLoadLog.Source.INDUSTRIAL, + ), + SourceItemDefinition( + code="industrial_products", + title="Реестр промышленной продукции", + description=( + "Реестр промышленной продукции, произведенной на территории РФ." + ), + parser_source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, + ), + SourceItemDefinition( + code="manufactures", + title="Реестр производителей", + description="Реестр производителей из Минпромторга.", + parser_source=ParserLoadLog.Source.MANUFACTURES, + ), + ), + refresh_interval=timedelta(days=1), + ), + SourceCardDefinition( + slug="planned-inspections", + title="Плановые проверки Генпрокуратуры России", + description="Единый реестр проверок с плановыми и внеплановыми проверками.", + order=40, + task_names=( + "apps.parsers.tasks.parse_inspections", + "apps.parsers.tasks.sync_inspections", + ), + source_items=( + SourceItemDefinition( + code="inspections", + title="Единый реестр проверок", + description="Данные proverki.gov.ru по ФЗ-294 и ФЗ-248.", + parser_source=ParserLoadLog.Source.INSPECTIONS, + ), + ), + ), +) + +SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS} + + +class SourceCardService: + """Builds aggregated source cards for frontend pages.""" + + @classmethod + def list_cards(cls) -> list[dict[str, Any]]: + return [cls.get_card(definition.slug) for definition in SOURCE_CARD_DEFINITIONS] + + @classmethod + def list_task_statuses(cls) -> list[dict[str, Any]]: + """Возвращает табличный список статусов источников для экрана парсинга.""" + cards = cls.list_cards() + sorted_cards = sorted( + cards, + key=lambda item: ( + item["last_updated_at"] is None, + -(item["last_updated_at"].timestamp()) if item["last_updated_at"] else 0, + item["title"], + ), + ) + + rows = [] + for index, card in enumerate(sorted_cards, start=1): + rows.append( + { + "row_number": index, + "slug": card["slug"], + "source": card["title"], + "status": card["status"], + "status_label": card["status_label"], + "actualized_at": card["last_updated_at"], + "next_update_at": card["next_update_at"], + "records_count": card["records_count"], + "organizations_count": card["organizations_count"], + "progress": card["progress"], + "error_message": card["error_message"], + "active_tasks": card.get("active_tasks", []), + } + ) + return rows + + @classmethod + def get_card(cls, slug: str) -> dict[str, Any]: + definition = cls.get_definition(slug) + source_items = [cls._build_source_item(item) for item in definition.source_items] + records_count = sum(item["records_count"] for item in source_items) + organizations_count = cls._get_card_organizations_count(definition, source_items) + + latest_success_load = cls._get_latest_load( + definition, + statuses=SUCCESSFUL_LOAD_STATUSES, + ) + latest_load = cls._get_latest_load(definition) + last_updated_at = ( + latest_success_load.updated_at + if latest_success_load + else cls._get_latest_data_timestamp(source_items) + ) + active_tasks = cls._get_active_tasks(definition) + progress = cls._get_progress(active_tasks) + status = cls._get_status( + definition=definition, + active_tasks=active_tasks, + latest_load=latest_load, + last_updated_at=last_updated_at, + ) + + return { + "slug": definition.slug, + "title": definition.title, + "description": definition.description, + "order": definition.order, + "is_available": definition.is_available, + "status": status, + "status_label": cls._get_status_label(status), + "progress": progress, + "records_count": records_count, + "organizations_count": organizations_count, + "last_updated_at": last_updated_at, + "next_update_at": cls._get_next_update_at(definition, last_updated_at), + "error_message": latest_load.error_message if latest_load else "", + "task_names": list(definition.task_names), + "refresh_requires_params": any( + param.required for param in definition.refresh_params + ), + "refresh_params": [ + { + "name": param.name, + "label": param.label, + "description": param.description, + "required": param.required, + "type": param.param_type, + "default": param.default, + } + for param in definition.refresh_params + ], + "active_tasks": active_tasks, + "source_items": source_items, + "latest_load": cls._serialize_load_log(latest_load), + "latest_success_load": cls._serialize_load_log(latest_success_load), + } + + @classmethod + def refresh_card( + cls, + *, + slug: str, + requested_by_id: int | None, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + definition = cls.get_definition(slug) + params = cls._validate_refresh_params(definition, params or {}) + tasks = cls._launch_refresh(definition, requested_by_id=requested_by_id, params=params) + + return { + "source_card": definition.slug, + "status": "started", + "tasks": tasks, + "requested_at": timezone.now(), + } + + @classmethod + def get_definition(cls, slug: str) -> SourceCardDefinition: + definition = SOURCE_CARD_BY_SLUG.get(slug) + if definition is None: + raise Http404("Карточка источника не найдена") + return definition + + @classmethod + def _validate_refresh_params( + cls, + definition: SourceCardDefinition, + params: dict[str, Any], + ) -> dict[str, Any]: + allowed_params = {item.name: item for item in definition.refresh_params} + unknown_params = sorted(set(params) - set(allowed_params)) + if unknown_params: + raise ValidationError( + { + "params": ( + "Неизвестные параметры обновления: " + + ", ".join(unknown_params) + ) + } + ) + + errors: dict[str, list[str]] = {} + validated: dict[str, Any] = {} + for name, item in allowed_params.items(): + value = params.get(name, item.default) + if item.required and value in (None, ""): + errors[name] = ["Это поле обязательно."] + continue + + if value in (None, ""): + continue + + if item.param_type == "integer": + try: + value = int(value) + except (TypeError, ValueError) as exc: + raise ValidationError( + {"params": {name: ["Значение должно быть целым числом."]}} + ) from exc + + validated[name] = value + + if errors: + raise ValidationError({"params": errors}) + + return validated + + @classmethod + def _launch_refresh( + cls, + definition: SourceCardDefinition, + *, + requested_by_id: int | None, + params: dict[str, Any], + ) -> list[dict[str, str]]: + if definition.slug == "financial-indicators": + from apps.parsers.tasks import scan_fns_directory + + task_info = cls._enqueue_task( + task=scan_fns_directory, + task_name="apps.parsers.tasks.scan_fns_directory", + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": ParserLoadLog.Source.FNS_REPORTS, + }, + kwargs={"requested_by_id": requested_by_id}, + ) + return [task_info] + + if definition.slug == "manufacturers-and-products": + from apps.parsers.tasks import ( + parse_industrial_production, + parse_industrial_products, + parse_manufactures, + ) + + return [ + cls._enqueue_task( + task=parse_industrial_production, + task_name="apps.parsers.tasks.parse_industrial_production", + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": ParserLoadLog.Source.INDUSTRIAL, + }, + kwargs={"requested_by_id": requested_by_id}, + ), + cls._enqueue_task( + task=parse_industrial_products, + task_name="apps.parsers.tasks.parse_industrial_products", + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, + }, + kwargs={"requested_by_id": requested_by_id}, + ), + cls._enqueue_task( + task=parse_manufactures, + task_name="apps.parsers.tasks.parse_manufactures", + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": ParserLoadLog.Source.MANUFACTURES, + }, + kwargs={"requested_by_id": requested_by_id}, + ), + ] + + if definition.slug == "planned-inspections": + from apps.parsers.tasks import sync_inspections + + task_info = cls._enqueue_task( + task=sync_inspections, + task_name="apps.parsers.tasks.sync_inspections", + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": ParserLoadLog.Source.INSPECTIONS, + }, + kwargs={ + "requested_by_id": requested_by_id, + **{ + key: value + for key, value in params.items() + if key in {"current_year", "current_month", "use_playwright"} + }, + }, + ) + return [task_info] + + if definition.slug == "public-procurements": + from apps.parsers.tasks import sync_procurements + + task_info = cls._enqueue_task( + task=sync_procurements, + task_name="apps.parsers.tasks.sync_procurements", + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": ParserLoadLog.Source.PROCUREMENTS, + "region_code": params["region_code"], + "law_type": params.get("law_type", "44"), + }, + kwargs={ + "requested_by_id": requested_by_id, + "region_code": params["region_code"], + "law_type": params.get("law_type", "44"), + **{ + key: value + for key, value in params.items() + if key in {"current_year", "current_month"} + }, + }, + ) + return [task_info] + + raise ValidationError({"detail": "Обновление для карточки не поддерживается."}) + + @classmethod + def _enqueue_task( + cls, + *, + task, + task_name: str, + requested_by_id: int | None, + meta: dict[str, Any], + kwargs: dict[str, Any], + ) -> dict[str, str]: + task_id = str(uuid.uuid4()) + BackgroundJobService.create_job( + task_id=task_id, + task_name=task_name, + user_id=requested_by_id, + meta=meta, + ) + try: + async_result = task.apply_async(kwargs=kwargs, task_id=task_id) + except Exception: + with suppress(Exception): + BackgroundJobService.get_queryset().filter(task_id=task_id).delete() + raise + + return {"task_id": async_result.id, "task_name": task_name} + + @classmethod + def _build_source_item(cls, item: SourceItemDefinition) -> dict[str, Any]: + records_count = cls._get_source_records_count(item.code) + organizations_count = cls._get_source_organizations_count(item.code) + last_updated_at = cls._get_source_data_timestamp(item.code) + latest_load = cls._get_latest_load_by_source(item.parser_source) + latest_success_load = cls._get_latest_load_by_source( + item.parser_source, + statuses=SUCCESSFUL_LOAD_STATUSES, + ) + + return { + "code": item.code, + "title": item.title, + "description": item.description, + "parser_source": item.parser_source, + "parser_source_display": ( + latest_load.get_source_display() if latest_load else None + ), + "records_count": records_count, + "organizations_count": organizations_count, + "last_updated_at": ( + latest_success_load.updated_at if latest_success_load else last_updated_at + ), + "latest_load": cls._serialize_load_log(latest_load), + "latest_success_load": cls._serialize_load_log(latest_success_load), + } + + @classmethod + def _get_source_records_count(cls, item_code: str) -> int: + if item_code == "fns_reports": + return FinancialReportLine.objects.count() + if item_code == "industrial": + return IndustrialCertificateRecord.objects.count() + if item_code == "manufactures": + return ManufacturerRecord.objects.count() + if item_code == "industrial_products": + return IndustrialProductRecord.objects.count() + if item_code == "inspections": + return InspectionRecord.objects.count() + if item_code == "procurements": + return ProcurementRecord.objects.count() + return 0 + + @classmethod + def _get_source_organizations_count(cls, item_code: str) -> int: + if item_code == "fns_reports": + return ( + FinancialReport.objects.exclude(ogrn="") + .values("ogrn") + .distinct() + .count() + ) + if item_code == "industrial": + return ( + IndustrialCertificateRecord.objects.exclude(inn="") + .values("inn") + .distinct() + .count() + ) + if item_code == "manufactures": + return ManufacturerRecord.objects.exclude(inn="").values("inn").distinct().count() + if item_code == "industrial_products": + return ( + IndustrialProductRecord.objects.exclude(inn="") + .values("inn") + .distinct() + .count() + ) + if item_code == "inspections": + return InspectionRecord.objects.exclude(inn="").values("inn").distinct().count() + if item_code == "procurements": + return ( + ProcurementRecord.objects.exclude(customer_inn="") + .values("customer_inn") + .distinct() + .count() + ) + return 0 + + @classmethod + def _get_source_data_timestamp(cls, item_code: str): + if item_code == "fns_reports": + return FinancialReport.objects.aggregate(last_updated=Max("updated_at"))[ + "last_updated" + ] + if item_code == "industrial": + return IndustrialCertificateRecord.objects.aggregate( + last_updated=Max("updated_at") + )["last_updated"] + if item_code == "manufactures": + return ManufacturerRecord.objects.aggregate(last_updated=Max("updated_at"))[ + "last_updated" + ] + if item_code == "industrial_products": + return IndustrialProductRecord.objects.aggregate( + last_updated=Max("updated_at") + )["last_updated"] + if item_code == "inspections": + return InspectionRecord.objects.aggregate(last_updated=Max("updated_at"))[ + "last_updated" + ] + if item_code == "procurements": + return ProcurementRecord.objects.aggregate(last_updated=Max("updated_at"))[ + "last_updated" + ] + return None + + @classmethod + def _get_card_organizations_count( + cls, + definition: SourceCardDefinition, + source_items: list[dict[str, Any]], + ) -> int: + if definition.slug != "manufacturers-and-products": + return sum(item["organizations_count"] for item in source_items) + + industrial_inns = ( + IndustrialCertificateRecord.objects.exclude(inn="") + .order_by() + .values_list("inn", flat=True) + .distinct() + ) + manufacturer_inns = ( + ManufacturerRecord.objects.exclude(inn="") + .order_by() + .values_list("inn", flat=True) + .distinct() + ) + product_inns = ( + IndustrialProductRecord.objects.exclude(inn="") + .order_by() + .values_list("inn", flat=True) + .distinct() + ) + return industrial_inns.union(manufacturer_inns, product_inns).count() + + @classmethod + def _get_latest_load( + cls, + definition: SourceCardDefinition, + *, + statuses: set[str] | None = None, + ) -> ParserLoadLog | None: + parser_sources = [ + item.parser_source for item in definition.source_items if item.parser_source + ] + queryset = ParserLoadLog.objects.filter(source__in=parser_sources) + if statuses: + queryset = queryset.filter(status__in=statuses) + return queryset.order_by("-updated_at", "-created_at").first() + + @classmethod + def _get_latest_load_by_source( + cls, + parser_source: str | None, + *, + statuses: set[str] | None = None, + ) -> ParserLoadLog | None: + if not parser_source: + return None + queryset = ParserLoadLog.objects.filter(source=parser_source) + if statuses: + queryset = queryset.filter(status__in=statuses) + return queryset.order_by("-updated_at", "-created_at").first() + + @classmethod + def _get_active_tasks(cls, definition: SourceCardDefinition) -> list[dict[str, Any]]: + queryset = BackgroundJobService.get_queryset().filter( + task_name__in=definition.task_names, + status__in=ACTIVE_JOB_STATUSES, + ) + return [ + cls._serialize_job(job) + for job in queryset.order_by("-created_at")[:10] + ] + + @classmethod + def _get_progress(cls, active_tasks: list[dict[str, Any]]) -> int: + if not active_tasks: + return 0 + total = sum(int(task["progress"]) for task in active_tasks) + return round(total / len(active_tasks)) + + @classmethod + def _get_status( + cls, + *, + definition: SourceCardDefinition, + active_tasks: list[dict[str, Any]], + latest_load: ParserLoadLog | None, + last_updated_at, + ) -> str: + if not definition.is_available: + return "unavailable" + if active_tasks: + return "in_progress" + if latest_load and latest_load.status == "in_progress": + return "in_progress" + if latest_load and latest_load.status == "failed": + return "error" + if last_updated_at: + return "success" + return "idle" + + @classmethod + def _get_status_label(cls, status: str) -> str: + labels = { + "success": "Обновлено", + "in_progress": "В процессе", + "error": "Ошибка", + "idle": "Нет данных", + "unavailable": "Не подключено", + } + return labels.get(status, status) + + @classmethod + def _get_latest_data_timestamp(cls, source_items: list[dict[str, Any]]): + timestamps = [ + item["last_updated_at"] for item in source_items if item["last_updated_at"] + ] + return max(timestamps) if timestamps else None + + @classmethod + def _get_next_update_at( + cls, + definition: SourceCardDefinition, + last_updated_at, + ): + if not definition.refresh_interval or not last_updated_at: + return None + return last_updated_at + definition.refresh_interval + + @classmethod + def _serialize_job(cls, job) -> dict[str, Any]: + return { + "task_id": job.task_id, + "task_name": job.task_name, + "status": job.status, + "progress": job.progress, + "progress_message": job.progress_message, + "started_at": job.started_at, + "created_at": job.created_at, + "meta": job.meta, + } + + @classmethod + def _serialize_load_log(cls, load_log: ParserLoadLog | None) -> dict[str, Any] | None: + if load_log is None: + return None + return { + "batch_id": load_log.batch_id, + "source": load_log.source, + "source_display": load_log.get_source_display(), + "records_count": load_log.records_count, + "status": load_log.status, + "error_message": load_log.error_message, + "created_at": load_log.created_at, + "updated_at": load_log.updated_at, + } diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index dd683b4..39f7432 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -15,6 +15,7 @@ from pathlib import Path from apps.core.services import BackgroundJobService from apps.parsers.clients.minpromtorg import ( IndustrialProductionClient, + IndustrialProductsClient, ManufacturesClient, ) from apps.parsers.clients.proverki import ProverkiClient @@ -23,6 +24,7 @@ from apps.parsers.models import ParserLoadLog from apps.parsers.services import ( FNSReportService, IndustrialCertificateService, + IndustrialProductService, InspectionService, ManufacturerService, ParserLoadLogService, @@ -39,6 +41,33 @@ DEFAULT_START_YEAR = 2025 DEFAULT_START_MONTH = 1 +def _get_or_create_background_job( + *, + task_id: str, + task_name: str, + source: str, + batch_id: int | None = None, + requested_by_id: int | None = None, + meta: dict | None = None, +): + """Reuse a pre-created job or create a new one for the task.""" + job = BackgroundJobService.get_by_task_id_or_none(task_id) + if not job: + payload = {"source": source, **(meta or {})} + if batch_id is not None: + payload["batch_id"] = batch_id + job = BackgroundJobService.create_job( + task_id=task_id, + task_name=task_name, + user_id=requested_by_id, + meta=payload, + ) + elif requested_by_id is not None and job.user_id is None: + job.user_id = requested_by_id + job.save(update_fields=["user_id", "updated_at"]) + return job + + def _lock_path_for(file_path: Path) -> Path: return Path(f"{file_path}.lock") @@ -94,6 +123,7 @@ def _process_fns_file_sync( file_path: str | Path, *, task_id: str, + requested_by_id: int | None = None, raise_on_error: bool = False, ) -> dict: import hashlib @@ -119,12 +149,18 @@ def _process_fns_file_sync( file_path.name, ) - # Создаём BackgroundJob - job = BackgroundJobService.create_job( - task_id=task_id, - task_name="apps.parsers.tasks.process_fns_file", - meta={"source": source, "batch_id": batch_id, "file": file_path.name}, - ) + # Используем предсозданную задачу из API, если она уже существует. + job = BackgroundJobService.get_by_task_id_or_none(task_id) + if not job: + job = BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.parsers.tasks.process_fns_file", + user_id=requested_by_id, + meta={"source": source, "batch_id": batch_id, "file": file_path.name}, + ) + elif requested_by_id is not None and job.user_id is None: + job.user_id = requested_by_id + job.save(update_fields=["user_id", "updated_at"]) job.mark_started() job.update_progress(0, f"Обработка файла {file_path.name}...") @@ -245,6 +281,7 @@ def parse_industrial_production( self, proxies: list[str] | None = None, client_adapter: BaseAdapter | None = None, + requested_by_id: int | None = None, ) -> dict: """ Задача парсинга сертификатов промышленного производства. @@ -276,10 +313,12 @@ def parse_industrial_production( ) # Создаём запись BackgroundJob для отслеживания прогресса - job = BackgroundJobService.create_job( + job = _get_or_create_background_job( task_id=task_id, task_name="apps.parsers.tasks.parse_industrial_production", - meta={"source": source, "batch_id": batch_id}, + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, ) job.mark_started() job.update_progress(0, "Инициализация парсера...") @@ -336,6 +375,7 @@ def parse_manufactures( self, proxies: list[str] | None = None, client_adapter: BaseAdapter | None = None, + requested_by_id: int | None = None, ) -> dict: """ Задача парсинга реестра производителей. @@ -367,10 +407,12 @@ def parse_manufactures( ) # Создаём запись BackgroundJob для отслеживания прогресса - job = BackgroundJobService.create_job( + job = _get_or_create_background_job( task_id=task_id, task_name="apps.parsers.tasks.parse_manufactures", - meta={"source": source, "batch_id": batch_id}, + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, ) job.mark_started() job.update_progress(0, "Инициализация парсера...") @@ -422,6 +464,91 @@ def parse_manufactures( raise +@shared_task(bind=True) +def parse_industrial_products( + self, + proxies: list[str] | None = None, + client_adapter: BaseAdapter | None = None, + requested_by_id: int | None = None, +) -> dict: + """ + Задача парсинга реестра промышленной продукции. + + Args: + proxies: Список прокси-серверов (опционально). + client_adapter: HTTP-адаптер (опционально). + + Returns: + Результат: batch_id, saved, status + """ + source = ParserLoadLog.Source.INDUSTRIAL_PRODUCTS + load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( + source=source, + status="in_progress", + ) + task_id = self.request.id or str(uuid.uuid4()) + + if proxies is None: + proxies = ProxyService.get_active_proxies_or_none() + + logger.info( + "Starting industrial products parsing (task_id=%s, batch_id=%d, proxies=%d)", + task_id, + batch_id, + len(proxies) if proxies else 0, + ) + + job = _get_or_create_background_job( + task_id=task_id, + task_name="apps.parsers.tasks.parse_industrial_products", + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, + ) + job.mark_started() + job.update_progress(0, "Инициализация парсера...") + + try: + job.update_progress(10, "Загрузка данных с API Минпромторга...") + client_kwargs = {"proxies": proxies} + if client_adapter: + client_kwargs["http_adapter"] = client_adapter + with IndustrialProductsClient(**client_kwargs) as client: + products = client.fetch_products() + + job.update_progress(50, f"Сохранение {len(products)} записей продукции...") + saved_count = IndustrialProductService.save_products( + products, + batch_id=batch_id, + ) + + ParserLoadLogService.update( + load_log, + status="success", + records_count=saved_count, + ) + + job.complete(result={"batch_id": batch_id, "saved": saved_count}) + + logger.info( + "Industrial products parsing completed (batch_id=%d, saved=%d)", + batch_id, + saved_count, + ) + + return { + "batch_id": batch_id, + "saved": saved_count, + "status": "success", + } + + except Exception as e: + logger.error("Industrial products parsing failed: %s", e, exc_info=True) + ParserLoadLogService.mark_failed(load_log, str(e)) + job.fail(error=str(e)) + raise + + @shared_task def parse_all_minpromtorg( proxies: list[str] | None = None, @@ -444,6 +571,9 @@ def parse_all_minpromtorg( industrial_result = parse_industrial_production.apply( kwargs={"proxies": proxies, "client_adapter": client_adapter} ) + industrial_products_result = parse_industrial_products.apply( + kwargs={"proxies": proxies, "client_adapter": client_adapter} + ) manufactures_result = parse_manufactures.apply( kwargs={"proxies": proxies, "client_adapter": client_adapter} ) @@ -452,6 +582,10 @@ def parse_all_minpromtorg( proxies=proxies, client_adapter=client_adapter, ) + industrial_products_result = parse_industrial_products.delay( + proxies=proxies, + client_adapter=client_adapter, + ) manufactures_result = parse_manufactures.delay( proxies=proxies, client_adapter=client_adapter, @@ -459,6 +593,7 @@ def parse_all_minpromtorg( results = { "industrial": industrial_result.id, + "industrial_products": industrial_products_result.id, "manufactures": manufactures_result.id, } @@ -475,6 +610,7 @@ def parse_inspections( proxies: list[str] | None = None, client_adapter: BaseAdapter | None = None, use_playwright: bool | None = None, + requested_by_id: int | None = None, ) -> dict: """ Задача парсинга данных о проверках с proverki.gov.ru. @@ -512,10 +648,13 @@ def parse_inspections( ) # Создаём запись BackgroundJob для отслеживания прогресса - job = BackgroundJobService.create_job( + job = _get_or_create_background_job( task_id=task_id, task_name="apps.parsers.tasks.parse_inspections", - meta={"source": source, "batch_id": batch_id, "year": year, "month": month}, + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, + meta={"year": year, "month": month}, ) job.mark_started() job.update_progress(0, "Инициализация парсера...") @@ -602,6 +741,9 @@ def parse_all_sources( industrial_result = parse_industrial_production.apply( kwargs={"proxies": proxies, "client_adapter": client_adapter} ) + industrial_products_result = parse_industrial_products.apply( + kwargs={"proxies": proxies, "client_adapter": client_adapter} + ) manufactures_result = parse_manufactures.apply( kwargs={"proxies": proxies, "client_adapter": client_adapter} ) @@ -617,6 +759,10 @@ def parse_all_sources( proxies=proxies, client_adapter=client_adapter, ) + industrial_products_result = parse_industrial_products.delay( + proxies=proxies, + client_adapter=client_adapter, + ) manufactures_result = parse_manufactures.delay( proxies=proxies, client_adapter=client_adapter, @@ -629,6 +775,7 @@ def parse_all_sources( results = { "industrial": industrial_result.id, + "industrial_products": industrial_products_result.id, "manufactures": manufactures_result.id, "inspections": inspections_result.id, } @@ -652,6 +799,7 @@ def sync_inspections( # noqa: C901 use_playwright: bool | None = None, current_year: int | None = None, current_month: int | None = None, + requested_by_id: int | None = None, ) -> dict: """ Синхронизация данных о проверках с proverki.gov.ru. @@ -689,10 +837,12 @@ def sync_inspections( # noqa: C901 ) # Создаём запись BackgroundJob - job = BackgroundJobService.create_job( + job = _get_or_create_background_job( task_id=task_id, task_name="apps.parsers.tasks.sync_inspections", - meta={"source": source, "batch_id": batch_id}, + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, ) job.mark_started() job.update_progress(0, "Инициализация синхронизации...") @@ -865,6 +1015,7 @@ def parse_procurements( client_host: str | None = None, client_scheme: str | None = None, client_adapter: BaseAdapter | None = None, + requested_by_id: int | None = None, ) -> dict: """ Задача парсинга данных о государственных закупках с zakupki.gov.ru. @@ -905,12 +1056,13 @@ def parse_procurements( ) # Создаём запись BackgroundJob для отслеживания прогресса - job = BackgroundJobService.create_job( + job = _get_or_create_background_job( task_id=task_id, task_name="apps.parsers.tasks.parse_procurements", + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, meta={ - "source": source, - "batch_id": batch_id, "region_code": region_code, "year": year, "month": month, @@ -997,6 +1149,7 @@ def sync_procurements( # noqa: C901 client_adapter: BaseAdapter | None = None, current_year: int | None = None, current_month: int | None = None, + requested_by_id: int | None = None, ) -> dict: """ Синхронизация данных о закупках с zakupki.gov.ru. @@ -1034,12 +1187,13 @@ def sync_procurements( # noqa: C901 ) # Создаём запись BackgroundJob - job = BackgroundJobService.create_job( + job = _get_or_create_background_job( task_id=task_id, task_name="apps.parsers.tasks.sync_procurements", + source=source, + batch_id=batch_id, + requested_by_id=requested_by_id, meta={ - "source": source, - "batch_id": batch_id, "region_code": region_code, "law_type": law_type, }, @@ -1193,7 +1347,10 @@ def sync_procurements( # noqa: C901 @shared_task(bind=True) -def scan_fns_directory(self) -> dict: +def scan_fns_directory( + self, + requested_by_id: int | None = None, +) -> dict: """ Периодическая задача: сканирует папку fns на новые файлы. @@ -1207,14 +1364,24 @@ def scan_fns_directory(self) -> dict: from django.conf import settings - task_id = self.request.id + task_id = self.request.id or str(uuid.uuid4()) + job = _get_or_create_background_job( + task_id=task_id, + task_name="apps.parsers.tasks.scan_fns_directory", + source=ParserLoadLog.Source.FNS_REPORTS, + requested_by_id=requested_by_id, + ) + job.mark_started() + job.update_progress(0, "Сканирование директории ФНС...") logger.info("Starting FNS directory scan (task_id=%s)", task_id) watch_dir = Path(settings.FNS_WATCH_DIRECTORY) if not watch_dir.exists(): logger.warning("FNS watch directory does not exist: %s", watch_dir) watch_dir.mkdir(parents=True, exist_ok=True) - return {"scanned": 0, "queued": 0, "skipped": 0} + result = {"scanned": 0, "queued": 0, "skipped": 0} + job.complete(result=result) + return result queued = 0 skipped = 0 @@ -1247,7 +1414,13 @@ def scan_fns_directory(self) -> dict: # Ставим в очередь на обработку try: - process_fns_file.delay(str(file_path)) + if requested_by_id is None: + process_fns_file.delay(str(file_path)) + else: + process_fns_file.delay( + str(file_path), + requested_by_id=requested_by_id, + ) except Exception as e: logger.error("Failed to enqueue FNS file: %s - %s", file_path, e) _remove_lock(file_path) @@ -1264,19 +1437,26 @@ def scan_fns_directory(self) -> dict: skipped, ) - return { + result = { "scanned": len(files_found), "queued": queued, "skipped": skipped, } + job.complete(result=result) + return result @shared_task(bind=True) -def process_fns_file(self, file_path: str) -> dict: +def process_fns_file( + self, + file_path: str, + requested_by_id: int | None = None, +) -> dict: """Обработка одного файла FNS.""" return _process_fns_file_sync( file_path, task_id=self.request.id, + requested_by_id=requested_by_id, raise_on_error=True, ) diff --git a/src/apps/parsers/urls.py b/src/apps/parsers/urls.py index 12c5438..4093338 100644 --- a/src/apps/parsers/urls.py +++ b/src/apps/parsers/urls.py @@ -8,11 +8,16 @@ from apps.parsers.views import ( FinancialReportViewSet, FNSReportUploadView, IndustrialCertificateViewSet, + IndustrialProductViewSet, InspectionViewSet, ManufacturerViewSet, ParserLoadLogViewSet, ProcurementViewSet, ProxyViewSet, + SourceCardDetailView, + SourceCardListView, + SourceCardRefreshView, + SourceTaskStatusListView, ) from django.urls import include, path from rest_framework.routers import DefaultRouter @@ -30,6 +35,9 @@ minpromtorg_router.register( minpromtorg_router.register( r"manufacturers", ManufacturerViewSet, basename="manufacturers" ) +minpromtorg_router.register( + r"products", IndustrialProductViewSet, basename="industrial-products" +) minpromtorg_urlpatterns = [ path("", include(minpromtorg_router.urls)), @@ -69,6 +77,21 @@ fns_urlpatterns = [ path("", include(fns_router.urls)), ] +# ============================================================================= +# Frontend sources: /api/v1/sources/ +# ============================================================================= + +sources_urlpatterns = [ + path("", SourceCardListView.as_view(), name="source-cards-list"), + path("statuses/", SourceTaskStatusListView.as_view(), name="source-cards-statuses"), + path("/", SourceCardDetailView.as_view(), name="source-cards-detail"), + path( + "/refresh/", + SourceCardRefreshView.as_view(), + name="source-cards-refresh", + ), +] + # ============================================================================= # Системные (логи, прокси): /api/v1/system/ # ============================================================================= diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index a3fdf68..7d5ceff 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -7,12 +7,16 @@ Views для приложения парсеров. import hashlib import time +import uuid from pathlib import Path from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag +from apps.core.response import api_response +from apps.core.services import BackgroundJobService from apps.parsers.models import ( FinancialReport, IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -24,12 +28,19 @@ from apps.parsers.serializers import ( FinancialReportSerializer, FNSFileUploadSerializer, IndustrialCertificateSerializer, + IndustrialProductSerializer, InspectionSerializer, ManufacturerSerializer, ParserLoadLogSerializer, ProcurementSerializer, ProxySerializer, + SourceCardDetailSerializer, + SourceCardRefreshRequestSerializer, + SourceCardRefreshResponseSerializer, + SourceCardSerializer, + SourceTaskStatusSerializer, ) +from apps.parsers.source_cards import SourceCardService from apps.parsers.tasks import process_fns_file from django.conf import settings from django.db.models import Count @@ -50,6 +61,7 @@ MINPROMTORG_TAG = swagger_tag("Минпромторг", "minpromtorg") PROVERKI_TAG = swagger_tag("Единый реестр проверок", "inspections_registry") ZAKUPKI_TAG = swagger_tag("Государственные закупки", "public_procurements") FNS_TAG = swagger_tag("ФНС - Бухгалтерская отчетность", "fns_financial_reports") +SOURCES_TAG = swagger_tag("Источники для фронта", "frontend_sources") SYSTEM_TAG = swagger_tag("Системные", "system") @@ -155,6 +167,66 @@ class ManufacturerViewSet(ReadOnlyModelViewSet): return super().retrieve(request, *args, **kwargs) +class IndustrialProductViewSet(ReadOnlyModelViewSet): + """ + API для просмотра реестра промышленной продукции. + + Данные загружаются из Минпромторга. + Только чтение - добавление через парсер/админку. + """ + + queryset = IndustrialProductRecord.objects.all().order_by("-created_at") + serializer_class = IndustrialProductSerializer + permission_classes = [IsAuthenticated] + filterset_fields = [ + "inn", + "ogrn", + "registry_number", + "load_batch", + "registry_organization", + ] + search_fields = [ + "full_organisation_name", + "product_name", + "product_model", + "registry_number", + "inn", + "ogrn", + ] + + @swagger_auto_schema( + tags=[MINPROMTORG_TAG], + operation_summary="Список промышленной продукции", + operation_description=( + "Возвращает список записей реестра промышленной продукции " + "Минпромторга.\n" + "Поддерживает фильтрацию по: inn, ogrn, registry_number, load_batch.\n" + "Поддерживает поиск по: full_organisation_name, product_name, " + "product_model, registry_number, inn, ogrn." + ), + responses={ + 200: IndustrialProductSerializer(many=True), + **ErrorResponses.AUTHENTICATED, + }, + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + + @swagger_auto_schema( + tags=[MINPROMTORG_TAG], + operation_summary="Детали записи промышленной продукции", + operation_description=( + "Возвращает информацию о конкретной записи реестра промышленной продукции." + ), + responses={ + 200: IndustrialProductSerializer, + **ErrorResponses.AUTHENTICATED_NOT_FOUND, + }, + ) + def retrieve(self, request, *args, **kwargs): + return super().retrieve(request, *args, **kwargs) + + # ============================================================================= # Единый реестр проверок (proverki.gov.ru) # ============================================================================= @@ -364,7 +436,7 @@ class FNSReportUploadView(APIView): """ parser_classes = [MultiPartParser] - permission_classes = [IsAuthenticated] + permission_classes = [IsAdminUser] @swagger_auto_schema( tags=[FNS_TAG], @@ -410,7 +482,7 @@ class FNSReportUploadView(APIView): ), ), 400: CommonResponses.BAD_REQUEST, - **ErrorResponses.AUTHENTICATED, + **ErrorResponses.ADMIN, }, ) def post(self, request): # noqa @@ -479,9 +551,26 @@ class FNSReportUploadView(APIView): # Ставим в очередь try: - task = process_fns_file.delay(str(file_path)) + task_id = str(uuid.uuid4()) + BackgroundJobService.create_job( + task_id=task_id, + task_name="apps.parsers.tasks.process_fns_file", + user_id=request.user.id, + meta={ + "source": ParserLoadLog.Source.FNS_REPORTS, + "file": file.name, + }, + ) + task = process_fns_file.apply_async( + args=[str(file_path)], + kwargs={"requested_by_id": request.user.id}, + task_id=task_id, + ) except Exception: lock_path.unlink(missing_ok=True) + from apps.core.models import BackgroundJob + + BackgroundJob.objects.filter(task_id=task_id).delete() raise task_ids.append(task.id) queued += 1 @@ -496,6 +585,111 @@ class FNSReportUploadView(APIView): ) +# ============================================================================= +# Frontend-oriented source cards +# ============================================================================= + + +class SourceCardListView(APIView): + """List of aggregated source cards for frontend pages.""" + + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + tags=[SOURCES_TAG], + operation_summary="Список карточек источников", + operation_description=( + "Возвращает агрегированный список карточек источников данных " + "для фронтенда." + ), + responses={ + 200: SourceCardSerializer(many=True), + **ErrorResponses.AUTHENTICATED, + }, + ) + def get(self, request): + cards = SourceCardService.list_cards() + serializer = SourceCardSerializer(cards, many=True) + return api_response(serializer.data) + + +class SourceTaskStatusListView(APIView): + """Tabular list of parsing source statuses for frontend screens.""" + + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + tags=[SOURCES_TAG], + operation_summary="Статусы задач парсинга", + operation_description=( + "Возвращает табличный список статусов источников данных для экрана " + "мониторинга парсинга." + ), + responses={ + 200: SourceTaskStatusSerializer(many=True), + **ErrorResponses.AUTHENTICATED, + }, + ) + def get(self, request): + rows = SourceCardService.list_task_statuses() + serializer = SourceTaskStatusSerializer(rows, many=True) + return api_response(serializer.data) + + +class SourceCardDetailView(APIView): + """Detailed frontend card for a single source.""" + + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + tags=[SOURCES_TAG], + operation_summary="Детали карточки источника", + operation_description=( + "Возвращает детальную информацию по одной карточке источника, " + "включая подисточники, последние загрузки и активные задачи." + ), + responses={ + 200: SourceCardDetailSerializer, + **ErrorResponses.AUTHENTICATED_NOT_FOUND, + }, + ) + def get(self, request, slug: str): + card = SourceCardService.get_card(slug) + serializer = SourceCardDetailSerializer(card) + return api_response(serializer.data) + + +class SourceCardRefreshView(APIView): + """Manual refresh trigger for a frontend source card.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[SOURCES_TAG], + operation_summary="Запуск обновления карточки источника", + operation_description=( + "Ставит обновление карточки в очередь и возвращает связанные task_id." + ), + request_body=SourceCardRefreshRequestSerializer, + responses={ + 202: SourceCardRefreshResponseSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN_NOT_FOUND, + }, + ) + def post(self, request, slug: str): + serializer = SourceCardRefreshRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + payload = SourceCardService.refresh_card( + slug=slug, + requested_by_id=request.user.id if request.user.is_authenticated else None, + params=serializer.validated_data.get("params", {}), + ) + output = SourceCardRefreshResponseSerializer(payload) + return api_response(output.data, status_code=status.HTTP_202_ACCEPTED) + + # ============================================================================= # Системные (логи загрузок, прокси) # ============================================================================= diff --git a/src/apps/registers/models.py b/src/apps/registers/models.py index 26e4217..3bb3d97 100644 --- a/src/apps/registers/models.py +++ b/src/apps/registers/models.py @@ -191,7 +191,8 @@ class RegistryMembershipPeriod(TimestampMixin, models.Model): ] constraints = [ models.CheckConstraint( - check=Q(ended_at__isnull=True) | Q(ended_at__gte=models.F("started_at")), + check=Q(ended_at__isnull=True) + | Q(ended_at__gte=models.F("started_at")), name="check_membership_period_dates", ), models.UniqueConstraint( diff --git a/src/apps/registers/serializers.py b/src/apps/registers/serializers.py index 311165c..6c7c888 100644 --- a/src/apps/registers/serializers.py +++ b/src/apps/registers/serializers.py @@ -96,7 +96,9 @@ class OrganizationListQuerySerializer(serializers.Serializer): def validate(self, attrs): if attrs.get("actual_date") and not attrs.get("registry"): raise serializers.ValidationError( - {"actual_date": "Параметр actual_date допустим только вместе с registry"} + { + "actual_date": "Параметр actual_date допустим только вместе с registry" + } ) return attrs diff --git a/src/apps/registers/services.py b/src/apps/registers/services.py index 50793f6..6bb73aa 100644 --- a/src/apps/registers/services.py +++ b/src/apps/registers/services.py @@ -139,7 +139,9 @@ class RegisterImportService: if created: organizations_created += 1 else: - updated = cls._update_organization_fields(organization=organization, row=row) + updated = cls._update_organization_fields( + organization=organization, row=row + ) if updated: organizations_updated += 1 @@ -470,7 +472,9 @@ class RegisterImportService: workbook.close() @classmethod - def _validate_snapshot_date(cls, *, registry: Register, snapshot_date: date) -> None: + def _validate_snapshot_date( + cls, *, registry: Register, snapshot_date: date + ) -> None: latest_upload = ( RegisterUpload.objects.filter(registry=registry) .order_by("-actual_date", "-id") diff --git a/src/apps/registers/views.py b/src/apps/registers/views.py index bdda439..1dbf72f 100644 --- a/src/apps/registers/views.py +++ b/src/apps/registers/views.py @@ -23,7 +23,7 @@ from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.generics import ListAPIView from rest_framework.parsers import MultiPartParser -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView from rest_framework.viewsets import ReadOnlyModelViewSet @@ -93,10 +93,15 @@ class OrganizationViewSet(ReadOnlyModelViewSet): .prefetch_related("membership_periods__registry") ) - params_serializer = OrganizationListQuerySerializer(data=self.request.query_params) + params_serializer = OrganizationListQuerySerializer( + data=self.request.query_params + ) params_serializer.is_valid(raise_exception=True) - queryset, self.actual_date_meta = RegisterImportService.get_organizations_queryset( + ( + queryset, + self.actual_date_meta, + ) = RegisterImportService.get_organizations_queryset( **params_serializer.validated_data ) @@ -205,11 +210,12 @@ class RegistryOrganizationListView(ListAPIView): ) params_serializer.is_valid(raise_exception=True) - queryset, self.actual_date_meta = ( - RegisterImportService.get_registry_organizations_queryset( - registry=registry, - **params_serializer.validated_data, - ) + ( + queryset, + self.actual_date_meta, + ) = RegisterImportService.get_registry_organizations_queryset( + registry=registry, + **params_serializer.validated_data, ) return queryset @@ -280,7 +286,7 @@ class RegisterUploadView(APIView): """API загрузки Excel файла организаций в выбранный реестр.""" parser_classes = [MultiPartParser] - permission_classes = [IsAuthenticated] + permission_classes = [IsAdminUser] @swagger_auto_schema( tags=[REGISTERS_TAG], @@ -345,7 +351,7 @@ class RegisterUploadView(APIView): ), ), 400: CommonResponses.BAD_REQUEST, - **ErrorResponses.AUTHENTICATED, + **ErrorResponses.ADMIN, }, ) def post(self, request): diff --git a/src/apps/user/migrations/0005_create_default_admin_superuser.py b/src/apps/user/migrations/0005_create_default_admin_superuser.py index 5c6e322..328524b 100644 --- a/src/apps/user/migrations/0005_create_default_admin_superuser.py +++ b/src/apps/user/migrations/0005_create_default_admin_superuser.py @@ -1,5 +1,6 @@ # Generated by Django 3.2.25 on 2026-03-04 +from django.contrib.auth.hashers import make_password from django.db import migrations TARGET_USERNAME = "admin" @@ -35,7 +36,7 @@ def create_or_update_admin_superuser(apps, schema_editor): user.is_staff = True user.is_superuser = True user.is_active = True - user.set_password(TARGET_PASSWORD) + user.password = make_password(TARGET_PASSWORD) user.save(using=db_alias) diff --git a/src/apps/user/migrations/0006_create_default_role_groups.py b/src/apps/user/migrations/0006_create_default_role_groups.py new file mode 100644 index 0000000..fb16c82 --- /dev/null +++ b/src/apps/user/migrations/0006_create_default_role_groups.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.25 on 2026-03-17 + +from django.db import migrations + +ROLE_ADMIN = "admin" +ROLE_USER = "user" + + +def create_default_role_groups(apps, schema_editor): + Group = apps.get_model("auth", "Group") + User = apps.get_model("user", "User") + db_alias = schema_editor.connection.alias + + admin_group, _ = Group.objects.using(db_alias).get_or_create(name=ROLE_ADMIN) + user_group, _ = Group.objects.using(db_alias).get_or_create(name=ROLE_USER) + + for user in User.objects.using(db_alias).all().iterator(): + role_groups = user.groups.filter(name__in=[ROLE_ADMIN, ROLE_USER]) + if role_groups.exists(): + continue + + if user.is_staff or user.is_superuser: + user.groups.add(admin_group) + else: + user.groups.add(user_group) + + +class Migration(migrations.Migration): + dependencies = [ + ("user", "0005_create_default_admin_superuser"), + ] + + operations = [ + migrations.RunPython( + create_default_role_groups, + migrations.RunPython.noop, + ), + ] diff --git a/src/apps/user/serializers.py b/src/apps/user/serializers.py index b6b423e..555f2be 100644 --- a/src/apps/user/serializers.py +++ b/src/apps/user/serializers.py @@ -3,6 +3,7 @@ from rest_framework import serializers from rest_framework.validators import UniqueValidator from .models import Profile +from .services import UserService User = get_user_model() @@ -39,10 +40,7 @@ class UserRegistrationSerializer(serializers.ModelSerializer): def create(self, validated_data): validated_data.pop("password_confirm") password = validated_data.pop("password") - user = User.objects.create_user(**validated_data) - user.set_password(password) - user.save() - return user + return UserService.create_user(password=password, **validated_data) class UserProfileSerializer(serializers.ModelSerializer): @@ -69,6 +67,9 @@ class UserSerializer(serializers.ModelSerializer): """Сериализатор для пользователя""" profile = UserProfileSerializer(read_only=True) + role = serializers.SerializerMethodField() + role_label = serializers.SerializerMethodField() + capabilities = serializers.SerializerMethodField() class Meta: model = User @@ -77,12 +78,34 @@ class UserSerializer(serializers.ModelSerializer): "email", "username", "phone", + "is_active", "is_verified", + "role", + "role_label", + "capabilities", "profile", "created_at", "updated_at", ) - read_only_fields = ("id", "is_verified", "created_at", "updated_at") + read_only_fields = ( + "id", + "is_active", + "is_verified", + "role", + "role_label", + "capabilities", + "created_at", + "updated_at", + ) + + def get_role(self, obj) -> str: + return UserService.get_user_role(obj) + + def get_role_label(self, obj) -> str: + return UserService.get_role_label(self.get_role(obj)) + + def get_capabilities(self, obj) -> dict: + return UserService.get_user_capabilities(obj) class UserUpdateSerializer(serializers.ModelSerializer): @@ -93,6 +116,79 @@ class UserUpdateSerializer(serializers.ModelSerializer): fields = ("username", "phone") +class AdminUserCreateSerializer(serializers.ModelSerializer): + """Сериализатор для создания пользователя администратором.""" + + password = serializers.CharField( + write_only=True, min_length=8, help_text="Пароль (минимум 8 символов)" + ) + role = serializers.ChoiceField( + choices=UserService.ROLE_CHOICES, + default=UserService.ROLE_USER, + help_text="Прикладная роль пользователя", + ) + first_name = serializers.CharField( + required=False, allow_blank=True, allow_null=True + ) + last_name = serializers.CharField(required=False, allow_blank=True, allow_null=True) + + class Meta: + model = User + fields = ( + "email", + "username", + "phone", + "password", + "role", + "is_active", + "is_verified", + "first_name", + "last_name", + ) + extra_kwargs = { + "email": { + "validators": [UniqueValidator(queryset=User.objects.all())], + }, + "username": { + "validators": [UniqueValidator(queryset=User.objects.all())], + }, + } + + +class AdminUserUpdateSerializer(serializers.ModelSerializer): + """Сериализатор для обновления пользователя администратором.""" + + password = serializers.CharField( + write_only=True, + required=False, + min_length=8, + help_text="Новый пароль (опционально)", + ) + role = serializers.ChoiceField( + choices=UserService.ROLE_CHOICES, + required=False, + help_text="Прикладная роль пользователя", + ) + first_name = serializers.CharField( + required=False, allow_blank=True, allow_null=True + ) + last_name = serializers.CharField(required=False, allow_blank=True, allow_null=True) + + class Meta: + model = User + fields = ( + "email", + "username", + "phone", + "password", + "role", + "is_active", + "is_verified", + "first_name", + "last_name", + ) + + class ProfileUpdateSerializer(serializers.ModelSerializer): """Сериализатор для обновления профиля""" diff --git a/src/apps/user/services.py b/src/apps/user/services.py index 3398035..a113b5b 100644 --- a/src/apps/user/services.py +++ b/src/apps/user/services.py @@ -2,6 +2,7 @@ from typing import Any from apps.core.exceptions import NotFoundError from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from django.db import transaction from rest_framework_simplejwt.tokens import RefreshToken @@ -13,9 +14,27 @@ User = get_user_model() class UserService: """Сервисный слой для работы с пользователями""" + ROLE_USER = "user" + ROLE_ADMIN = "admin" + ROLE_CHOICES = ( + (ROLE_USER, "Пользователь"), + (ROLE_ADMIN, "Администратор"), + ) + ROLE_LABELS = dict(ROLE_CHOICES) + SETTINGS_SECTION_EXCHANGE = "exchange" + SETTINGS_SECTION_PARSERS = "parsers" + SETTINGS_SECTION_DATABASE_CONNECTION = "database_connection" + SETTINGS_SECTION_REGISTERS_UPLOAD = "registers_upload" + @classmethod def create_user( - cls, *, email: str, username: str, password: str, **extra_fields + cls, + *, + email: str, + username: str, + password: str, + role: str | None = None, + **extra_fields, ) -> User: """ Создает нового пользователя @@ -32,12 +51,24 @@ class UserService: Raises: ValidationError: При некорректных данных """ + role = role or cls.ROLE_USER with transaction.atomic(): user = User.objects.create_user( email=email, username=username, password=password, **extra_fields ) + cls.assign_role(user, role) return user + @classmethod + def get_users_queryset(cls): + """Базовый queryset для админского списка пользователей.""" + return ( + User.objects.all() + .select_related("profile") + .prefetch_related("groups") + .order_by("-created_at") + ) + @classmethod def get_user_by_email(cls, email: str) -> User: """Получает пользователя по email @@ -107,6 +138,69 @@ class UserService: user.save() return user + @classmethod + @transaction.atomic + def create_managed_user( + cls, + *, + email: str, + username: str, + password: str, + role: str, + first_name: str | None = None, + last_name: str | None = None, + **extra_fields, + ) -> User: + """Создаёт пользователя администратором и назначает роль.""" + user = cls.create_user( + email=email, + username=username, + password=password, + role=role, + **extra_fields, + ) + cls._update_or_create_profile( + user=user, + first_name=first_name, + last_name=last_name, + ) + return cls.get_users_queryset().get(id=user.id) + + @classmethod + @transaction.atomic + def update_managed_user(cls, user_id: int, **fields) -> User: + """Обновляет пользователя и его роль из админского интерфейса.""" + user = cls.get_user_by_id(user_id) + role = fields.pop("role", None) + password = fields.pop("password", None) + profile_fields = { + key: fields.pop(key) for key in ("first_name", "last_name") if key in fields + } + + for field, value in fields.items(): + setattr(user, field, value) + + if password: + user.set_password(password) + + user.save() + + if role is not None: + cls.assign_role(user, role) + + if profile_fields: + cls._update_or_create_profile(user=user, **profile_fields) + + return cls.get_users_queryset().get(id=user.id) + + @classmethod + def deactivate_user(cls, user_id: int) -> User: + """Деактивирует пользователя без физического удаления.""" + user = cls.get_user_by_id(user_id) + user.is_active = False + user.save(update_fields=["is_active"]) + return user + @classmethod def delete_user(cls, user_id: int) -> None: """ @@ -138,6 +232,76 @@ class UserService: "access": str(refresh.access_token), } + @classmethod + def ensure_role_groups(cls) -> dict[str, Group]: + """Гарантирует существование системных role-групп.""" + groups: dict[str, Group] = {} + for role, _label in cls.ROLE_CHOICES: + group, _ = Group.objects.get_or_create(name=role) + groups[role] = group + return groups + + @classmethod + def get_user_role(cls, user: User) -> str: + """Возвращает прикладную роль пользователя.""" + if user.is_superuser or user.is_staff: + return cls.ROLE_ADMIN + + group_names = {group.name for group in user.groups.all()} + if cls.ROLE_ADMIN in group_names: + return cls.ROLE_ADMIN + return cls.ROLE_USER + + @classmethod + def get_role_label(cls, role: str) -> str: + """Возвращает человекочитаемое название роли.""" + return cls.ROLE_LABELS.get(role, role) + + @classmethod + def get_user_capabilities(cls, user: User) -> dict[str, Any]: + """Возвращает фронтовые capability flags по роли пользователя.""" + is_admin = cls.get_user_role(user) == cls.ROLE_ADMIN + return { + "can_manage_users": is_admin, + "can_manage_exchange": is_admin, + "can_manage_parsers": is_admin, + "can_manage_database_connection": is_admin, + "can_upload_registers": is_admin, + "can_refresh_dashboard": is_admin, + "settings_sections": ( + [ + cls.SETTINGS_SECTION_EXCHANGE, + cls.SETTINGS_SECTION_PARSERS, + cls.SETTINGS_SECTION_DATABASE_CONNECTION, + cls.SETTINGS_SECTION_REGISTERS_UPLOAD, + ] + if is_admin + else [] + ), + } + + @classmethod + def assign_role(cls, user: User, role: str) -> User: + """Назначает одну из системных ролей через auth.Group и is_staff.""" + if role not in cls.ROLE_LABELS: + raise ValueError(f"Unsupported role: {role}") + + groups = cls.ensure_role_groups() + role_group_names = list(groups.keys()) + current_role_groups = list(user.groups.filter(name__in=role_group_names)) + if current_role_groups: + user.groups.remove(*current_role_groups) + user.groups.add(groups[role]) + + if role == cls.ROLE_ADMIN: + user.is_staff = True + else: + user.is_staff = False + user.is_superuser = False + + user.save() + return user + @classmethod def verify_email(cls, user_id: int) -> User: """ @@ -157,6 +321,22 @@ class UserService: user.save() return user + @classmethod + def _update_or_create_profile( + cls, + *, + user: User, + first_name: str | None = None, + last_name: str | None = None, + ) -> Profile: + profile, _ = Profile.objects.get_or_create(user=user) + if first_name is not None: + profile.first_name = first_name + if last_name is not None: + profile.last_name = last_name + profile.save() + return profile + class ProfileService: """Сервисный слой для работы с профилями""" diff --git a/src/apps/user/urls.py b/src/apps/user/urls.py index 819f45e..80ab1f3 100644 --- a/src/apps/user/urls.py +++ b/src/apps/user/urls.py @@ -16,6 +16,17 @@ urlpatterns = [ path("me/update/", views.UserUpdateView.as_view(), name="user_update"), path("profile/", views.ProfileDetailView.as_view(), name="profile_detail"), path("profile/full/", views.user_profile_detail, name="profile_full"), + path("admin/users/", views.AdminUserListCreateView.as_view(), name="admin-users"), + path( + "admin/users//", + views.AdminUserDetailView.as_view(), + name="admin-user-detail", + ), + path( + "admin/users//deactivate/", + views.AdminUserDeactivateView.as_view(), + name="admin-user-deactivate", + ), # Безопасность path( "password/change/", views.PasswordChangeView.as_view(), name="password_change" diff --git a/src/apps/user/views.py b/src/apps/user/views.py index 9da0b7c..08ac756 100644 --- a/src/apps/user/views.py +++ b/src/apps/user/views.py @@ -1,17 +1,20 @@ from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from django.contrib.auth import authenticate from django.contrib.auth.hashers import check_password +from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework import generics, status from rest_framework.decorators import api_view, permission_classes -from rest_framework.permissions import AllowAny, IsAuthenticated +from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView from rest_framework_simplejwt.views import TokenRefreshView as SimpleJWTTokenRefreshView from rest_framework_simplejwt.views import TokenVerifyView as SimpleJWTTokenVerifyView from .serializers import ( + AdminUserCreateSerializer, + AdminUserUpdateSerializer, LoginSerializer, PasswordChangeSerializer, ProfileUpdateSerializer, @@ -25,6 +28,7 @@ from .services import ProfileService, UserService # Swagger теги для группировки AUTH_TAG = swagger_tag("Аутентификация", "authentication") USER_TAG = swagger_tag("Пользователь", "user") +USER_ADMIN_TAG = swagger_tag("Управление пользователями", "user_management") class RegisterView(APIView): @@ -151,6 +155,129 @@ class CurrentUserView(APIView): return Response(serializer.data) +class AdminUserListCreateView(APIView): + """Список пользователей и создание пользователя администратором.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[USER_ADMIN_TAG], + operation_summary="Список пользователей", + operation_description="Возвращает список пользователей. Доступно только администраторам.", + responses={ + 200: UserSerializer(many=True), + **ErrorResponses.ADMIN, + }, + ) + def get(self, request): + serializer = UserSerializer(UserService.get_users_queryset(), many=True) + return Response(serializer.data) + + @swagger_auto_schema( + tags=[USER_ADMIN_TAG], + operation_summary="Создать пользователя", + operation_description=( + "Создаёт пользователя и назначает ему одну из ролей: " "`user` или `admin`." + ), + request_body=AdminUserCreateSerializer, + responses={ + 201: UserSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN, + }, + ) + def post(self, request): + serializer = AdminUserCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + user = UserService.create_managed_user(**serializer.validated_data) + return Response(UserSerializer(user).data, status=status.HTTP_201_CREATED) + + +class AdminUserDetailView(APIView): + """Просмотр и редактирование пользователя администратором.""" + + permission_classes = [IsAdminUser] + + def _get_user(self, user_id): + return get_object_or_404(UserService.get_users_queryset(), id=user_id) + + @swagger_auto_schema( + tags=[USER_ADMIN_TAG], + operation_summary="Детали пользователя", + operation_description="Возвращает данные конкретного пользователя.", + responses={ + 200: UserSerializer, + **ErrorResponses.ADMIN_NOT_FOUND, + }, + ) + def get(self, request, user_id: int): + user = self._get_user(user_id) + return Response(UserSerializer(user).data) + + @swagger_auto_schema( + tags=[USER_ADMIN_TAG], + operation_summary="Обновить пользователя", + operation_description=( + "Частично обновляет пользователя. Администратор может изменить " + "роль, активность и базовые учётные данные." + ), + request_body=AdminUserUpdateSerializer, + responses={ + 200: UserSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN_NOT_FOUND, + }, + ) + def patch(self, request, user_id: int): + user = self._get_user(user_id) + serializer = AdminUserUpdateSerializer(user, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + + if request.user.id == user.id: + if serializer.validated_data.get("is_active") is False: + return Response( + {"detail": "Нельзя деактивировать самого себя."}, + status=status.HTTP_400_BAD_REQUEST, + ) + if serializer.validated_data.get("role") == UserService.ROLE_USER: + return Response( + {"detail": "Нельзя снять у себя роль администратора."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + updated_user = UserService.update_managed_user( + user_id=user.id, + **serializer.validated_data, + ) + return Response(UserSerializer(updated_user).data) + + +class AdminUserDeactivateView(APIView): + """Деактивация пользователя администратором.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[USER_ADMIN_TAG], + operation_summary="Деактивировать пользователя", + operation_description="Помечает пользователя как неактивного без удаления записи.", + responses={ + 200: UserSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN_NOT_FOUND, + }, + ) + def post(self, request, user_id: int): + if request.user.id == user_id: + return Response( + {"detail": "Нельзя деактивировать самого себя."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + user = UserService.deactivate_user(user_id) + return Response(UserSerializer(user).data) + + class UserUpdateView(APIView): """Обновление данных пользователя.""" diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index d746f88..69b1068 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -10,6 +10,7 @@ API v1 URL configuration. - /api/v1/proverki/ - Единый реестр проверок - /api/v1/zakupki/ - Государственные закупки - /api/v1/fns/ - ФНС (бухгалтерская отчетность) +- /api/v1/sources/ - Агрегированные карточки источников для фронтенда - /api/v1/registers/ - Реестры организаций - /api/v1/exchange/ - Обмен с внешней БД - /api/v1/backups/ - Экспорт защищённых backup-архивов @@ -23,6 +24,7 @@ from apps.parsers.urls import ( fns_urlpatterns, minpromtorg_urlpatterns, proverki_urlpatterns, + sources_urlpatterns, system_urlpatterns, zakupki_urlpatterns, ) @@ -50,6 +52,8 @@ urlpatterns = [ path("zakupki/", include((zakupki_urlpatterns, "zakupki"))), # Парсеры - ФНС бухгалтерская отчетность path("fns/", include((fns_urlpatterns, "fns"))), + # Агрегированные карточки источников для фронтенда + path("sources/", include((sources_urlpatterns, "sources"))), # Реестры организаций path("registers/", include((registers_urlpatterns, "registers"))), # Обмен с внешней БД diff --git a/src/core/celery.py b/src/core/celery.py index cfc7379..0faf39c 100644 --- a/src/core/celery.py +++ b/src/core/celery.py @@ -53,6 +53,11 @@ app.conf.beat_schedule = { "task": "apps.parsers.tasks.parse_manufactures", "schedule": 86400.0, # Every 24 hours }, + # Парсинг реестра промышленной продукции - каждый день в 5:00 + "parse-industrial-products-daily": { + "task": "apps.parsers.tasks.parse_industrial_products", + "schedule": 86400.0, # Every 24 hours + }, # Сканирование папки FNS - каждые 5 минут "scan-fns-directory": { "task": "apps.parsers.tasks.scan_fns_directory", diff --git a/src/settings/production.py b/src/settings/production.py index b371419..1f4c842 100644 --- a/src/settings/production.py +++ b/src/settings/production.py @@ -22,9 +22,7 @@ def _parse_allowed_hosts(raw_value: str) -> list[str]: if not hosts: raise ImproperlyConfigured("ALLOWED_HOSTS must contain at least one host") if "*" in hosts: - raise ImproperlyConfigured( - "ALLOWED_HOSTS must not contain '*' in production" - ) + raise ImproperlyConfigured("ALLOWED_HOSTS must not contain '*' in production") return hosts diff --git a/src/settings/test_postgres.py b/src/settings/test_postgres.py index 318579b..09d2a76 100644 --- a/src/settings/test_postgres.py +++ b/src/settings/test_postgres.py @@ -10,12 +10,16 @@ from .test import * DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", - "NAME": os.getenv("TEST_POSTGRES_DB", os.getenv("POSTGRES_DB", "mostovik_test")), + "NAME": os.getenv( + "TEST_POSTGRES_DB", os.getenv("POSTGRES_DB", "mostovik_test") + ), "USER": os.getenv("TEST_POSTGRES_USER", os.getenv("POSTGRES_USER", "postgres")), "PASSWORD": os.getenv( "TEST_POSTGRES_PASSWORD", os.getenv("POSTGRES_PASSWORD", "postgres") ), - "HOST": os.getenv("TEST_POSTGRES_HOST", os.getenv("POSTGRES_HOST", "127.0.0.1")), + "HOST": os.getenv( + "TEST_POSTGRES_HOST", os.getenv("POSTGRES_HOST", "127.0.0.1") + ), "PORT": os.getenv("TEST_POSTGRES_PORT", os.getenv("POSTGRES_PORT", "5432")), "CONN_MAX_AGE": 0, "TEST": { diff --git a/tests/apps/backups/__init__.py b/tests/apps/backups/__init__.py index 8b13789..e69de29 100644 --- a/tests/apps/backups/__init__.py +++ b/tests/apps/backups/__init__.py @@ -1 +0,0 @@ - diff --git a/tests/apps/backups/test_models.py b/tests/apps/backups/test_models.py new file mode 100644 index 0000000..0d7f7f1 --- /dev/null +++ b/tests/apps/backups/test_models.py @@ -0,0 +1,17 @@ +"""Tests for backups models.""" + +from datetime import date + +from apps.backups.models import BackupExportJob +from django.test import TestCase + + +class BackupExportJobModelTest(TestCase): + def test_string_representation(self): + job = BackupExportJob.objects.create( + actual_date=date(2026, 3, 17), + status=BackupExportJob.Status.SUCCESS, + task_id="task-1", + ) + + self.assertEqual(str(job), "Backup 2026-03-17 [success]") diff --git a/tests/apps/backups/test_services.py b/tests/apps/backups/test_services.py new file mode 100644 index 0000000..748b217 --- /dev/null +++ b/tests/apps/backups/test_services.py @@ -0,0 +1,498 @@ +from __future__ import annotations + +import base64 +import json +import struct +import zlib +from datetime import date, datetime +from decimal import Decimal +from io import BytesIO +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch +from uuid import uuid4 +from zipfile import ZipFile + +from apps.backups.models import BackupExportJob +from apps.backups.services import ( + BackupArtifact, + BackupExportError, + BackupExportJobService, + BackupExportService, +) +from apps.parsers.models import FinancialReport, FinancialReportLine +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from django.db import IntegrityError +from django.test import TestCase, override_settings + +from tests.apps.parsers.factories import ( + IndustrialCertificateRecordFactory, + InspectionRecordFactory, + ManufacturerRecordFactory, + ProcurementRecordFactory, +) +from tests.apps.registers.factories import ( + OrganizationFactory, + RegisterFactory, + RegisterUploadFactory, + RegistryMembershipPeriodFactory, +) +from tests.apps.user.factories import UserFactory + + +TEST_BACKUP_KEY = base64.urlsafe_b64encode(b"k" * 32).decode("ascii").rstrip("=") + + +def _decode_backup_payload(bin_bytes: bytes) -> tuple[dict, dict]: + header_size = struct.unpack(">I", bin_bytes[5:9])[0] + header = json.loads(bin_bytes[9 : 9 + header_size].decode("utf-8")) + encrypted_payload = bin_bytes[9 + header_size :] + nonce = header["nonce"] + normalized_nonce = nonce + ("=" * (-len(nonce) % 4)) + raw_nonce = base64.urlsafe_b64decode(normalized_nonce) + payload_bytes = AESGCM(BackupExportService._read_encryption_key()).decrypt( + raw_nonce, + encrypted_payload, + BackupExportService.AAD, + ) + payload = json.loads(zlib.decompress(payload_bytes).decode("utf-8")) + return header, payload + + +@override_settings(BACKUP_ENCRYPTION_KEY=TEST_BACKUP_KEY, BACKUP_KEY_ID="test-key") +class BackupExportServiceTest(TestCase): + def test_build_backup_archive_raises_when_no_active_organizations(self): + with self.assertRaisesMessage( + BackupExportError, + "Нет актуальных организаций для экспорта", + ): + BackupExportService.build_backup_archive(actual_date=date(2026, 3, 1)) + + def test_build_backup_archive_exports_zip_and_payload(self): + registry = RegisterFactory(name="Main registry") + active_upload = RegisterUploadFactory( + registry=registry, + actual_date=date(2026, 3, 1), + ) + organization = OrganizationFactory( + pn_name="Active Org", + mn_ogrn=10_277_001_189_840, + mn_inn=7_702_000_000, + mn_okpo="12345678", + ) + RegistryMembershipPeriodFactory( + registry=registry, + organization=organization, + started_at=date(2026, 3, 1), + started_by_upload=active_upload, + ) + + inactive_org = OrganizationFactory( + pn_name="Inactive Org", + mn_ogrn=10_277_001_189_841, + mn_inn=7_702_000_001, + mn_okpo="87654321", + ) + old_upload = RegisterUploadFactory( + registry=registry, + actual_date=date(2026, 2, 1), + ) + RegistryMembershipPeriodFactory( + registry=registry, + organization=inactive_org, + started_at=date(2026, 2, 1), + ended_at=date(2026, 2, 20), + started_by_upload=old_upload, + ended_by_upload=active_upload, + ) + + IndustrialCertificateRecordFactory( + registry_organization=organization, + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + ) + ManufacturerRecordFactory( + registry_organization=organization, + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + ) + InspectionRecordFactory( + registry_organization=organization, + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + ) + ProcurementRecordFactory( + registry_organization=organization, + customer_inn=str(organization.mn_inn), + customer_ogrn=str(organization.mn_ogrn), + ) + report = FinancialReport.objects.create( + external_id="100500", + ogrn=str(organization.mn_ogrn), + registry_organization=organization, + file_name="fin_100500_10277001189840.xlsx", + file_hash="f" * 64, + load_batch=1, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + FinancialReportLine.objects.create( + report=report, + form_code="1", + line_code="1100", + line_name="Assets", + year=2025, + period_start=100, + period_end=200, + ) + + artifact = BackupExportService.build_backup_archive(actual_date=date(2026, 3, 15)) + + self.assertIsInstance(artifact, BackupArtifact) + self.assertEqual(artifact.organizations_count, 1) + self.assertEqual(artifact.actual_date, date(2026, 3, 15)) + self.assertTrue(artifact.archive_filename.endswith(".zip")) + self.assertTrue(artifact.bin_filename.endswith(".bin")) + self.assertTrue(artifact.checksum_filename.endswith(".sha256")) + + with ZipFile(BytesIO(artifact.archive_bytes)) as archive: + self.assertEqual( + sorted(archive.namelist()), + sorted([artifact.bin_filename, artifact.checksum_filename]), + ) + checksum_content = archive.read(artifact.checksum_filename).decode("utf-8") + bin_bytes = archive.read(artifact.bin_filename) + + self.assertTrue(bin_bytes.startswith(BackupExportService.MAGIC)) + self.assertIn(artifact.checksum_sha256, checksum_content) + self.assertIn(artifact.bin_filename, checksum_content) + + header, payload = _decode_backup_payload(bin_bytes) + self.assertEqual(header["format"], "mostovik-backup-bin") + self.assertEqual(header["key_id"], "test-key") + self.assertEqual(payload["format"], "mostovik-backup-payload") + self.assertEqual(payload["actual_date"], "2026-03-15") + self.assertEqual(payload["organizations_count"], 1) + self.assertEqual(len(payload["data"]["registers.Organization"]), 1) + self.assertEqual(payload["data"]["registers.Organization"][0]["pn_name"], "Active Org") + self.assertEqual(len(payload["data"]["parsers.FinancialReportLine"]), 1) + + def test_normalize_value_supports_scalar_types(self): + random_uuid = uuid4() + + self.assertEqual( + BackupExportService._normalize_value(date(2026, 3, 15)), + "2026-03-15", + ) + self.assertEqual( + BackupExportService._normalize_value(datetime(2026, 3, 15, 12, 30, 0)), + "2026-03-15T12:30:00", + ) + self.assertEqual( + BackupExportService._normalize_value(Decimal("12.34")), + "12.34", + ) + self.assertEqual( + BackupExportService._normalize_value(random_uuid), + str(random_uuid), + ) + self.assertEqual( + BackupExportService._normalize_value(b"payload"), + { + "__type__": "bytes", + "base64": base64.b64encode(b"payload").decode("ascii"), + }, + ) + self.assertEqual(BackupExportService._normalize_value("plain"), "plain") + + @override_settings(BACKUP_ENCRYPTION_KEY="") + def test_read_encryption_key_requires_setting(self): + with self.assertRaisesMessage( + BackupExportError, + "Не задан BACKUP_ENCRYPTION_KEY в настройках", + ): + BackupExportService._read_encryption_key() + + def test_read_encryption_key_rejects_invalid_base64(self): + with patch( + "apps.backups.services.base64.urlsafe_b64decode", + side_effect=ValueError("bad base64"), + ): + with self.assertRaisesMessage( + BackupExportError, + "BACKUP_ENCRYPTION_KEY должен быть base64-url кодированным ключом", + ): + BackupExportService._read_encryption_key() + + @override_settings( + BACKUP_ENCRYPTION_KEY=base64.urlsafe_b64encode(b"short").decode("ascii") + ) + def test_read_encryption_key_requires_32_bytes(self): + with self.assertRaisesMessage( + BackupExportError, + "BACKUP_ENCRYPTION_KEY после декодирования должен быть 32 байта", + ): + BackupExportService._read_encryption_key() + + def test_build_bin_container_rejects_oversized_header(self): + class HugeBytes(bytes): + def __len__(self): + return 2**32 + + class HugeJson(str): + def encode(self, *_args, **_kwargs): + return HugeBytes(b"{}") + + with patch("apps.backups.services.json.dumps", return_value=HugeJson("{}")): + with self.assertRaisesMessage( + BackupExportError, + "Заголовок backup контейнера слишком большой", + ): + BackupExportService._build_bin_container( + encrypted_payload=b"payload", + header_payload={}, + ) + + +class BackupExportJobServiceTest(TestCase): + def test_result_for_existing_job_returns_wait_and_download(self): + today = date(2026, 3, 15) + pending_job = BackupExportJob.objects.create( + actual_date=today, + status=BackupExportJob.Status.PENDING, + task_id="task-pending", + ) + + wait_result = BackupExportJobService._result_for_existing_job( + actual_date=today, + job=pending_job, + ) + + self.assertEqual(wait_result.action, "wait") + self.assertEqual(wait_result.task_id, "task-pending") + + with TemporaryDirectory() as tmp_dir: + archive_path = Path(tmp_dir) / "backup.zip" + archive_path.write_bytes(b"zip") + pending_job.status = BackupExportJob.Status.SUCCESS + pending_job.archive_path = str(archive_path) + pending_job.save(update_fields=["status", "archive_path", "updated_at"]) + + download_result = BackupExportJobService._result_for_existing_job( + actual_date=today, + job=pending_job, + ) + + self.assertEqual(download_result.action, "download") + + def test_enqueue_backup_task_calls_celery(self): + with patch("apps.backups.tasks.generate_backup_for_date.apply_async") as apply_async: + BackupExportJobService._enqueue_backup_task(job_id=5, task_id="task-5") + + apply_async.assert_called_once_with(kwargs={"job_id": 5}, task_id="task-5") + + def test_consume_ready_archive_raises_when_job_missing_or_not_ready(self): + with self.assertRaisesMessage(BackupExportError, "Задача бэкапа не найдена"): + BackupExportJobService.consume_ready_archive(actual_date=date(2026, 3, 20)) + + job = BackupExportJob.objects.create( + actual_date=date(2026, 3, 21), + status=BackupExportJob.Status.STARTED, + task_id="task-started", + ) + + with self.assertRaisesMessage(BackupExportError, "Бэкап еще не готов"): + BackupExportJobService.consume_ready_archive(actual_date=job.actual_date) + + def test_consume_ready_archive_deletes_job_when_file_missing(self): + job = BackupExportJob.objects.create( + actual_date=date(2026, 3, 22), + status=BackupExportJob.Status.SUCCESS, + task_id="task-success", + archive_path="/tmp/does-not-exist.zip", + ) + + with self.assertRaisesMessage( + BackupExportError, + "Файл бэкапа отсутствует, запустите формирование снова", + ): + BackupExportJobService.consume_ready_archive(actual_date=job.actual_date) + + self.assertTrue(BackupExportJob.objects.filter(id=job.id).exists()) + + def test_consume_ready_archive_reads_file_and_uses_path_name_as_fallback(self): + with TemporaryDirectory() as tmp_dir: + archive_path = Path(tmp_dir) / "backup-export.zip" + archive_bytes = b"archive-bytes" + archive_path.write_bytes(archive_bytes) + job = BackupExportJob.objects.create( + actual_date=date(2026, 3, 23), + status=BackupExportJob.Status.SUCCESS, + task_id="task-success-2", + archive_path=str(archive_path), + archive_filename="", + checksum_filename="backup-export.zip.sha256", + organizations_count=3, + ) + + artifact = BackupExportJobService.consume_ready_archive(actual_date=job.actual_date) + + self.assertEqual(artifact.archive_bytes, archive_bytes) + self.assertEqual(artifact.archive_filename, "backup-export.zip") + self.assertEqual(artifact.organizations_count, 3) + self.assertFalse(BackupExportJob.objects.filter(id=job.id).exists()) + + def test_check_or_start_job_replaces_stale_failed_job(self): + today = date(2026, 3, 24) + user = UserFactory.create_user() + with TemporaryDirectory() as tmp_dir: + stale_path = Path(tmp_dir) / "stale.zip" + stale_path.write_bytes(b"stale") + stale_job = BackupExportJob.objects.create( + actual_date=today, + status=BackupExportJob.Status.FAILURE, + task_id="stale-task", + archive_path=str(stale_path), + ) + + with patch("apps.backups.services.uuid.uuid4", return_value="new-task-id"): + with patch.object( + BackupExportJobService, + "_enqueue_backup_task", + ) as enqueue_mock: + with self.captureOnCommitCallbacks(execute=True): + result = BackupExportJobService.check_or_start_job( + actual_date=today, + requested_by_id=user.id, + ) + + self.assertEqual(result.action, "started") + self.assertEqual(result.task_id, "new-task-id") + self.assertFalse(BackupExportJob.objects.filter(id=stale_job.id).exists()) + self.assertFalse(stale_path.exists()) + new_job = BackupExportJob.objects.get(actual_date=today) + self.assertEqual(new_job.task_id, "new-task-id") + enqueue_mock.assert_called_once_with(job_id=new_job.id, task_id="new-task-id") + + def test_check_or_start_job_retries_create_after_integrity_error_with_stale_job(self): + today = date(2026, 3, 26) + user = UserFactory.create_user() + stale_job = BackupExportJob.objects.create( + actual_date=today, + status=BackupExportJob.Status.FAILURE, + task_id="stale-task", + ) + original_create = BackupExportJob.objects.create + + def create_side_effect(*args, **kwargs): + if not hasattr(create_side_effect, "called"): + create_side_effect.called = True + raise IntegrityError("duplicate") + return original_create(*args, **kwargs) + + with patch.object( + BackupExportJobService, + "_get_job_for_update", + side_effect=[None, stale_job], + ): + with patch.object( + BackupExportJobService, + "_result_for_existing_job", + return_value=None, + ): + with patch( + "apps.backups.services.BackupExportJob.objects.create", + side_effect=create_side_effect, + ): + with patch( + "apps.backups.services.uuid.uuid4", + return_value="retry-task-id", + ): + with patch.object( + BackupExportJobService, + "_enqueue_backup_task", + ) as enqueue_mock: + with self.captureOnCommitCallbacks(execute=True): + result = BackupExportJobService.check_or_start_job( + actual_date=today, + requested_by_id=user.id, + ) + + self.assertEqual(result.action, "started") + self.assertEqual(result.task_id, "retry-task-id") + self.assertFalse(BackupExportJob.objects.filter(id=stale_job.id).exists()) + new_job = BackupExportJob.objects.get(actual_date=today) + self.assertEqual(new_job.task_id, "retry-task-id") + enqueue_mock.assert_called_once_with(job_id=new_job.id, task_id="retry-task-id") + + def test_check_or_start_job_retries_create_after_integrity_error_without_concurrent_job(self): + today = date(2026, 3, 28) + user = UserFactory.create_user() + original_create = BackupExportJob.objects.create + + def create_side_effect(*args, **kwargs): + if not hasattr(create_side_effect, "called"): + create_side_effect.called = True + raise IntegrityError("duplicate") + return original_create(*args, **kwargs) + + with patch.object( + BackupExportJobService, + "_get_job_for_update", + side_effect=[None, None], + ): + with patch.object( + BackupExportJobService, + "_result_for_existing_job", + return_value=None, + ): + with patch( + "apps.backups.services.BackupExportJob.objects.create", + side_effect=create_side_effect, + ): + with patch( + "apps.backups.services.uuid.uuid4", + return_value="retry-task-id-2", + ): + with patch.object( + BackupExportJobService, + "_enqueue_backup_task", + ) as enqueue_mock: + with self.captureOnCommitCallbacks(execute=True): + result = BackupExportJobService.check_or_start_job( + actual_date=today, + requested_by_id=user.id, + ) + + self.assertEqual(result.action, "started") + self.assertEqual(result.task_id, "retry-task-id-2") + new_job = BackupExportJob.objects.get(actual_date=today) + self.assertEqual(new_job.task_id, "retry-task-id-2") + enqueue_mock.assert_called_once_with(job_id=new_job.id, task_id="retry-task-id-2") + + def test_archive_exists_and_cleanup_job_artifact(self): + with TemporaryDirectory() as tmp_dir: + archive_path = Path(tmp_dir) / "backup.zip" + archive_path.write_bytes(b"zip") + job = BackupExportJob.objects.create( + actual_date=date(2026, 3, 25), + status=BackupExportJob.Status.SUCCESS, + task_id="task-cleanup", + archive_path=str(archive_path), + ) + + self.assertTrue(BackupExportJobService._archive_exists(job)) + BackupExportJobService._cleanup_job_artifact(job) + self.assertFalse(archive_path.exists()) + + def test_cleanup_job_artifact_is_noop_without_archive_path(self): + job = BackupExportJob.objects.create( + actual_date=date(2026, 3, 27), + status=BackupExportJob.Status.FAILURE, + task_id="task-no-artifact", + archive_path="", + ) + + BackupExportJobService._cleanup_job_artifact(job) + + self.assertTrue(BackupExportJob.objects.filter(id=job.id).exists()) diff --git a/tests/apps/backups/test_tasks.py b/tests/apps/backups/test_tasks.py new file mode 100644 index 0000000..e8d80e7 --- /dev/null +++ b/tests/apps/backups/test_tasks.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from apps.backups.models import BackupExportJob +from apps.backups.services import BackupArtifact +from apps.backups.tasks import _resolve_backup_target_path, generate_backup_for_date +from django.test import TestCase, override_settings + +from tests.apps.user.factories import UserFactory + + +class BackupTasksTest(TestCase): + def test_resolve_backup_target_path_creates_directory_and_renames_existing_file(self): + with TemporaryDirectory() as tmp_dir: + with override_settings(BACKUP_EXPORT_DIRECTORY=tmp_dir): + existing = Path(tmp_dir) / "backup.zip" + existing.write_bytes(b"existing") + + with patch("apps.backups.tasks.uuid.uuid4") as uuid_mock: + uuid_mock.return_value.hex = "deadbeefcafebabe" + target = _resolve_backup_target_path("backup.zip") + + self.assertEqual(target.name, "backup_deadbeef.zip") + + def test_generate_backup_for_date_returns_skipped_when_job_is_missing(self): + generate_backup_for_date.push_request(id="task-missing") + try: + result = generate_backup_for_date.run(job_id=999999) + finally: + generate_backup_for_date.pop_request() + + self.assertEqual(result, {"status": "skipped", "reason": "job_not_found"}) + + def test_generate_backup_for_date_builds_archive_and_updates_job(self): + user = UserFactory.create_user() + job = BackupExportJob.objects.create( + actual_date=user.date_joined.date(), + requested_by=user, + status=BackupExportJob.Status.PENDING, + ) + background_job = MagicMock() + artifact = BackupArtifact( + archive_bytes=b"zip-bytes", + archive_filename="backup.zip", + bin_filename="backup.bin", + checksum_filename="backup.zip.sha256", + checksum_sha256="a" * 64, + organizations_count=5, + actual_date=job.actual_date, + ) + + with TemporaryDirectory() as tmp_dir: + with override_settings(BACKUP_EXPORT_DIRECTORY=tmp_dir): + generate_backup_for_date.push_request(id="task-success") + try: + with patch( + "apps.backups.tasks.BackgroundJobService.get_by_task_id_or_none", + return_value=None, + ): + with patch( + "apps.backups.tasks.BackgroundJobService.create_job", + return_value=background_job, + ) as create_job_mock: + with patch( + "apps.backups.tasks.BackupExportService.build_backup_archive", + return_value=artifact, + ) as build_mock: + result = generate_backup_for_date.run(job_id=job.id) + finally: + generate_backup_for_date.pop_request() + + job.refresh_from_db() + self.assertEqual(job.status, BackupExportJob.Status.SUCCESS) + self.assertEqual(job.task_id, "task-success") + self.assertEqual(job.archive_filename, "backup.zip") + self.assertEqual(job.checksum_filename, "backup.zip.sha256") + self.assertEqual(job.organizations_count, 5) + self.assertTrue(Path(job.archive_path).is_file()) + + self.assertEqual(result["status"], "success") + self.assertEqual(result["archive_filename"], "backup.zip") + create_job_mock.assert_called_once() + build_mock.assert_called_once_with(actual_date=job.actual_date) + background_job.mark_started.assert_called_once_with() + background_job.update_progress.assert_any_call(10, "Подготовка backup-данных") + background_job.update_progress.assert_any_call(70, "Запись архива на диск") + background_job.complete.assert_called_once_with(result=result) + + def test_generate_backup_for_date_marks_failure(self): + user = UserFactory.create_user() + job = BackupExportJob.objects.create( + actual_date=user.date_joined.date(), + requested_by=user, + status=BackupExportJob.Status.PENDING, + ) + background_job = MagicMock() + + generate_backup_for_date.push_request(id="task-failure") + try: + with patch( + "apps.backups.tasks.BackgroundJobService.get_by_task_id_or_none", + return_value=background_job, + ): + with patch( + "apps.backups.tasks.BackupExportService.build_backup_archive", + side_effect=RuntimeError("boom"), + ): + with patch("apps.backups.tasks.logger.exception") as logger_mock: + with self.assertRaisesMessage(RuntimeError, "boom"): + generate_backup_for_date.run(job_id=job.id) + finally: + generate_backup_for_date.pop_request() + + logger_mock.assert_called_once() + background_job.fail.assert_called_once_with(error="boom") + job.refresh_from_db() + self.assertEqual(job.status, BackupExportJob.Status.FAILURE) + self.assertEqual(job.error, "boom") diff --git a/tests/apps/backups/test_views.py b/tests/apps/backups/test_views.py index 41769b0..00e2fb8 100644 --- a/tests/apps/backups/test_views.py +++ b/tests/apps/backups/test_views.py @@ -5,10 +5,14 @@ from __future__ import annotations import hashlib from pathlib import Path from tempfile import TemporaryDirectory -from unittest.mock import Mock, patch +from unittest.mock import patch from apps.backups.models import BackupExportJob -from apps.backups.services import BackupExportJobService +from apps.backups.services import ( + BackupExportError, + BackupExportJobService, + BackupRequestResult, +) from django.db import IntegrityError from django.urls import reverse from django.utils import timezone @@ -34,29 +38,27 @@ class BackupExportViewTest(APITestCase): response = self.client.post(self.url, {}, format="json") self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - @patch("apps.backups.tasks.generate_backup_for_date.delay") - def test_export_starts_job_when_absent(self, delay_mock): - delay_mock.return_value = Mock(id="task-backup-1") - + @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") + def test_export_starts_job_when_absent(self, enqueue_mock): self.client.force_authenticate(self.admin) today = timezone.localdate() - response = self.client.post( - self.url, - {"actual_date": today.isoformat()}, - format="json", - ) + with self.captureOnCommitCallbacks(execute=True): + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(response.data["status"], "started") - self.assertEqual(response.data["task_id"], "task-backup-1") job = BackupExportJob.objects.get(actual_date=today) self.assertEqual(job.status, BackupExportJob.Status.PENDING) - self.assertEqual(job.task_id, "task-backup-1") - delay_mock.assert_called_once_with(job_id=job.id) + self.assertEqual(response.data["task_id"], job.task_id) + enqueue_mock.assert_called_once_with(job_id=job.id, task_id=job.task_id) - @patch("apps.backups.tasks.generate_backup_for_date.delay") - def test_export_returns_wait_when_job_in_progress(self, delay_mock): + @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") + def test_export_returns_wait_when_job_in_progress(self, enqueue_mock): today = timezone.localdate() BackupExportJob.objects.create( actual_date=today, @@ -74,10 +76,10 @@ class BackupExportViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(response.data["status"], "wait") self.assertEqual(response.data["task_id"], "task-running-1") - delay_mock.assert_not_called() + enqueue_mock.assert_not_called() - @patch("apps.backups.tasks.generate_backup_for_date.delay") - def test_export_returns_file_and_deletes_after_download(self, delay_mock): + @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") + def test_export_returns_file_and_deletes_after_download(self, enqueue_mock): with TemporaryDirectory() as tmp_dir: tmp_path = Path(tmp_dir) archive_bytes = b"zip-content" @@ -106,7 +108,9 @@ class BackupExportViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.content, archive_bytes) self.assertEqual(response["Content-Type"], "application/zip") - self.assertIn('attachment; filename="backup.zip"', response["Content-Disposition"]) + self.assertIn( + 'attachment; filename="backup.zip"', response["Content-Disposition"] + ) self.assertEqual(response["X-Backup-Organizations"], "7") self.assertEqual( response["X-Backup-SHA256"], @@ -115,11 +119,10 @@ class BackupExportViewTest(APITestCase): self.assertFalse(archive_path.exists()) self.assertFalse(BackupExportJob.objects.filter(id=job.id).exists()) - delay_mock.assert_not_called() + enqueue_mock.assert_not_called() - @patch("apps.backups.tasks.generate_backup_for_date.delay") - def test_export_restarts_when_success_job_has_no_file(self, delay_mock): - delay_mock.return_value = Mock(id="task-backup-retry") + @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") + def test_export_restarts_when_success_job_has_no_file(self, enqueue_mock): today = timezone.localdate() with TemporaryDirectory() as tmp_dir: missing_archive_path = Path(tmp_dir) / "non-existent-backup.zip" @@ -132,16 +135,21 @@ class BackupExportViewTest(APITestCase): ) self.client.force_authenticate(self.admin) - response = self.client.post( - self.url, - {"actual_date": today.isoformat()}, - format="json", - ) + with self.captureOnCommitCallbacks(execute=True): + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(response.data["status"], "started") - self.assertEqual(response.data["task_id"], "task-backup-retry") - delay_mock.assert_called_once() + restarted_job = BackupExportJob.objects.get(actual_date=today) + self.assertEqual(response.data["task_id"], restarted_job.task_id) + enqueue_mock.assert_called_once_with( + job_id=restarted_job.id, + task_id=restarted_job.task_id, + ) @patch("apps.backups.services.BackupExportJob.objects.create") def test_check_or_start_job_handles_integrity_race(self, create_mock): @@ -166,3 +174,64 @@ class BackupExportViewTest(APITestCase): self.assertEqual(result.action, "wait") self.assertEqual(result.task_id, "task-race-running") + + @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") + def test_check_or_start_job_enqueues_on_commit(self, enqueue_mock): + today = timezone.localdate() + + with self.captureOnCommitCallbacks(execute=False) as callbacks: + result = BackupExportJobService.check_or_start_job( + actual_date=today, + requested_by_id=self.admin.id, + ) + + self.assertEqual(result.action, "started") + self.assertEqual(len(callbacks), 1) + enqueue_mock.assert_not_called() + + job = BackupExportJob.objects.get(actual_date=today) + self.assertEqual(job.task_id, result.task_id) + + callbacks[0]() + enqueue_mock.assert_called_once_with(job_id=job.id, task_id=job.task_id) + + @patch("apps.backups.views.BackupExportJobService.check_or_start_job") + def test_export_returns_400_when_job_start_fails(self, check_or_start_mock): + self.client.force_authenticate(self.admin) + check_or_start_mock.side_effect = BackupExportError("broken") + + response = self.client.post(self.url, {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.data["errors"][0]["details"]["fields"]["backup"], ["broken"] + ) + + @patch("apps.backups.views.BackupExportJobService.consume_ready_archive") + @patch("apps.backups.views.BackupExportJobService.check_or_start_job") + def test_export_returns_400_when_archive_consumption_fails( + self, + check_or_start_mock, + consume_mock, + ): + today = timezone.localdate() + self.client.force_authenticate(self.admin) + check_or_start_mock.return_value = BackupRequestResult( + action="download", + message="ready", + actual_date=today, + task_id="task-ready", + ) + consume_mock.side_effect = BackupExportError("missing archive") + + response = self.client.post( + self.url, + {"actual_date": today.isoformat()}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.data["errors"][0]["details"]["fields"]["backup"], + ["missing archive"], + ) diff --git a/tests/apps/core/test_celery_module.py b/tests/apps/core/test_celery_module.py new file mode 100644 index 0000000..a09b22c --- /dev/null +++ b/tests/apps/core/test_celery_module.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + + +CELERY_MODULE_PATH = ( + Path(__file__).resolve().parents[3] / "src" / "core" / "celery.py" +) + + +def _load_module(module_name: str): + spec = importlib.util.spec_from_file_location(module_name, CELERY_MODULE_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class CeleryModuleTest(SimpleTestCase): + def test_import_requires_django_settings_module(self): + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesMessage( + RuntimeError, + "DJANGO_SETTINGS_MODULE is not set.", + ): + _load_module("isolated_core_celery_missing") + + def test_import_runs_startup_checks_for_worker_runtime(self): + app_mock = MagicMock() + app_mock.conf = SimpleNamespace() + + with patch.dict(os.environ, {"DJANGO_SETTINGS_MODULE": "settings.test"}, clear=True): + with patch.object(sys, "argv", ["celery", "-A", "project", "worker"]): + with patch("apps.core.startup_checks.run_startup_checks") as checks_mock: + with patch("celery.Celery", return_value=app_mock): + module = _load_module("isolated_core_celery_worker") + + checks_mock.assert_called_once_with(component="celery") + app_mock.config_from_object.assert_called_once_with( + "django.conf:settings", + namespace="CELERY", + ) + app_mock.autodiscover_tasks.assert_called_once_with() + self.assertEqual(module.app, app_mock) + + def test_debug_task_prints_request(self): + with patch.dict(os.environ, {"DJANGO_SETTINGS_MODULE": "settings.test"}, clear=True): + with patch.object(sys, "argv", ["python", "manage.py", "shell"]): + module = _load_module("isolated_core_celery_debug") + + with patch("builtins.print") as print_mock: + module.debug_task.run() + + print_mock.assert_called_once() diff --git a/tests/apps/core/test_exception_handler.py b/tests/apps/core/test_exception_handler.py index f0174a1..d23b8a6 100644 --- a/tests/apps/core/test_exception_handler.py +++ b/tests/apps/core/test_exception_handler.py @@ -51,6 +51,12 @@ class CustomExceptionHandlerTest(SimpleTestCase): self.assertEqual(response.status_code, 400) self.assertEqual(len(response.data["errors"]), 2) + def test_validation_error_scalar_message(self): + exc = ValidationError("plain error") + response = custom_exception_handler(exc, self._context()) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data["errors"][0]["message"], "plain error") + def test_unhandled_exception(self): response = custom_exception_handler(RuntimeError("boom"), self._context()) self.assertEqual(response.status_code, 500) diff --git a/tests/apps/core/test_filters.py b/tests/apps/core/test_filters.py index 609761f..b2a1706 100644 --- a/tests/apps/core/test_filters.py +++ b/tests/apps/core/test_filters.py @@ -126,3 +126,13 @@ class FilterMixinTest(TestCase): self.assertIn(filters.DjangoFilterBackend, backends) self.assertIn(StandardSearchFilter, backends) self.assertIn(StandardOrderingFilter, backends) + + def test_get_queryset_returns_super_queryset(self): + class Parent: + def get_queryset(self): + return ["ok"] + + class DummyFilterMixin(FilterMixin, Parent): + pass + + self.assertEqual(DummyFilterMixin().get_queryset(), ["ok"]) diff --git a/tests/apps/core/test_management_commands.py b/tests/apps/core/test_management_commands.py index 73a0dc0..14eb083 100644 --- a/tests/apps/core/test_management_commands.py +++ b/tests/apps/core/test_management_commands.py @@ -156,8 +156,7 @@ class BaseAppCommandTest(TestCase): cmd.silent = True def generator(): - for idx in range(3): - yield idx + yield from range(3) result = list(cmd.progress_iter(generator(), "Iter")) self.assertEqual(result, [0, 1, 2]) diff --git a/tests/apps/core/test_mixins.py b/tests/apps/core/test_mixins.py index 6afa107..9621c03 100644 --- a/tests/apps/core/test_mixins.py +++ b/tests/apps/core/test_mixins.py @@ -130,6 +130,9 @@ class MixinsBehaviorTest(TransactionTestCase): ): name = models.CharField(max_length=50) + def __str__(self): + return self.name + class Meta: app_label = "core" diff --git a/tests/apps/core/test_pagination.py b/tests/apps/core/test_pagination.py new file mode 100644 index 0000000..75440c3 --- /dev/null +++ b/tests/apps/core/test_pagination.py @@ -0,0 +1,22 @@ +"""Tests for core pagination classes.""" + +from apps.core.pagination import StandardCursorPagination, StandardPagination +from django.test import SimpleTestCase + + +class PaginationSchemaTest(SimpleTestCase): + def test_standard_pagination_schema(self): + schema = StandardPagination().get_paginated_response_schema({"type": "array"}) + + self.assertEqual(schema["type"], "object") + self.assertEqual(schema["properties"]["data"]["type"], "array") + + def test_cursor_pagination_schema(self): + schema = StandardCursorPagination().get_paginated_response_schema( + {"type": "array"} + ) + + self.assertEqual(schema["type"], "object") + pagination = schema["properties"]["meta"]["properties"]["pagination"]["properties"] + self.assertIn("next_cursor", pagination) + self.assertIn("previous_cursor", pagination) diff --git a/tests/apps/core/test_permissions.py b/tests/apps/core/test_permissions.py index 10d7dea..dbb90b3 100644 --- a/tests/apps/core/test_permissions.py +++ b/tests/apps/core/test_permissions.py @@ -101,6 +101,14 @@ class IsOwnerOrReadOnlyTest(TestCase): result = self.permission.has_object_permission(request, APIView(), obj) self.assertTrue(result) + def test_unsafe_methods_use_owner_fallback_field(self): + request = self.factory.patch("/") + request.user = self.user + obj = MockObject(user=None, owner=self.user) + + result = self.permission.has_object_permission(request, APIView(), obj) + self.assertTrue(result) + class IsAdminOrReadOnlyTest(TestCase): """Tests for IsAdminOrReadOnly permission""" @@ -250,3 +258,11 @@ class IsOwnerOrAdminTest(TestCase): result = self.permission.has_object_permission(request, APIView(), obj) self.assertFalse(result) + + def test_owner_fallback_field_is_used_for_non_admin(self): + request = self.factory.get("/") + request.user = self.user + obj = MockObject(user=None, owner=self.user) + + result = self.permission.has_object_permission(request, APIView(), obj) + self.assertTrue(result) diff --git a/tests/apps/core/test_signals.py b/tests/apps/core/test_signals.py index 5514571..c0a1834 100644 --- a/tests/apps/core/test_signals.py +++ b/tests/apps/core/test_signals.py @@ -17,6 +17,10 @@ from django.test import TestCase from tests.utils.fixtures import fake +def _password() -> str: + return fake.password(length=12, special_chars=False) + + class SignalDispatcherTest(TestCase): def test_register_connect_disconnect(self): dispatcher = SignalDispatcher() @@ -34,7 +38,7 @@ class SignalDispatcherTest(TestCase): dispatcher.connect_all() user = get_user_model().objects.create_user( - email=fake.email(), username=fake.user_name(), password="pass" + email=fake.email(), username=fake.user_name(), password=_password() ) self.assertIn(user.pk, events) @@ -81,7 +85,7 @@ class SignalDispatcherTest(TestCase): signal_dispatcher.connect_all() user = get_user_model().objects.create_user( - email=fake.email(), username=fake.user_name(), password="pass" + email=fake.email(), username=fake.user_name(), password=_password() ) self.assertIn(user.pk, events) self.assertTrue(signal_dispatcher.list_handlers()) diff --git a/tests/apps/core/test_startup_checks.py b/tests/apps/core/test_startup_checks.py new file mode 100644 index 0000000..6a7026d --- /dev/null +++ b/tests/apps/core/test_startup_checks.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import importlib +import os +import sys +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from apps.core import startup_checks +from django.test import SimpleTestCase, override_settings + + +TEST_DATABASES = { + "default": { + "NAME": "mostovik", + "USER": "postgres", + "PASSWORD": "secret", + "HOST": "db.example.test", + "PORT": 5432, + "OPTIONS": {"sslmode": "require"}, + } +} + +TEST_CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": "redis://redis.example.test:6379/1", + } +} + + +class StartupChecksTest(SimpleTestCase): + @override_settings(DATABASES=TEST_DATABASES) + @patch("apps.core.startup_checks.psycopg2.connect") + def test_check_db_success(self, connect_mock): + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + connect_mock.return_value = connection + + success, message = startup_checks._check_db(7) + + self.assertTrue(success) + self.assertEqual(message, "OK") + connect_mock.assert_called_once_with( + dbname="mostovik", + user="postgres", + password="secret", + host="db.example.test", + port=5432, + connect_timeout=7, + sslmode="require", + ) + cursor.execute.assert_called_once_with("SELECT 1") + cursor.fetchone.assert_called_once_with() + connection.close.assert_called_once_with() + + @override_settings(DATABASES=TEST_DATABASES) + @patch( + "apps.core.startup_checks.psycopg2.connect", + side_effect=RuntimeError("database down"), + ) + def test_check_db_failure(self, connect_mock): + success, message = startup_checks._check_db(5) + + self.assertFalse(success) + self.assertIn("db.example.test:5432/mostovik", message) + self.assertIn("database down", message) + connect_mock.assert_called_once() + + @override_settings(CACHES=TEST_CACHES) + @patch("apps.core.startup_checks.redis.Redis.from_url") + def test_check_redis_success(self, from_url_mock): + client = MagicMock() + from_url_mock.return_value = client + + success, message = startup_checks._check_redis(4) + + self.assertTrue(success) + self.assertEqual(message, "OK") + from_url_mock.assert_called_once_with( + "redis://redis.example.test:6379/1", + socket_connect_timeout=4, + socket_timeout=4, + ) + client.ping.assert_called_once_with() + + @override_settings(CACHES=TEST_CACHES) + @patch( + "apps.core.startup_checks.redis.Redis.from_url", + side_effect=RuntimeError("redis down"), + ) + def test_check_redis_failure(self, from_url_mock): + success, message = startup_checks._check_redis(6) + + self.assertFalse(success) + self.assertIn("redis.example.test:6379/1", message) + self.assertIn("redis down", message) + from_url_mock.assert_called_once() + + @override_settings(STARTUP_CHECKS_ENABLED=False) + @patch("apps.core.startup_checks._check_db") + @patch("apps.core.startup_checks._check_redis") + def test_run_startup_checks_skips_when_disabled( + self, + redis_mock, + db_mock, + ): + startup_checks.run_startup_checks(component="web") + + db_mock.assert_not_called() + redis_mock.assert_not_called() + + @override_settings( + STARTUP_CHECKS_ENABLED=True, + STARTUP_DB_TIMEOUT_SECONDS=9, + STARTUP_REDIS_TIMEOUT_SECONDS=11, + ) + @patch("apps.core.startup_checks._check_db", return_value=(True, "OK")) + @patch("apps.core.startup_checks._check_redis", return_value=(True, "OK")) + def test_run_startup_checks_success(self, redis_mock, db_mock): + startup_checks.run_startup_checks(component="worker") + + db_mock.assert_called_once_with(9) + redis_mock.assert_called_once_with(11) + + @override_settings(STARTUP_CHECKS_ENABLED=True, STARTUP_DB_TIMEOUT_SECONDS=8) + @patch("apps.core.startup_checks._check_db", return_value=(False, "db failed")) + @patch("apps.core.startup_checks._log") + def test_run_startup_checks_exits_on_db_failure(self, log_mock, db_mock): + with self.assertRaises(SystemExit) as error: + startup_checks.run_startup_checks(component="wsgi") + + self.assertEqual(error.exception.code, 1) + db_mock.assert_called_once_with(8) + log_mock.assert_called_once() + self.assertIn("[startup:wsgi] DB check failed", log_mock.call_args.args[0]) + self.assertIn("db failed", log_mock.call_args.args[0]) + + @override_settings( + STARTUP_CHECKS_ENABLED=True, + STARTUP_DB_TIMEOUT_SECONDS=3, + STARTUP_REDIS_TIMEOUT_SECONDS=12, + ) + @patch("apps.core.startup_checks._check_db", return_value=(True, "OK")) + @patch("apps.core.startup_checks._check_redis", return_value=(False, "redis failed")) + @patch("apps.core.startup_checks._log") + def test_run_startup_checks_exits_on_redis_failure( + self, + log_mock, + redis_mock, + db_mock, + ): + with self.assertRaises(SystemExit) as error: + startup_checks.run_startup_checks(component="asgi") + + self.assertEqual(error.exception.code, 1) + db_mock.assert_called_once_with(3) + redis_mock.assert_called_once_with(12) + self.assertIn("[startup:asgi] Redis check failed", log_mock.call_args.args[0]) + self.assertIn("redis failed", log_mock.call_args.args[0]) + + +class EntryPointImportTest(SimpleTestCase): + def _import_fresh(self, module_name: str): + sys.modules.pop("core", None) + sys.modules.pop(module_name, None) + return importlib.import_module(module_name) + + def test_import_core_asgi_runs_startup_checks_and_sets_default_settings(self): + sentinel_application = object() + celery_stub = SimpleNamespace(app=object()) + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DJANGO_SETTINGS_MODULE", None) + with patch.dict(sys.modules, {"core.celery": celery_stub}): + with patch("apps.core.startup_checks.run_startup_checks") as checks_mock: + with patch( + "django.core.asgi.get_asgi_application", + return_value=sentinel_application, + ): + module = self._import_fresh("core.asgi") + + self.assertEqual(os.environ["DJANGO_SETTINGS_MODULE"], "settings.test") + checks_mock.assert_called_once_with(component="asgi") + self.assertIs(module.application, sentinel_application) + sys.modules.pop("core.asgi", None) + + def test_import_core_wsgi_runs_startup_checks_and_sets_default_settings(self): + sentinel_application = object() + celery_stub = SimpleNamespace(app=object()) + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DJANGO_SETTINGS_MODULE", None) + with patch.dict(sys.modules, {"core.celery": celery_stub}): + with patch("apps.core.startup_checks.run_startup_checks") as checks_mock: + with patch( + "django.core.wsgi.get_wsgi_application", + return_value=sentinel_application, + ): + module = self._import_fresh("core.wsgi") + + self.assertEqual(os.environ["DJANGO_SETTINGS_MODULE"], "settings.test") + checks_mock.assert_called_once_with(component="wsgi") + self.assertIs(module.application, sentinel_application) + sys.modules.pop("core.wsgi", None) diff --git a/tests/apps/core/test_views.py b/tests/apps/core/test_views.py index ee94104..26e97e6 100644 --- a/tests/apps/core/test_views.py +++ b/tests/apps/core/test_views.py @@ -352,6 +352,13 @@ class BackgroundJobsViewTest(APITestCase): response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_200_OK) + def test_job_status_forbidden_for_unowned_job(self): + job = self._create_job(task_id="job-unowned", user_id=None, status="success") + self.client.force_authenticate(self.other) + url = reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + def test_job_list_filters_status(self): self._create_job(task_id="job-1", user_id=self.user.id, status="success") self._create_job(task_id="job-2", user_id=self.user.id, status="pending") @@ -375,7 +382,9 @@ class BackgroundJobsViewTest(APITestCase): self.assertLessEqual(len(response.data), 2) def test_job_list_invalid_limit_returns_400(self): - self._create_job(task_id="job-invalid-limit", user_id=self.user.id, status="success") + self._create_job( + task_id="job-invalid-limit", user_id=self.user.id, status="success" + ) self.client.force_authenticate(self.user) url = reverse("api_v1:jobs:job-list") response = self.client.get(url, {"limit": "abc"}) diff --git a/tests/apps/exchange/test_models.py b/tests/apps/exchange/test_models.py new file mode 100644 index 0000000..37037b6 --- /dev/null +++ b/tests/apps/exchange/test_models.py @@ -0,0 +1,27 @@ +"""Tests for exchange models.""" + +from apps.exchange.models import ExchangeConnection +from django.test import TestCase + +from tests.apps.exchange.factories import ExchangeConnectionFactory + + +class ExchangeConnectionModelTest(TestCase): + def test_string_representation_and_plain_password_passthrough(self): + connection = ExchangeConnectionFactory( + username="postgres", + server="127.0.0.1", + port=5432, + database_name="target_db", + schema_name="public", + ) + + self.assertEqual(str(connection), "postgres@127.0.0.1:5432/target_db[public]") + self.assertEqual(ExchangeConnection.decrypt_password("legacy-pass"), "legacy-pass") + + def test_decrypt_password_raises_for_invalid_encrypted_token(self): + with self.assertRaisesMessage( + ValueError, + "Не удалось расшифровать пароль exchange connection", + ): + ExchangeConnection.decrypt_password(f"{ExchangeConnection.PASSWORD_PREFIX}invalid") diff --git a/tests/apps/exchange/test_serializers.py b/tests/apps/exchange/test_serializers.py new file mode 100644 index 0000000..03ccd66 --- /dev/null +++ b/tests/apps/exchange/test_serializers.py @@ -0,0 +1,29 @@ +"""Tests for exchange serializers.""" + +from apps.exchange.serializers import ExchangeCopyRequestSerializer +from django.test import SimpleTestCase + + +class ExchangeCopyRequestSerializerTest(SimpleTestCase): + def test_single_mode_requires_table(self): + serializer = ExchangeCopyRequestSerializer(data={"mode": "single"}) + self.assertFalse(serializer.is_valid()) + self.assertIn("table", serializer.errors) + + def test_selected_mode_requires_tables(self): + serializer = ExchangeCopyRequestSerializer(data={"mode": "selected"}) + self.assertFalse(serializer.is_valid()) + self.assertIn("tables", serializer.errors) + + def test_table_and_tables_are_rejected_for_wrong_modes(self): + serializer_with_table = ExchangeCopyRequestSerializer( + data={"mode": "all", "table": "parsers_manufacturer"} + ) + self.assertFalse(serializer_with_table.is_valid()) + self.assertIn("table", serializer_with_table.errors) + + serializer_with_tables = ExchangeCopyRequestSerializer( + data={"mode": "all", "tables": ["parsers_manufacturer"]} + ) + self.assertFalse(serializer_with_tables.is_valid()) + self.assertIn("tables", serializer_with_tables.errors) diff --git a/tests/apps/exchange/test_service_units.py b/tests/apps/exchange/test_service_units.py new file mode 100644 index 0000000..25ee333 --- /dev/null +++ b/tests/apps/exchange/test_service_units.py @@ -0,0 +1,546 @@ +from __future__ import annotations + +from contextlib import suppress +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from apps.exchange.services import ExchangeConnectionService, ExchangeServiceError +from apps.parsers.models import IndustrialCertificateRecord, ManufacturerRecord, ParserLoadLog +from django.test import TestCase + +from tests.apps.exchange.factories import ExchangeConnectionFactory + + +class _FakeModel: + _meta = SimpleNamespace( + app_label="tests", + model_name="fake_model", + db_table="fake_table", + local_fields=[ + SimpleNamespace(attname="id", name="id", column="id"), + SimpleNamespace(attname="name", name="name", column="name"), + ], + pk=SimpleNamespace(attname="id"), + ) + objects = MagicMock() + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class _AnotherFakeModel: + _meta = SimpleNamespace( + app_label="tests", + model_name="another_model", + db_table="another_table", + local_fields=[SimpleNamespace(attname="id", name="id", column="id")], + pk=SimpleNamespace(attname="id"), + ) + + +class ExchangeConnectionServiceUnitTest(TestCase): + def test_create_active_connection_and_prepare_updates_last_check_on_success(self): + with patch.object( + ExchangeConnectionService, + "test_connection", + return_value="target_alias", + ) as test_connection_mock: + with patch.object( + ExchangeConnectionService, + "validate_target_structure", + ) as validate_mock: + connection = ExchangeConnectionService.create_active_connection_and_prepare( + server="127.0.0.1", + port=5432, + username="postgres", + password="secret", + database_name="target_db", + schema_name="public", + ) + + self.assertTrue(connection.is_active) + self.assertIsNotNone(connection.last_checked_at) + self.assertEqual(connection.last_error, "") + test_connection_mock.assert_called_once_with(connection) + validate_mock.assert_called_once_with( + connection=connection, + alias="target_alias", + schema_name="public", + ) + + def test_get_active_connection_raises_when_missing(self): + with self.assertRaisesMessage( + ExchangeServiceError, + "Активное подключение не найдено", + ): + ExchangeConnectionService.get_active_connection() + + def test_test_connection_success_runs_select(self): + connection = ExchangeConnectionFactory() + db_connection = MagicMock() + cursor = MagicMock() + db_connection.cursor.return_value.__enter__.return_value = cursor + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch.object( + ExchangeConnectionService, + "_configure_alias", + return_value="exchange_target_1", + ): + with patch("apps.exchange.services.connections", connections_mock): + alias = ExchangeConnectionService.test_connection(connection) + + self.assertEqual(alias, "exchange_target_1") + db_connection.ensure_connection.assert_called_once_with() + cursor.execute.assert_called_once_with("SELECT 1") + + def test_test_connection_marks_error_on_failure(self): + connection = ExchangeConnectionFactory(last_error="") + db_connection = MagicMock() + db_connection.ensure_connection.side_effect = RuntimeError("boom") + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch.object( + ExchangeConnectionService, + "_configure_alias", + return_value="exchange_target_1", + ): + with patch("apps.exchange.services.connections", connections_mock): + with self.assertRaisesMessage( + ExchangeServiceError, + "Ошибка подключения к целевой БД: boom", + ): + ExchangeConnectionService.test_connection(connection) + + connection.refresh_from_db() + self.assertEqual(connection.last_error, "boom") + self.assertIsNotNone(connection.last_checked_at) + + def test_validate_target_structure_calls_all_validation_steps(self): + connection = ExchangeConnectionFactory() + db_connection = MagicMock() + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with patch.object( + ExchangeConnectionService, + "_extend_models_with_dependencies", + return_value=[_FakeModel], + ) as extend_mock: + with patch.object( + ExchangeConnectionService, + "_get_parser_models", + return_value=[_FakeModel], + ): + with patch.object( + ExchangeConnectionService, + "_validate_schema_exists", + ) as schema_mock: + with patch.object( + ExchangeConnectionService, + "_validate_tables_exist", + ) as tables_mock: + with patch.object( + ExchangeConnectionService, + "_validate_columns_exist", + ) as columns_mock: + ExchangeConnectionService.validate_target_structure( + connection=connection, + alias="target_alias", + schema_name="public", + ) + + db_connection.ensure_connection.assert_called_once_with() + extend_mock.assert_called_once() + schema_mock.assert_called_once_with(alias="target_alias", schema_name="public") + tables_mock.assert_called_once_with( + alias="target_alias", + schema_name="public", + models_to_copy=[_FakeModel], + ) + columns_mock.assert_called_once_with( + alias="target_alias", + schema_name="public", + models_to_copy=[_FakeModel], + ) + + def test_validate_target_structure_marks_and_reraises_exchange_error(self): + connection = ExchangeConnectionFactory(last_error="") + db_connection = MagicMock() + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with patch.object( + ExchangeConnectionService, + "_validate_schema_exists", + side_effect=ExchangeServiceError("bad schema"), + ): + with self.assertRaisesMessage(ExchangeServiceError, "bad schema"): + ExchangeConnectionService.validate_target_structure( + connection=connection, + alias="target_alias", + schema_name="public", + models_to_copy=[_FakeModel], + ) + + connection.refresh_from_db() + self.assertEqual(connection.last_error, "bad schema") + + def test_validate_target_structure_wraps_generic_error(self): + connection = ExchangeConnectionFactory(last_error="") + db_connection = MagicMock() + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with patch.object( + ExchangeConnectionService, + "_validate_schema_exists", + side_effect=RuntimeError("unexpected"), + ): + with self.assertRaisesMessage( + ExchangeServiceError, + "Ошибка проверки структуры целевой БД: unexpected", + ): + ExchangeConnectionService.validate_target_structure( + connection=connection, + alias="target_alias", + schema_name="public", + models_to_copy=[_FakeModel], + ) + + connection.refresh_from_db() + self.assertEqual(connection.last_error, "unexpected") + + def test_copy_parsers_data_success(self): + connection = ExchangeConnectionFactory(schema_name="target_schema") + db_connection = MagicMock() + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with patch.object( + ExchangeConnectionService, + "_configure_alias", + return_value="target_alias", + ): + with patch.object( + ExchangeConnectionService, + "_resolve_models", + return_value=[_FakeModel, _AnotherFakeModel], + ): + with patch.object( + ExchangeConnectionService, + "_extend_models_with_dependencies", + return_value=[_FakeModel, _AnotherFakeModel], + ): + with patch.object( + ExchangeConnectionService, + "validate_target_structure", + ) as validate_mock: + with patch.object( + ExchangeConnectionService, + "_copy_model_data", + side_effect=[2, 3], + ) as copy_mock: + result = ExchangeConnectionService.copy_parsers_data( + connection=connection, + mode="selected", + tables=["fake_table", "another_table"], + truncate_before_copy=False, + ) + + self.assertEqual(result["mode"], "selected") + self.assertEqual(result["tables"], ["fake_table", "another_table"]) + self.assertEqual(result["rows_by_table"], {"fake_table": 2, "another_table": 3}) + self.assertEqual(result["total_rows"], 5) + self.assertFalse(result["truncate_before_copy"]) + validate_mock.assert_called_once_with( + connection=connection, + alias="target_alias", + schema_name="target_schema", + models_to_copy=[_FakeModel, _AnotherFakeModel], + ) + self.assertEqual(copy_mock.call_count, 2) + connection.refresh_from_db() + self.assertEqual(connection.last_error, "") + self.assertIsNotNone(connection.last_checked_at) + + def test_copy_parsers_data_marks_connection_error_on_connect_failure(self): + connection = ExchangeConnectionFactory(last_error="") + db_connection = MagicMock() + db_connection.ensure_connection.side_effect = RuntimeError("target unavailable") + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with patch.object( + ExchangeConnectionService, + "_configure_alias", + return_value="target_alias", + ): + with patch.object( + ExchangeConnectionService, + "_resolve_models", + return_value=[_FakeModel], + ): + with patch.object( + ExchangeConnectionService, + "_extend_models_with_dependencies", + return_value=[_FakeModel], + ): + with self.assertRaisesMessage( + ExchangeServiceError, + "Ошибка подключения к целевой БД: target unavailable", + ): + ExchangeConnectionService.copy_parsers_data( + connection=connection, + mode="all", + ) + + connection.refresh_from_db() + self.assertEqual(connection.last_error, "target unavailable") + + def test_configure_alias_closes_existing_connection_and_clears_cache(self): + connection = ExchangeConnectionFactory(password="secret") + alias = f"exchange_target_{connection.id}" + existing_db_connection = MagicMock() + storage = SimpleNamespace(**{alias: "stale"}) + connections_mock = MagicMock() + connections_mock.databases = {alias: {"ENGINE": "old"}} + connections_mock.__getitem__.return_value = existing_db_connection + connections_mock._connections = storage + + with patch("apps.exchange.services.connections", connections_mock): + configured_alias = ExchangeConnectionService._configure_alias(connection) + + self.assertEqual(configured_alias, alias) + existing_db_connection.close.assert_called_once_with() + self.assertEqual(connections_mock.databases[alias]["NAME"], connection.database_name) + self.assertEqual(connections_mock.databases[alias]["PASSWORD"], "secret") + self.assertNotIn(alias, storage.__dict__) + + def test_validate_schema_exists_raises_when_schema_missing(self): + cursor = MagicMock() + cursor.fetchone.return_value = None + db_connection = MagicMock() + db_connection.cursor.return_value.__enter__.return_value = cursor + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with self.assertRaisesMessage( + ExchangeServiceError, + "Схема 'public' отсутствует в целевой БД", + ): + ExchangeConnectionService._validate_schema_exists( + alias="target_alias", + schema_name="public", + ) + + def test_validate_tables_exist_raises_when_tables_missing(self): + cursor = MagicMock() + cursor.fetchall.return_value = [("fake_table",)] + db_connection = MagicMock() + db_connection.cursor.return_value.__enter__.return_value = cursor + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with self.assertRaisesMessage( + ExchangeServiceError, + "В целевой БД отсутствуют таблицы: another_table", + ): + ExchangeConnectionService._validate_tables_exist( + alias="target_alias", + schema_name="public", + models_to_copy=[_FakeModel, _AnotherFakeModel], + ) + + def test_validate_columns_exist_raises_when_columns_missing(self): + cursor_context = MagicMock() + cursor = MagicMock() + cursor.fetchall.return_value = [("id",)] + cursor_context.__enter__.return_value = cursor + db_connection = MagicMock() + db_connection.cursor.return_value = cursor_context + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + with self.assertRaisesMessage( + ExchangeServiceError, + "В таблице 'fake_table' отсутствуют колонки: name", + ): + ExchangeConnectionService._validate_columns_exist( + alias="target_alias", + schema_name="public", + models_to_copy=[_FakeModel], + ) + + def test_get_parser_models_uses_configured_labels(self): + resolved_models = [_FakeModel for _ in ExchangeConnectionService.PARSER_MODEL_LABELS] + + with patch( + "apps.exchange.services.django_apps.get_model", + side_effect=resolved_models, + ) as get_model_mock: + result = ExchangeConnectionService._get_parser_models() + + self.assertEqual(result, resolved_models) + self.assertEqual( + [call.args[0] for call in get_model_mock.call_args_list], + ExchangeConnectionService.PARSER_MODEL_LABELS, + ) + + def test_resolve_models_supports_table_and_class_names(self): + with patch.object( + ExchangeConnectionService, + "_get_parser_models", + return_value=[ParserLoadLog, ManufacturerRecord], + ): + selected = ExchangeConnectionService._resolve_models( + mode="selected", + table=None, + tables=["parsers_load_log", "manufacturerrecord"], + ) + + self.assertEqual(selected, [ParserLoadLog, ManufacturerRecord]) + + def test_resolve_models_raises_on_unknown_table(self): + with patch.object( + ExchangeConnectionService, + "_get_parser_models", + return_value=[ParserLoadLog], + ): + with self.assertRaisesMessage(ExchangeServiceError, "Неизвестная таблица"): + ExchangeConnectionService._resolve_models( + mode="single", + table="unknown_table", + tables=None, + ) + + def test_truncate_tables_executes_in_reverse_order(self): + cursor = MagicMock() + db_connection = MagicMock() + db_connection.cursor.return_value.__enter__.return_value = cursor + connections_mock = MagicMock() + connections_mock.__getitem__.return_value = db_connection + + with patch("apps.exchange.services.connections", connections_mock): + ExchangeConnectionService._truncate_tables( + alias="target_alias", + models_to_copy=[_FakeModel, _AnotherFakeModel], + ) + + executed_sql = [call.args[0] for call in cursor.execute.call_args_list] + self.assertEqual( + executed_sql, + [ + 'TRUNCATE TABLE "another_table" RESTART IDENTITY CASCADE', + 'TRUNCATE TABLE "fake_table" RESTART IDENTITY CASCADE', + ], + ) + + def test_copy_model_data_splits_batches(self): + source_objects = [ + SimpleNamespace(id=1, name="A"), + SimpleNamespace(id=2, name="B"), + SimpleNamespace(id=3, name="C"), + ] + queryset = MagicMock() + queryset.order_by.return_value = queryset + queryset.iterator.return_value = source_objects + default_manager = MagicMock() + default_manager.all.return_value = queryset + _FakeModel.objects = MagicMock() + _FakeModel.objects.using.return_value = default_manager + + with patch.object( + ExchangeConnectionService, + "_insert_batch", + side_effect=[2, 1], + ) as insert_mock: + total = ExchangeConnectionService._copy_model_data( + model=_FakeModel, + alias="target_alias", + truncate_before_copy=True, + chunk_size=2, + ) + + self.assertEqual(total, 3) + self.assertEqual(insert_mock.call_count, 2) + + def test_insert_batch_returns_batch_size_in_truncate_mode(self): + manager = MagicMock() + _FakeModel.objects = MagicMock() + _FakeModel.objects.using.return_value = manager + batch = [_FakeModel(id=1, name="A"), _FakeModel(id=2, name="B")] + + created_count = ExchangeConnectionService._insert_batch( + model=_FakeModel, + alias="target_alias", + batch=batch, + pk_name="id", + chunk_size=100, + truncate_before_copy=True, + ) + + self.assertEqual(created_count, 2) + manager.bulk_create.assert_called_once_with( + batch, + batch_size=100, + ignore_conflicts=False, + ) + + def test_insert_batch_counts_only_new_rows_without_truncate(self): + manager = MagicMock() + manager.filter.side_effect = [ + MagicMock(values_list=MagicMock(return_value=[1])), + MagicMock(values_list=MagicMock(return_value=[1, 2])), + ] + _FakeModel.objects = MagicMock() + _FakeModel.objects.using.return_value = manager + batch = [_FakeModel(id=1, name="A"), _FakeModel(id=2, name="B")] + + created_count = ExchangeConnectionService._insert_batch( + model=_FakeModel, + alias="target_alias", + batch=batch, + pk_name="id", + chunk_size=100, + truncate_before_copy=False, + ) + + self.assertEqual(created_count, 1) + manager.bulk_create.assert_called_once_with( + batch, + batch_size=100, + ignore_conflicts=True, + ) + + def test_mark_connection_error_updates_connection(self): + connection = ExchangeConnectionFactory(last_error="") + + ExchangeConnectionService._mark_connection_error(connection, "broken") + + connection.refresh_from_db() + self.assertEqual(connection.last_error, "broken") + self.assertIsNotNone(connection.last_checked_at) + + def tearDown(self): + for alias in list(getattr(ExchangeConnectionService, "__dict__", {})): + if alias.startswith("exchange_target_"): + with suppress(Exception): + from django.db import connections + + connections[alias].close() + with suppress(Exception): + from django.db import connections + + connections.databases.pop(alias, None) diff --git a/tests/apps/exchange/test_services.py b/tests/apps/exchange/test_services.py index cb55c5f..a50bc1a 100644 --- a/tests/apps/exchange/test_services.py +++ b/tests/apps/exchange/test_services.py @@ -1,8 +1,12 @@ """Tests for exchange services.""" +from contextlib import suppress + +from apps.exchange.models import ExchangeConnection from apps.exchange.services import ExchangeConnectionService from apps.parsers.models import IndustrialCertificateRecord, ParserLoadLog from apps.registers.models import Organization +from django.db import connections from django.test import TestCase @@ -23,3 +27,63 @@ class ExchangeConnectionServiceDependenciesTest(TestCase): self.assertEqual(models_to_copy[0], Organization) self.assertEqual(models_to_copy[1], IndustrialCertificateRecord) + + +class ExchangeConnectionEncryptionTest(TestCase): + def test_save_encrypts_password(self): + connection = ExchangeConnection.objects.create( + server="127.0.0.1", + port=5432, + username="postgres", + password="secret", # noqa: S106 + database_name="target_db", + schema_name="public", + ) + + self.assertNotEqual(connection.password, "secret") + self.assertTrue(ExchangeConnection.is_password_encrypted(connection.password)) + self.assertEqual(connection.get_decrypted_password(), "secret") + + def test_save_encrypts_legacy_password_on_partial_update(self): + connection = ExchangeConnection.objects.create( + server="127.0.0.1", + port=5432, + username="postgres", + password="secret", # noqa: S106 + database_name="target_db", + schema_name="public", + ) + ExchangeConnection.objects.filter(id=connection.id).update( + password="legacy-pass" # noqa: S106 + ) + + connection.refresh_from_db() + self.assertEqual(connection.password, "legacy-pass") + + connection.last_error = "checked" + connection.save(update_fields=["last_error", "updated_at"]) + connection.refresh_from_db() + + self.assertNotEqual(connection.password, "legacy-pass") + self.assertEqual(connection.get_decrypted_password(), "legacy-pass") + + def test_configure_alias_uses_decrypted_password(self): + connection = ExchangeConnection.objects.create( + server="127.0.0.1", + port=5432, + username="postgres", + password="secret", # noqa: S106 + database_name="target_db", + schema_name="public", + ) + + alias = ExchangeConnectionService._configure_alias(connection) + try: + self.assertEqual(connections.databases[alias]["PASSWORD"], "secret") + finally: + with suppress(Exception): + connections[alias].close() + connections.databases.pop(alias, None) + storage = getattr(connections, "_connections", None) + if storage is not None and hasattr(storage, "__dict__"): + storage.__dict__.pop(alias, None) diff --git a/tests/apps/exchange/test_tasks.py b/tests/apps/exchange/test_tasks.py new file mode 100644 index 0000000..5966163 --- /dev/null +++ b/tests/apps/exchange/test_tasks.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from apps.exchange.tasks import copy_parsers_data_async +from django.test import SimpleTestCase + + +class ExchangeTasksTest(SimpleTestCase): + def test_copy_parsers_data_async_completes_with_existing_job(self): + background_job = MagicMock() + connection = MagicMock() + + copy_parsers_data_async.push_request(id="task-1") + try: + with patch( + "apps.exchange.tasks.BackgroundJobService.get_by_task_id_or_none", + return_value=background_job, + ) as get_job_mock: + with patch( + "apps.exchange.tasks.ExchangeConnection.objects.filter", + ) as filter_mock: + with patch( + "apps.exchange.tasks.ExchangeConnectionService.copy_parsers_data", + return_value={ + "mode": "all", + "tables": ["fake_table"], + "rows_by_table": {"fake_table": 3}, + "total_rows": 3, + "truncate_before_copy": True, + }, + ) as copy_mock: + filter_mock.return_value.first.return_value = connection + + result = copy_parsers_data_async.run( + connection_id=11, + payload={"mode": "all", "truncate_before_copy": True}, + requested_by_id=7, + ) + finally: + copy_parsers_data_async.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["connection_id"], 11) + get_job_mock.assert_called_once_with("task-1") + background_job.mark_started.assert_called_once_with() + background_job.update_progress.assert_any_call( + 10, + "Проверка структуры целевой БД", + ) + background_job.update_progress.assert_any_call(90, "Фиксация результата") + background_job.complete.assert_called_once_with(result=result) + copy_mock.assert_called_once_with( + connection=connection, + mode="all", + truncate_before_copy=True, + ) + + def test_copy_parsers_data_async_creates_job_and_fails_when_connection_missing(self): + background_job = MagicMock() + + copy_parsers_data_async.push_request(id=None) + try: + with patch("apps.exchange.tasks.uuid.uuid4", return_value="generated-task-id"): + with patch( + "apps.exchange.tasks.BackgroundJobService.get_by_task_id_or_none", + return_value=None, + ): + with patch( + "apps.exchange.tasks.BackgroundJobService.create_job", + return_value=background_job, + ) as create_job_mock: + with patch( + "apps.exchange.tasks.ExchangeConnection.objects.filter", + ) as filter_mock: + filter_mock.return_value.first.return_value = None + + with self.assertRaisesMessage( + ValueError, + "Active exchange connection not found: 42", + ): + copy_parsers_data_async.run( + connection_id=42, + payload={"mode": "all"}, + requested_by_id=3, + ) + finally: + copy_parsers_data_async.pop_request() + + create_job_mock.assert_called_once_with( + task_id="generated-task-id", + task_name="apps.exchange.tasks.copy_parsers_data_async", + user_id=3, + meta={"connection_id": 42, "mode": "all"}, + ) + background_job.fail.assert_called_once_with(error="Активное подключение не найдено") + + def test_copy_parsers_data_async_marks_failure_and_reraises(self): + background_job = MagicMock() + connection = MagicMock() + + copy_parsers_data_async.push_request(id="task-2") + try: + with patch( + "apps.exchange.tasks.BackgroundJobService.get_by_task_id_or_none", + return_value=background_job, + ): + with patch( + "apps.exchange.tasks.ExchangeConnection.objects.filter", + ) as filter_mock: + with patch( + "apps.exchange.tasks.ExchangeConnectionService.copy_parsers_data", + side_effect=RuntimeError("copy failed"), + ): + with patch("apps.exchange.tasks.logger.exception") as logger_mock: + filter_mock.return_value.first.return_value = connection + + with self.assertRaisesMessage(RuntimeError, "copy failed"): + copy_parsers_data_async.run( + connection_id=9, + payload={"mode": "selected", "tables": ["fake_table"]}, + ) + finally: + copy_parsers_data_async.pop_request() + + background_job.fail.assert_called_once_with(error="copy failed") + logger_mock.assert_called_once() diff --git a/tests/apps/exchange/test_views.py b/tests/apps/exchange/test_views.py index 111c2c3..2a7eaf9 100644 --- a/tests/apps/exchange/test_views.py +++ b/tests/apps/exchange/test_views.py @@ -56,6 +56,8 @@ class ExchangeViewsTest(APITestCase): new_connection = ExchangeConnection.objects.get(id=response.data["data"]["id"]) self.assertTrue(new_connection.is_active) + self.assertNotEqual(new_connection.password, payload["password"]) + self.assertEqual(new_connection.get_decrypted_password(), payload["password"]) old_active.refresh_from_db() self.assertFalse(old_active.is_active) diff --git a/tests/apps/parsers/factories.py b/tests/apps/parsers/factories.py index 16e3336..3238667 100644 --- a/tests/apps/parsers/factories.py +++ b/tests/apps/parsers/factories.py @@ -7,6 +7,7 @@ from datetime import timedelta import factory from apps.parsers.models import ( IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -54,6 +55,11 @@ def generate_legal_address() -> str: return fake.address().replace("\n", ", ") +def generate_registry_number() -> str: + """Генерация регистрационного номера записи.""" + return f"MPP-{_digits(8)}" + + def generate_proxy_address() -> str: """Генерация адреса прокси.""" return f"http://{fake.ipv4()}:{fake.port_number()}" @@ -163,6 +169,26 @@ class ManufacturerRecordFactory(factory.django.DjangoModelFactory): address = factory.LazyFunction(generate_legal_address) +class IndustrialProductRecordFactory(factory.django.DjangoModelFactory): + """Factory for IndustrialProductRecord model.""" + + class Meta: + model = IndustrialProductRecord + + load_batch = factory.Sequence(lambda n: n + 1) + full_organisation_name = factory.LazyFunction(generate_company_name) + ogrn = factory.LazyFunction(generate_ogrn) + inn = factory.LazyFunction(generate_inn_legal) + registry_number = factory.LazyFunction(generate_registry_number) + product_name = factory.LazyAttribute(lambda _: fake.sentence(nb_words=4)) + product_model = factory.LazyAttribute(lambda _: fake.bothify(text="MODEL-###")) + okpd2_code = factory.LazyAttribute( + lambda _: f"{fake.random_int(min=10, max=99)}.{fake.random_int(min=10, max=99)}" + ) + tnved_code = factory.LazyAttribute(lambda _: _digits(10)) + regulatory_document = factory.LazyAttribute(lambda _: fake.sentence(nb_words=5)) + + class InspectionRecordFactory(factory.django.DjangoModelFactory): """Factory for InspectionRecord model.""" diff --git a/tests/apps/parsers/test_clients.py b/tests/apps/parsers/test_clients.py index 9faaab5..c13e980 100644 --- a/tests/apps/parsers/test_clients.py +++ b/tests/apps/parsers/test_clients.py @@ -13,7 +13,12 @@ from apps.parsers.clients.base import ( ) from apps.parsers.clients.minpromtorg.industrial import IndustrialProductionClient from apps.parsers.clients.minpromtorg.manufactures import ManufacturesClient -from apps.parsers.clients.minpromtorg.schemas import IndustrialCertificate, Manufacturer +from apps.parsers.clients.minpromtorg.products import IndustrialProductsClient +from apps.parsers.clients.minpromtorg.schemas import ( + IndustrialCertificate, + IndustrialProduct, + Manufacturer, +) from apps.parsers.clients.proverki import ProverkiClient from apps.parsers.clients.proverki.schemas import Inspection from django.test import TestCase, tag @@ -23,6 +28,7 @@ from tests.utils import Response, TestHTTPServer from tests.utils.fixtures import ( build_minpromtorg_certificates_excel, build_minpromtorg_manufacturers_excel, + build_minpromtorg_products_excel, build_proverki_xml, fake, ) @@ -43,6 +49,10 @@ def _proxy_address() -> str: return f"http://{fake.ipv4()}:{fake.port_number()}" +def _digits(length: int) -> str: + return "".join(str(fake.random_int(0, 9)) for _ in range(length)) + + class _RaisingAdapter(BaseAdapter): def __init__(self, exc: Exception) -> None: super().__init__() @@ -440,6 +450,126 @@ class ManufacturesClientTest(TestCase): self.assertEqual(result.address, "") +class IndustrialProductsClientTest(TestCase): + """Tests for IndustrialProductsClient.""" + + def test_client_initialization(self): + client = IndustrialProductsClient() + self.assertIsNone(client.proxies) + self.assertEqual(client.host, "minpromtorg.gov.ru") + + def test_fetch_products_success(self): + excel_bytes, rows = build_minpromtorg_products_excel(count=5) + date_str = fake.date_between(start_date="-30d", end_date="today").strftime( + "%Y%m%d" + ) + file_name = f"industrial_products_{date_str}.xlsx" + + with TestHTTPServer() as server: + server.add_json( + "/api/kss-document-preview", + { + "data": [ + { + "name": IndustrialProductsClient().query, + "files": [ + {"name": file_name, "url": f"/files/{file_name}"} + ], + } + ] + }, + ) + server.add_bytes( + f"/files/{file_name}", + excel_bytes, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + client = IndustrialProductsClient( + host=_host_from_base_url(server.base_url), + scheme="http", + http_adapter=server.adapter, + ) + products = client.fetch_products() + + self.assertEqual(len(products), len(rows)) + self.assertIsInstance(products[0], IndustrialProduct) + self.assertSetEqual( + {product.registry_number for product in products}, + {row.registry_number for row in rows}, + ) + + def test_fetch_products_no_files(self): + with TestHTTPServer() as server: + server.add_json("/api/kss-document-preview", {"data": []}) + client = IndustrialProductsClient( + host=_host_from_base_url(server.base_url), + scheme="http", + http_adapter=server.adapter, + ) + products = client.fetch_products() + + self.assertEqual(products, []) + + def test_get_latest_file_url_falls_back_to_excel_file(self): + client = IndustrialProductsClient() + files = [ + {"name": "readme.txt", "url": "/files/readme.txt"}, + {"name": "registry.xlsx", "url": "/files/registry.xlsx"}, + ] + + url = client._get_latest_file_url(files) + self.assertEqual(url, "https://minpromtorg.gov.ru/files/registry.xlsx") + + def test_parse_row_valid(self): + client = IndustrialProductsClient() + header_map = { + "full_organisation_name": 0, + "ogrn": 1, + "inn": 2, + "registry_number": 3, + "product_name": 4, + "product_model": 5, + "okpd2_code": 6, + "tnved_code": 7, + "regulatory_document": 8, + } + row = ( + fake.company(), + _digits(13), + _digits(10), + f"MPP-{_digits(8)}", + fake.sentence(nb_words=4), + fake.bothify(text="MODEL-###"), + "25.11", + _digits(10), + fake.sentence(nb_words=5), + ) + + result = client._parse_row(row, header_map) + + self.assertIsInstance(result, IndustrialProduct) + self.assertEqual(result.registry_number, row[3]) + self.assertEqual(result.product_name, row[4]) + + def test_parse_row_without_required_fields(self): + client = IndustrialProductsClient() + header_map = { + "full_organisation_name": 0, + "ogrn": 1, + "inn": 2, + "registry_number": 3, + "product_name": 4, + } + + result = client._parse_row( + (fake.company(), _digits(13), _digits(10), "", ""), header_map + ) + self.assertIsNone(result) + + @tag("integration", "slow") class IndustrialProductionClientIntegrationTest(TestCase): """Integration test using local HTTP server instead of external API.""" @@ -516,6 +646,44 @@ class ManufacturesClientIntegrationTest(TestCase): self.assertEqual(len(manufacturers), len(rows)) +@tag("integration", "slow") +class IndustrialProductsClientIntegrationTest(TestCase): + """Integration test using local HTTP server instead of external API.""" + + def test_fetch_products_local_server(self): + excel_bytes, rows = build_minpromtorg_products_excel(count=3) + date_str = fake.date_between(start_date="-30d", end_date="today").strftime( + "%Y%m%d" + ) + file_name = f"industrial_products_{date_str}.xlsx" + + with TestHTTPServer() as server: + server.add_json( + "/api/kss-document-preview", + { + "data": [ + { + "name": IndustrialProductsClient().query, + "files": [ + {"name": file_name, "url": f"/files/{file_name}"} + ], + } + ] + }, + ) + server.add_bytes(f"/files/{file_name}", excel_bytes) + + client = IndustrialProductsClient( + host=_host_from_base_url(server.base_url), + scheme="http", + timeout=30, + http_adapter=server.adapter, + ) + products = client.fetch_products() + + self.assertEqual(len(products), len(rows)) + + class ProverkiClientTest(TestCase): """Tests for ProverkiClient.""" @@ -574,7 +742,7 @@ class ProverkiClientTest(TestCase): self.assertEqual(inspections[0].control_authority, authority) def test_parse_xml_record_with_attributes(self): - from xml.etree import ElementTree as ET + from defusedxml import ElementTree as ET row_inn = "".join(str(fake.random_int(0, 9)) for _ in range(10)) reg_num = "".join(str(fake.random_int(0, 9)) for _ in range(12)) @@ -590,7 +758,7 @@ class ProverkiClientTest(TestCase): self.assertEqual(result.registration_number, reg_num) def test_parse_xml_record_invalid(self): - from xml.etree import ElementTree as ET + from defusedxml import ElementTree as ET element = ET.fromstring("") # noqa: S314 client = ProverkiClient() diff --git a/tests/apps/parsers/test_fns_upload.py b/tests/apps/parsers/test_fns_upload.py index ff468d2..5462c65 100644 --- a/tests/apps/parsers/test_fns_upload.py +++ b/tests/apps/parsers/test_fns_upload.py @@ -4,7 +4,9 @@ import io import os import tempfile import time +from unittest.mock import patch +from apps.core.models import BackgroundJob from apps.parsers.models import FinancialReport, FinancialReportLine from django.core.files.uploadedfile import SimpleUploadedFile from django.test import override_settings @@ -41,7 +43,9 @@ class FNSUploadIntegrationTest(APITestCase): def setUp(self): self.user = UserFactory.create_user() - self.client.force_authenticate(self.user) + self.admin = UserFactory.create_user(is_staff=True) + self.other = UserFactory.create_user() + self.client.force_authenticate(self.admin) self.upload_url = reverse("api_v1:fns:fns-upload") def _dirs(self, base_dir: str) -> tuple[str, str, str]: @@ -136,6 +140,47 @@ class FNSUploadIntegrationTest(APITestCase): os.path.exists(os.path.join(watch_dir, f"{filename}.lock")) ) + def test_upload_creates_owned_background_job(self): + content = _build_fns_excel_bytes() + external_id = _digits(5) + ogrn = _digits(13) + filename = f"fin_{external_id}_{ogrn}.xlsx" + upload = SimpleUploadedFile( + filename, + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ): + response = self.client.post( + self.upload_url, {"files": [upload]}, format="multipart" + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + task_id = response.data["task_ids"][0] + job = BackgroundJob.objects.get(task_id=task_id) + self.assertEqual(job.user_id, self.admin.id) + + jobs_response = self.client.get(reverse("api_v1:jobs:job-list")) + self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) + self.assertEqual(len(jobs_response.data), 1) + self.assertEqual(jobs_response.data[0]["task_id"], task_id) + + other_client = self.client_class() + other_client.force_authenticate(self.other) + status_response = other_client.get( + reverse("api_v1:jobs:job-status", kwargs={"task_id": task_id}) + ) + self.assertEqual(status_response.status_code, status.HTTP_403_FORBIDDEN) + def test_upload_invalid_filename_rejected(self): content = _build_fns_excel_bytes() upload = SimpleUploadedFile( @@ -196,6 +241,29 @@ class FNSUploadIntegrationTest(APITestCase): self.assertEqual(response.data["queued"], 0) self.assertEqual(response.data["skipped"], 1) + def test_regular_user_cannot_upload(self): + self.client.force_authenticate(self.user) + upload = SimpleUploadedFile( + f"fin_{_digits(5)}_{_digits(13)}.xlsx", + _build_fns_excel_bytes(), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ): + response = self.client.post( + self.upload_url, {"files": [upload]}, format="multipart" + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + def test_upload_skips_when_file_already_exists(self): content = _build_fns_excel_bytes() external_id = _digits(5) @@ -228,3 +296,133 @@ class FNSUploadIntegrationTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(response.data["queued"], 0) self.assertEqual(response.data["skipped"], 1) + + def test_upload_removes_stale_lock_and_queues_file(self): + content = _build_fns_excel_bytes() + external_id = _digits(5) + ogrn = _digits(13) + filename = f"fin_{external_id}_{ogrn}.xlsx" + upload = SimpleUploadedFile( + filename, + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + os.makedirs(watch_dir, exist_ok=True) + lock_path = os.path.join(watch_dir, f"{filename}.lock") + with open(lock_path, "w") as handle: + handle.write("lock") + stale = time.time() - 7200 + os.utime(lock_path, (stale, stale)) + + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + FNS_LOCK_TTL_SECONDS=1, + ): + response = self.client.post( + self.upload_url, {"files": [upload]}, format="multipart" + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["queued"], 1) + self.assertEqual(response.data["skipped"], 0) + + def test_upload_skips_when_lock_creation_races(self): + upload = SimpleUploadedFile( + f"fin_{_digits(5)}_{_digits(13)}.xlsx", + _build_fns_excel_bytes(), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ), patch("apps.parsers.views.Path.touch", side_effect=FileExistsError): + response = self.client.post( + self.upload_url, + {"files": [upload]}, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["queued"], 0) + self.assertEqual(response.data["skipped"], 1) + + def test_upload_cleans_up_lock_when_file_write_fails(self): + filename = f"fin_{_digits(5)}_{_digits(13)}.xlsx" + upload = SimpleUploadedFile( + filename, + _build_fns_excel_bytes(), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ), patch("apps.parsers.views.open", side_effect=OSError("disk full")): + response = self.client.post( + self.upload_url, + {"files": [upload]}, + format="multipart", + ) + + self.assertEqual( + response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR + ) + self.assertFalse( + os.path.exists(os.path.join(watch_dir, f"{filename}.lock")) + ) + + def test_upload_deletes_background_job_when_task_enqueue_fails(self): + filename = f"fin_{_digits(5)}_{_digits(13)}.xlsx" + upload = SimpleUploadedFile( + filename, + _build_fns_excel_bytes(), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir, processed_dir, failed_dir = self._dirs(tmpdir) + with override_settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ), patch( + "apps.parsers.views.uuid.uuid4", return_value="job-task-id" + ), patch( + "apps.parsers.views.process_fns_file.apply_async", + side_effect=RuntimeError("queue down"), + ): + response = self.client.post( + self.upload_url, + {"files": [upload]}, + format="multipart", + ) + + self.assertEqual( + response.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR + ) + self.assertFalse( + BackgroundJob.objects.filter(task_id="job-task-id").exists() + ) + self.assertFalse( + os.path.exists(os.path.join(watch_dir, f"{filename}.lock")) + ) diff --git a/tests/apps/parsers/test_models.py b/tests/apps/parsers/test_models.py index 30b57ee..27aaf62 100644 --- a/tests/apps/parsers/test_models.py +++ b/tests/apps/parsers/test_models.py @@ -2,6 +2,7 @@ from apps.parsers.models import ( IndustrialCertificateRecord, + IndustrialProductRecord, ManufacturerRecord, ParserLoadLog, Proxy, @@ -10,6 +11,7 @@ from django.test import TestCase from .factories import ( IndustrialCertificateRecordFactory, + IndustrialProductRecordFactory, ManufacturerRecordFactory, ParserLoadLogFactory, ProxyFactory, @@ -155,3 +157,35 @@ class ManufacturerRecordModelTest(TestCase): self.assertIsNotNone(manufacturer.created_at) self.assertIsNotNone(manufacturer.updated_at) + + +class IndustrialProductRecordModelTest(TestCase): + """Tests for IndustrialProductRecord model.""" + + def test_create_industrial_product(self): + """Test creating industrial product record.""" + product = IndustrialProductRecordFactory() + + self.assertIsInstance(product, IndustrialProductRecord) + self.assertIsNotNone(product.registry_number) + self.assertIsNotNone(product.product_name) + self.assertIsNotNone(product.inn) + self.assertIsNotNone(product.ogrn) + + def test_industrial_product_str(self): + """Test industrial product string representation.""" + registry_number = fake.bothify(text="MPP-########") + product_name = generate_company_name() + product = IndustrialProductRecordFactory( + registry_number=registry_number, + product_name=product_name, + ) + self.assertIn(registry_number, str(product)) + self.assertIn(product_name[:50], str(product)) + + def test_industrial_product_timestamps(self): + """Test industrial product has timestamps from mixin.""" + product = IndustrialProductRecordFactory() + + self.assertIsNotNone(product.created_at) + self.assertIsNotNone(product.updated_at) diff --git a/tests/apps/parsers/test_procurement_service.py b/tests/apps/parsers/test_procurement_service.py index 6198c0f..a813ddf 100644 --- a/tests/apps/parsers/test_procurement_service.py +++ b/tests/apps/parsers/test_procurement_service.py @@ -126,12 +126,12 @@ class ProcurementServiceSaveTestCase(TestCase): record = ProcurementRecord.objects.get(purchase_number=purchase_number) self.assertEqual(record.registry_organization_id, organization.id) - def test_save_ignores_duplicates(self): - """Дубликаты по purchase_number пропускаются.""" + def test_save_updates_duplicates(self): + """Повторная синхронизация обновляет существующую закупку.""" # Создаём существующую запись purchase_number = _digits(19) ProcurementRecordFactory(purchase_number=purchase_number) - initial_count = ProcurementRecord.objects.count() + original = ProcurementRecord.objects.get(purchase_number=purchase_number) # Пытаемся сохранить с тем же номером procurement = _build_procurement( @@ -139,13 +139,15 @@ class ProcurementServiceSaveTestCase(TestCase): customer_inn=_digits(10), ) - ProcurementService.save_procurements([procurement], batch_id=2) + saved = ProcurementService.save_procurements([procurement], batch_id=2) - # Дубликат пропущен - количество записей не изменилось - self.assertEqual(ProcurementRecord.objects.count(), initial_count) - # Оригинальная запись не была перезаписана - original = ProcurementRecord.objects.get(purchase_number=purchase_number) - self.assertNotEqual(original.customer_inn, procurement.customer_inn) + # Существующая запись обновляется в пределах той же строки + self.assertEqual(saved, 1) + self.assertEqual(ProcurementRecord.objects.count(), 1) + refreshed = ProcurementRecord.objects.get(purchase_number=purchase_number) + self.assertEqual(refreshed.customer_inn, procurement.customer_inn) + self.assertEqual(refreshed.load_batch, 2) + self.assertEqual(refreshed.id, original.id) def test_save_with_chunking(self): """Сохранение большого количества записей чанками.""" diff --git a/tests/apps/parsers/test_service_helpers.py b/tests/apps/parsers/test_service_helpers.py new file mode 100644 index 0000000..c0f767c --- /dev/null +++ b/tests/apps/parsers/test_service_helpers.py @@ -0,0 +1,214 @@ +"""Focused unit tests for parser service helpers and small query methods.""" + +from __future__ import annotations + +from datetime import date +from decimal import Decimal +from unittest.mock import patch + +from apps.parsers.models import ParserLoadLog +from apps.parsers.services import ( + FNSReportService, + IndustrialProductService, + InspectionService, + ParserLoadLogService, + ProcurementService, + RegistryOrganizationResolver, + normalize_to_date, + normalize_to_decimal, +) +from django.db import IntegrityError +from django.test import TestCase + +from tests.apps.parsers.factories import ( + IndustrialProductRecordFactory, + InspectionRecordFactory, + ParserLoadLogFactory, + ProcurementRecordFactory, +) +from tests.apps.registers.factories import OrganizationFactory + + +class NormalizeHelpersTest(TestCase): + def test_normalize_to_date_handles_direct_and_embedded_formats(self): + self.assertIsNone(normalize_to_date(None)) + self.assertIsNone(normalize_to_date(" ")) + self.assertEqual( + normalize_to_date("2026-03-17T10:15:30+03:00"), + date(2026, 3, 17), + ) + self.assertEqual( + normalize_to_date("report created at 2026-03-17 10:15"), + date(2026, 3, 17), + ) + self.assertEqual( + normalize_to_date("актуально на 17.03.2026 10:15"), + date(2026, 3, 17), + ) + self.assertIsNone(normalize_to_date("not a date")) + + def test_normalize_to_decimal_handles_common_formats_and_invalid_values(self): + self.assertIsNone(normalize_to_decimal(None)) + self.assertIsNone(normalize_to_decimal(" ")) + self.assertEqual(normalize_to_decimal("1.234,56 руб."), Decimal("1234.56")) + self.assertEqual(normalize_to_decimal("1,234.56"), Decimal("1234.56")) + self.assertIsNone(normalize_to_decimal("руб.")) + self.assertIsNone(normalize_to_decimal("--1")) + + +class RegistryOrganizationResolverTest(TestCase): + def test_normalize_identifier_rejects_non_digits(self): + self.assertIsNone(RegistryOrganizationResolver.normalize_identifier("")) + self.assertIsNone(RegistryOrganizationResolver.normalize_identifier("12 34")) + self.assertIsNone(RegistryOrganizationResolver.normalize_identifier("abc")) + + def test_build_lookup_returns_empty_indexes_when_identifiers_absent(self): + lookup = RegistryOrganizationResolver.build_lookup([(None, None), ("", "")]) + self.assertEqual(lookup.by_pair, {}) + self.assertEqual(lookup.by_inn, {}) + self.assertEqual(lookup.by_ogrn, {}) + + def test_resolve_organization_id_by_unique_inn_and_ogrn(self): + org_by_inn = OrganizationFactory(mn_inn=7_701_001_001, mn_ogrn=10_277_001_000_001) + org_by_ogrn = OrganizationFactory(mn_inn=7_701_001_002, mn_ogrn=10_277_001_000_002) + lookup = RegistryOrganizationResolver.build_lookup( + [ + (org_by_inn.mn_inn, None), + (None, org_by_ogrn.mn_ogrn), + ] + ) + + self.assertEqual( + RegistryOrganizationResolver.resolve_organization_id( + lookup=lookup, + inn=str(org_by_inn.mn_inn), + ogrn=None, + ), + org_by_inn.id, + ) + self.assertEqual( + RegistryOrganizationResolver.resolve_organization_id( + lookup=lookup, + inn=None, + ogrn=str(org_by_ogrn.mn_ogrn), + ), + org_by_ogrn.id, + ) + + +class ParserLoadLogServiceRetryTest(TestCase): + def test_create_load_log_with_next_batch_id_retries_after_integrity_error(self): + log = ParserLoadLogFactory.build( + source=ParserLoadLog.Source.INDUSTRIAL, + batch_id=2, + ) + + with patch.object(ParserLoadLogService, "get_next_batch_id", side_effect=[1, 2]): + with patch.object( + ParserLoadLogService, + "create_load_log", + side_effect=[IntegrityError("duplicate"), log], + ) as create_mock: + created_log, batch_id = ( + ParserLoadLogService.create_load_log_with_next_batch_id( + source=ParserLoadLog.Source.INDUSTRIAL + ) + ) + + self.assertEqual(created_log, log) + self.assertEqual(batch_id, 2) + self.assertEqual(create_mock.call_count, 2) + + def test_create_load_log_with_next_batch_id_raises_after_max_retries(self): + with patch.object(ParserLoadLogService, "get_next_batch_id", return_value=1): + with patch.object( + ParserLoadLogService, + "create_load_log", + side_effect=IntegrityError("duplicate"), + ): + with self.assertRaisesMessage( + RuntimeError, + "Failed to allocate unique batch_id", + ): + ParserLoadLogService.create_load_log_with_next_batch_id( + source=ParserLoadLog.Source.INDUSTRIAL, + max_retries=2, + ) + + +class SmallParserServiceQueryTest(TestCase): + def test_industrial_product_service_query_helpers(self): + record = IndustrialProductRecordFactory( + inn="7701001001", + ogrn="1027700100001", + load_batch=7, + ) + IndustrialProductRecordFactory( + inn="7701001001", + ogrn="1027700100002", + load_batch=8, + ) + + self.assertEqual(IndustrialProductService.find_by_inn("7701001001").count(), 2) + self.assertEqual( + IndustrialProductService.find_by_inn("7701001001", batch_id=7).count(), + 1, + ) + self.assertEqual( + IndustrialProductService.find_by_ogrn("1027700100001").first().id, + record.id, + ) + + def test_inspection_service_has_data_for_period(self): + InspectionRecordFactory(data_year=2026, data_month=3, is_federal_law_248=False) + InspectionRecordFactory(data_year=2026, data_month=4, is_federal_law_248=True) + + self.assertTrue(InspectionService.has_data_for_period(2026, 3)) + self.assertFalse(InspectionService.has_data_for_period(2026, 3, True)) + self.assertTrue(InspectionService.has_data_for_period(2026, 4, True)) + + def test_procurement_service_find_by_customer_name_with_batch(self): + ProcurementRecordFactory( + customer_name="АО Тестовый заказчик", + load_batch=11, + ) + ProcurementRecordFactory( + customer_name="АО Тестовый заказчик", + load_batch=12, + ) + + self.assertEqual(ProcurementService.find_by_customer_name("Тестовый").count(), 2) + self.assertEqual( + ProcurementService.find_by_customer_name("Тестовый", batch_id=11).count(), + 1, + ) + + +class FNSReportServiceHelpersTest(TestCase): + def test_exists_find_and_status_helpers(self): + report = FNSReportService.save_report( + external_id="EXT-100", + ogrn="1027700111111", + file_name="fin_EXT-100_1027700111111.xlsx", + file_hash="a" * 64, + source="api", + batch_id=1, + lines_data=[], + ) + + self.assertTrue(FNSReportService.exists_by_external_id("EXT-100")) + self.assertFalse(FNSReportService.exists_by_external_id("EXT-404")) + self.assertEqual(FNSReportService.find_by_external_id("EXT-100").id, report.id) + + FNSReportService.mark_processing(report) + report.refresh_from_db() + self.assertEqual(report.status, report.Status.PROCESSING) + + FNSReportService.mark_success(report) + report.refresh_from_db() + self.assertEqual(report.status, report.Status.SUCCESS) + + FNSReportService.mark_failed(report, "boom") + report.refresh_from_db() + self.assertEqual(report.status, report.Status.FAILED) + self.assertEqual(report.error_message, "boom") diff --git a/tests/apps/parsers/test_services.py b/tests/apps/parsers/test_services.py index a9d9ef7..f91235e 100644 --- a/tests/apps/parsers/test_services.py +++ b/tests/apps/parsers/test_services.py @@ -3,11 +3,16 @@ from urllib.parse import urlparse from apps.parsers.clients.minpromtorg.industrial import IndustrialProductionClient -from apps.parsers.clients.minpromtorg.schemas import IndustrialCertificate, Manufacturer +from apps.parsers.clients.minpromtorg.schemas import ( + IndustrialCertificate, + IndustrialProduct, + Manufacturer, +) from apps.parsers.clients.proverki.schemas import Inspection from apps.parsers.clients.zakupki.schemas import Procurement from apps.parsers.models import ( IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -16,6 +21,7 @@ from apps.parsers.models import ( ) from apps.parsers.services import ( IndustrialCertificateService, + IndustrialProductService, InspectionService, ManufacturerService, ParserLoadLogService, @@ -30,6 +36,7 @@ from tests.utils.fixtures import build_minpromtorg_certificates_excel, fake from .factories import ( IndustrialCertificateRecordFactory, + IndustrialProductRecordFactory, InspectionRecordFactory, ManufacturerRecordFactory, ParserLoadLogFactory, @@ -512,8 +519,8 @@ class ManufacturerServiceTest(TestCase): results = ManufacturerService.find_by_ogrn(ogrn_target) self.assertEqual(results.count(), 1) - def test_save_manufacturers_deduplication(self): - """Test saving manufacturers skips duplicates by INN.""" + def test_save_manufacturers_updates_existing_record(self): + """Test saving manufacturers refreshes existing record by INN.""" # Create initial manufacturer inn_value = _digits(10) ogrn_value = _digits(13) @@ -531,27 +538,139 @@ class ManufacturerServiceTest(TestCase): self.assertEqual(count1, 1) self.assertEqual(ManufacturerRecord.objects.count(), 1) - # Try to save with same INN - should be skipped + updated_name = fake.company() + updated_address = fake.address().replace("\n", ", ") + updated_ogrn = _digits(13) duplicate = [ Manufacturer( - full_legal_name=fake.company(), - inn=inn_value, # Same INN - will be skipped - ogrn=_digits(13), - address=fake.address().replace("\n", ", "), + full_legal_name=updated_name, + inn=inn_value, + ogrn=updated_ogrn, + address=updated_address, ) ] count2 = ManufacturerService.save_manufacturers(duplicate, batch_id=2) - # Should still be 1 record (duplicate skipped) - self.assertEqual(count2, 0) + # Existing record should be updated in place. + self.assertEqual(count2, 1) self.assertEqual(ManufacturerRecord.objects.count(), 1) - # Verify original data preserved + # Verify latest data preserved record = ManufacturerRecord.objects.first() - self.assertEqual(record.full_legal_name, company_name) - self.assertEqual(record.ogrn, ogrn_value) - self.assertEqual(record.address, address_value) - self.assertEqual(record.load_batch, 1) # Original batch + self.assertEqual(record.full_legal_name, updated_name) + self.assertEqual(record.ogrn, updated_ogrn) + self.assertEqual(record.address, updated_address) + self.assertEqual(record.load_batch, 2) + + +class IndustrialProductServiceTest(TestCase): + """Tests for IndustrialProductService.""" + + def test_save_products_empty(self): + """Test saving empty list returns 0.""" + count = IndustrialProductService.save_products([], batch_id=1) + self.assertEqual(count, 0) + + def test_save_products(self): + """Test saving industrial products from dataclass.""" + products = [ + IndustrialProduct( + full_organisation_name=fake.company(), + ogrn=_digits(13), + inn=_digits(10), + registry_number=f"MPP-{_digits(8)}", + product_name=fake.sentence(nb_words=4), + product_model=fake.bothify(text="MODEL-###"), + okpd2_code=f"{fake.random_int(min=10, max=99)}.{fake.random_int(min=10, max=99)}", + tnved_code=_digits(10), + regulatory_document=fake.sentence(nb_words=5), + ) + for _ in range(5) + ] + + count = IndustrialProductService.save_products(products, batch_id=1) + + self.assertEqual(count, 5) + self.assertEqual(IndustrialProductRecord.objects.count(), 5) + + def test_save_products_links_registry_organization_when_exists(self): + """Test linking industrial product to registers organization.""" + inn = _digits(10) + ogrn = _digits(13) + organization = _create_registry_organization(inn=inn, ogrn=ogrn) + registry_number = f"MPP-{_digits(8)}" + + products = [ + IndustrialProduct( + full_organisation_name=fake.company(), + ogrn=ogrn, + inn=inn, + registry_number=registry_number, + product_name=fake.sentence(nb_words=4), + product_model=fake.bothify(text="MODEL-###"), + okpd2_code=f"{fake.random_int(min=10, max=99)}.{fake.random_int(min=10, max=99)}", + tnved_code=_digits(10), + regulatory_document=fake.sentence(nb_words=5), + ) + ] + + saved = IndustrialProductService.save_products(products, batch_id=1) + + self.assertEqual(saved, 1) + record = IndustrialProductRecord.objects.get(registry_number=registry_number) + self.assertEqual(record.registry_organization_id, organization.id) + + def test_find_by_registry_number(self): + """Test finding industrial product by registry number.""" + registry_number = f"MPP-{_digits(8)}" + IndustrialProductRecordFactory(registry_number=registry_number) + IndustrialProductRecordFactory(registry_number=f"MPP-{_digits(8)}") + + results = IndustrialProductService.find_by_registry_number(registry_number) + self.assertEqual(results.count(), 1) + + def test_save_products_updates_existing_record(self): + """Test saving products refreshes existing record by registry number.""" + registry_number = f"MPP-{_digits(8)}" + initial = [ + IndustrialProduct( + full_organisation_name=fake.company(), + ogrn=_digits(13), + inn=_digits(10), + registry_number=registry_number, + product_name="Начальное имя", + product_model="MODEL-001", + okpd2_code="25.11", + tnved_code=_digits(10), + regulatory_document="ГОСТ 1", + ) + ] + count1 = IndustrialProductService.save_products(initial, batch_id=1) + self.assertEqual(count1, 1) + self.assertEqual(IndustrialProductRecord.objects.count(), 1) + + updated = [ + IndustrialProduct( + full_organisation_name=fake.company(), + ogrn=_digits(13), + inn=_digits(10), + registry_number=registry_number, + product_name="Обновленное имя", + product_model="MODEL-777", + okpd2_code="28.99", + tnved_code=_digits(10), + regulatory_document="ГОСТ 2", + ) + ] + count2 = IndustrialProductService.save_products(updated, batch_id=2) + + self.assertEqual(count2, 1) + self.assertEqual(IndustrialProductRecord.objects.count(), 1) + + record = IndustrialProductRecord.objects.first() + self.assertEqual(record.product_name, "Обновленное имя") + self.assertEqual(record.product_model, "MODEL-777") + self.assertEqual(record.load_batch, 2) class InspectionServiceTest(TestCase): @@ -686,8 +805,8 @@ class InspectionServiceTest(TestCase): ) self.assertEqual(results_batch1.count(), 2) - def test_save_inspections_deduplication(self): - """Test saving inspections skips duplicates by registration_number.""" + def test_save_inspections_updates_existing_record(self): + """Test saving inspections refreshes existing record by registration_number.""" # Create initial inspection reg_number = _digits(12) inn_value = _digits(10) @@ -721,36 +840,38 @@ class InspectionServiceTest(TestCase): self.assertEqual(count1, 1) self.assertEqual(InspectionRecord.objects.count(), 1) - # Try to save with same registration_number - should be skipped + updated_name = fake.company() + updated_authority = fake.company() + updated_status = fake.word() duplicate = [ Inspection( - registration_number=reg_number, # Same number - will be skipped + registration_number=reg_number, inn=_digits(10), ogrn=_digits(13), - organisation_name=fake.company(), - control_authority=fake.company(), + organisation_name=updated_name, + control_authority=updated_authority, inspection_type=fake.word(), inspection_form=fake.word(), start_date=str(fake.date()), end_date=str(fake.date()), - status=fake.word(), + status=updated_status, legal_basis=fake.sentence(nb_words=3), result=fake.sentence(nb_words=3), ) ] count2 = InspectionService.save_inspections(duplicate, batch_id=2) - # Should still be 1 record (duplicate skipped) - self.assertEqual(count2, 0) + # Existing record should be updated in place. + self.assertEqual(count2, 1) self.assertEqual(InspectionRecord.objects.count(), 1) - # Verify original data preserved + # Verify latest data preserved record = InspectionRecord.objects.first() - self.assertEqual(record.organisation_name, org_name) - self.assertEqual(record.inn, inn_value) - self.assertEqual(record.control_authority, control_authority) - self.assertEqual(record.status, status) - self.assertEqual(record.load_batch, 1) # Original batch + self.assertEqual(record.organisation_name, updated_name) + self.assertNotEqual(record.inn, inn_value) + self.assertEqual(record.control_authority, updated_authority) + self.assertEqual(record.status, updated_status) + self.assertEqual(record.load_batch, 2) class ProcurementServiceTest(TestCase): @@ -783,25 +904,34 @@ class ProcurementServiceTest(TestCase): saved = ProcurementService.save_procurements([procurement], batch_id=1) self.assertEqual(saved, 1) - record = ProcurementRecord.objects.get(purchase_number=procurement.purchase_number) + record = ProcurementRecord.objects.get( + purchase_number=procurement.purchase_number + ) self.assertEqual(str(record.max_price_amount), "1234567.89") self.assertEqual(str(record.publish_date_normalized), "2026-03-01") self.assertEqual(str(record.end_date_normalized), "2026-03-15") - def test_save_procurements_duplicate_returns_zero(self): + def test_save_procurements_duplicate_updates_existing_record(self): purchase_number = _digits(19) first = self._build_procurement(purchase_number=purchase_number) + updated_customer_name = fake.company() + updated_status = fake.word() duplicate = self._build_procurement( purchase_number=purchase_number, - customer_name=fake.company(), + customer_name=updated_customer_name, + status=updated_status, ) saved_first = ProcurementService.save_procurements([first], batch_id=1) saved_second = ProcurementService.save_procurements([duplicate], batch_id=2) self.assertEqual(saved_first, 1) - self.assertEqual(saved_second, 0) + self.assertEqual(saved_second, 1) self.assertEqual(ProcurementRecord.objects.count(), 1) + record = ProcurementRecord.objects.get(purchase_number=purchase_number) + self.assertEqual(record.customer_name, updated_customer_name) + self.assertEqual(record.status, updated_status) + self.assertEqual(record.load_batch, 2) @tag("integration", "slow", "e2e") diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py new file mode 100644 index 0000000..023aea4 --- /dev/null +++ b/tests/apps/parsers/test_source_cards_service.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from apps.parsers.source_cards import ( + RefreshParamDefinition, + SourceCardDefinition, + SourceCardService, + SourceItemDefinition, +) +from django.http import Http404 +from django.test import SimpleTestCase +from rest_framework.exceptions import ValidationError + + +class SourceCardServiceUnitTest(SimpleTestCase): + def test_get_definition_raises_for_unknown_slug(self): + with self.assertRaises(Http404): + SourceCardService.get_definition("missing-card") + + def test_validate_refresh_params_rejects_unknown_param(self): + definition = SourceCardService.get_definition("public-procurements") + + with self.assertRaises(ValidationError) as error: + SourceCardService._validate_refresh_params( + definition, + {"region_code": "77", "unexpected": "value"}, + ) + + self.assertIn("Неизвестные параметры обновления", str(error.exception.detail)) + + def test_validate_refresh_params_casts_integers(self): + definition = SourceCardService.get_definition("public-procurements") + + validated = SourceCardService._validate_refresh_params( + definition, + { + "region_code": "77", + "current_year": "2025", + "current_month": "2", + }, + ) + + self.assertEqual( + validated, + { + "region_code": "77", + "law_type": "44", + "current_year": 2025, + "current_month": 2, + }, + ) + + def test_validate_refresh_params_raises_on_invalid_integer(self): + definition = SourceCardService.get_definition("public-procurements") + + with self.assertRaises(ValidationError) as error: + SourceCardService._validate_refresh_params( + definition, + {"region_code": "77", "current_year": "not-a-number"}, + ) + + self.assertIn("Значение должно быть целым числом", str(error.exception.detail)) + + @patch( + "apps.parsers.source_cards.SourceCardService._enqueue_task", + side_effect=[ + {"task_id": "task-1", "task_name": "apps.parsers.tasks.parse_industrial_production"}, + {"task_id": "task-2", "task_name": "apps.parsers.tasks.parse_industrial_products"}, + {"task_id": "task-3", "task_name": "apps.parsers.tasks.parse_manufactures"}, + ], + ) + def test_refresh_card_for_manufacturers_enqueues_three_tasks(self, enqueue_mock): + result = SourceCardService.refresh_card( + slug="manufacturers-and-products", + requested_by_id=12, + ) + + self.assertEqual(result["source_card"], "manufacturers-and-products") + self.assertEqual([item["task_id"] for item in result["tasks"]], ["task-1", "task-2", "task-3"]) + self.assertEqual(enqueue_mock.call_count, 3) + + @patch( + "apps.parsers.source_cards.SourceCardService._enqueue_task", + return_value={"task_id": "task-1", "task_name": "apps.parsers.tasks.sync_inspections"}, + ) + def test_launch_refresh_for_inspections_passes_supported_kwargs_only(self, enqueue_mock): + definition = SourceCardService.get_definition("planned-inspections") + + result = SourceCardService._launch_refresh( + definition, + requested_by_id=44, + params={ + "current_year": 2025, + "current_month": 3, + "use_playwright": True, + "ignored": "value", + }, + ) + + self.assertEqual(result, [{"task_id": "task-1", "task_name": "apps.parsers.tasks.sync_inspections"}]) + self.assertEqual( + enqueue_mock.call_args.kwargs["kwargs"], + { + "requested_by_id": 44, + "current_year": 2025, + "current_month": 3, + "use_playwright": True, + }, + ) + + @patch( + "apps.parsers.source_cards.SourceCardService._enqueue_task", + return_value={"task_id": "task-9", "task_name": "apps.parsers.tasks.sync_procurements"}, + ) + def test_refresh_card_for_procurements_uses_default_law_type(self, enqueue_mock): + result = SourceCardService.refresh_card( + slug="public-procurements", + requested_by_id=10, + params={"region_code": "77", "current_year": "2026"}, + ) + + self.assertEqual(result["source_card"], "public-procurements") + self.assertEqual(result["tasks"][0]["task_id"], "task-9") + self.assertEqual( + enqueue_mock.call_args.kwargs["kwargs"], + { + "requested_by_id": 10, + "region_code": "77", + "law_type": "44", + "current_year": 2026, + }, + ) + + def test_launch_refresh_raises_for_unsupported_card(self): + definition = SourceCardDefinition( + slug="custom-source", + title="Custom", + description="Custom card", + order=999, + task_names=(), + source_items=( + SourceItemDefinition( + code="custom", + title="Custom", + description="Custom source", + ), + ), + ) + + with self.assertRaises(ValidationError) as error: + SourceCardService._launch_refresh( + definition, + requested_by_id=1, + params={}, + ) + + self.assertIn("Обновление для карточки не поддерживается", str(error.exception.detail)) + + def test_enqueue_task_deletes_background_job_on_async_error(self): + task = MagicMock() + task.apply_async.side_effect = RuntimeError("broker down") + queryset = MagicMock() + + with patch("apps.parsers.source_cards.uuid.uuid4", return_value="task-id-1"): + with patch("apps.parsers.source_cards.BackgroundJobService.create_job"): + with patch( + "apps.parsers.source_cards.BackgroundJobService.get_queryset", + return_value=queryset, + ): + with self.assertRaisesMessage(RuntimeError, "broker down"): + SourceCardService._enqueue_task( + task=task, + task_name="apps.parsers.tasks.sync_procurements", + requested_by_id=5, + meta={"source_card": "public-procurements"}, + kwargs={"region_code": "77"}, + ) + + queryset.filter.assert_called_once_with(task_id="task-id-1") + queryset.filter.return_value.delete.assert_called_once_with() + + def test_helper_methods_cover_unknown_codes_and_status_variants(self): + self.assertEqual(SourceCardService._get_source_records_count("unknown"), 0) + self.assertEqual(SourceCardService._get_source_organizations_count("unknown"), 0) + self.assertIsNone(SourceCardService._get_source_data_timestamp("unknown")) + self.assertIsNone(SourceCardService._get_latest_load_by_source(None)) + self.assertEqual(SourceCardService._get_status_label("custom"), "custom") + + unavailable_definition = SourceCardDefinition( + slug="unavailable", + title="Unavailable", + description="Unavailable source", + order=1, + task_names=(), + source_items=(), + is_available=False, + ) + in_progress_load = SimpleNamespace(status="in_progress") + failed_load = SimpleNamespace(status="failed") + + self.assertEqual( + SourceCardService._get_status( + definition=unavailable_definition, + active_tasks=[], + latest_load=None, + last_updated_at=None, + ), + "unavailable", + ) + self.assertEqual( + SourceCardService._get_status( + definition=SourceCardService.get_definition("financial-indicators"), + active_tasks=[{"progress": 10}], + latest_load=None, + last_updated_at=None, + ), + "in_progress", + ) + self.assertEqual( + SourceCardService._get_status( + definition=SourceCardService.get_definition("financial-indicators"), + active_tasks=[], + latest_load=in_progress_load, + last_updated_at=None, + ), + "in_progress", + ) + self.assertEqual( + SourceCardService._get_status( + definition=SourceCardService.get_definition("financial-indicators"), + active_tasks=[], + latest_load=failed_load, + last_updated_at=None, + ), + "error", + ) + self.assertEqual( + SourceCardService._get_status( + definition=SourceCardService.get_definition("financial-indicators"), + active_tasks=[], + latest_load=None, + last_updated_at=object(), + ), + "success", + ) + self.assertEqual( + SourceCardService._get_status( + definition=SourceCardService.get_definition("financial-indicators"), + active_tasks=[], + latest_load=None, + last_updated_at=None, + ), + "idle", + ) diff --git a/tests/apps/parsers/test_source_cards_views.py b/tests/apps/parsers/test_source_cards_views.py new file mode 100644 index 0000000..58723f1 --- /dev/null +++ b/tests/apps/parsers/test_source_cards_views.py @@ -0,0 +1,219 @@ +"""Tests for frontend source cards API.""" + +from __future__ import annotations + +from apps.core.models import BackgroundJob, JobStatus +from apps.parsers.models import FinancialReport, FinancialReportLine, ParserLoadLog +from django.test import override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.parsers.factories import ( + IndustrialCertificateRecordFactory, + IndustrialProductRecordFactory, + InspectionRecordFactory, + ManufacturerRecordFactory, + ParserLoadLogFactory, +) +from tests.apps.user.factories import UserFactory +from tests.utils.fixtures import fake + + +def _digits(length: int) -> str: + return "".join(str(fake.random_int(0, 9)) for _ in range(length)) + + +class SourceCardsApiTestCase(APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + self.admin = UserFactory.create_user(is_staff=True) + self.client.force_authenticate(self.user) + + def test_source_cards_list_returns_aggregated_data(self): + report = FinancialReport.objects.create( + external_id=_digits(5), + ogrn=_digits(13), + file_name=f"fin_{_digits(5)}_{_digits(13)}.xlsx", + file_hash=fake.sha256(raw_output=False), + load_batch=1, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + FinancialReportLine.objects.create( + report=report, + form_code="1", + line_code="1100", + line_name="Активы", + year=2025, + period_start=100, + period_end=200, + ) + FinancialReportLine.objects.create( + report=report, + form_code="2", + line_code="2110", + line_name="Выручка", + year=2025, + period_start=300, + period_end=400, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.FNS_REPORTS, + status="success", + records_count=2, + ) + InspectionRecordFactory() + BackgroundJob.objects.create( + task_id="job-inspections-active", + task_name="apps.parsers.tasks.sync_inspections", + status=JobStatus.STARTED, + progress=63, + progress_message="sync", + user_id=self.user.id, + meta={"source": ParserLoadLog.Source.INSPECTIONS}, + ) + + response = self.client.get(reverse("api_v1:sources:source-cards-list")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(response.data["success"]) + cards = {item["slug"]: item for item in response.data["data"]} + + self.assertIn("financial-indicators", cards) + self.assertIn("planned-inspections", cards) + + fns_card = cards["financial-indicators"] + self.assertEqual(fns_card["records_count"], 2) + self.assertEqual(fns_card["organizations_count"], 1) + self.assertEqual(fns_card["status"], "success") + self.assertFalse(fns_card["refresh_requires_params"]) + + inspections_card = cards["planned-inspections"] + self.assertEqual(inspections_card["status"], "in_progress") + self.assertEqual(inspections_card["progress"], 63) + + def test_source_card_detail_returns_combined_minprom_stats(self): + shared_inn = _digits(10) + IndustrialCertificateRecordFactory(inn=shared_inn) + IndustrialProductRecordFactory(inn=shared_inn) + ManufacturerRecordFactory(inn=shared_inn) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.MANUFACTURES, + status="success", + records_count=1, + ) + + response = self.client.get( + reverse( + "api_v1:sources:source-cards-detail", + kwargs={"slug": "manufacturers-and-products"}, + ) + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(response.data["success"]) + card = response.data["data"] + + self.assertEqual(card["records_count"], 3) + self.assertEqual(card["organizations_count"], 1) + self.assertEqual(card["status"], "success") + self.assertEqual(len(card["source_items"]), 3) + self.assertEqual(card["source_items"][0]["latest_load"]["status"], "success") + + def test_source_task_statuses_returns_table_rows(self): + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL, + status="success", + records_count=12, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INSPECTIONS, + status="failed", + records_count=0, + ) + + response = self.client.get(reverse("api_v1:sources:source-cards-statuses")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(response.data["success"]) + rows = response.data["data"] + self.assertGreaterEqual(len(rows), 1) + + row = rows[0] + self.assertIn("row_number", row) + self.assertIn("source", row) + self.assertIn("actualized_at", row) + self.assertIn("next_update_at", row) + self.assertIn("records_count", row) + self.assertIn("organizations_count", row) + self.assertIn("status", row) + self.assertIn("status_label", row) + self.assertIn("active_tasks", row) + + @override_settings( + FNS_WATCH_DIRECTORY="/tmp/mostovik-test-fns/watch", + FNS_PROCESSED_DIRECTORY="/tmp/mostovik-test-fns/processed", + FNS_FAILED_DIRECTORY="/tmp/mostovik-test-fns/failed", + ) + def test_refresh_creates_background_job_and_returns_task(self): + self.client.force_authenticate(self.admin) + response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "financial-indicators"}, + ), + {}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertTrue(response.data["success"]) + payload = response.data["data"] + self.assertEqual(payload["source_card"], "financial-indicators") + self.assertEqual(len(payload["tasks"]), 1) + + task_id = payload["tasks"][0]["task_id"] + self.assertTrue( + BackgroundJob.objects.filter( + task_id=task_id, + task_name="apps.parsers.tasks.scan_fns_directory", + user_id=self.admin.id, + ).exists() + ) + + def test_refresh_procurements_requires_region_code(self): + self.client.force_authenticate(self.admin) + response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "public-procurements"}, + ), + {}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertFalse(response.data["success"]) + + def test_refresh_forbidden_for_regular_user(self): + response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "financial-indicators"}, + ), + {}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) diff --git a/tests/apps/parsers/test_sources_api_e2e.py b/tests/apps/parsers/test_sources_api_e2e.py new file mode 100644 index 0000000..f206c58 --- /dev/null +++ b/tests/apps/parsers/test_sources_api_e2e.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +from apps.core.models import BackgroundJob, JobStatus +from apps.parsers.models import FinancialReport, FinancialReportLine, ParserLoadLog +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.parsers.factories import ( + IndustrialCertificateRecordFactory, + IndustrialProductRecordFactory, + InspectionRecordFactory, + ManufacturerRecordFactory, + ParserLoadLogFactory, + ProcurementRecordFactory, +) +from tests.apps.user.factories import UserFactory +from tests.utils.fixtures import fake + + +def _digits(length: int) -> str: + return "".join(str(fake.random_int(0, 9)) for _ in range(length)) + + +class SourcesApiE2ETest(APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + self.admin = UserFactory.create_user(is_staff=True) + + def test_source_cards_and_statuses_flow(self): + shared_inn = _digits(10) + report = FinancialReport.objects.create( + external_id=_digits(5), + ogrn=_digits(13), + file_name=f"fin_{_digits(5)}_{_digits(13)}.xlsx", + file_hash=fake.sha256(raw_output=False), + load_batch=1, + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + FinancialReportLine.objects.create( + report=report, + form_code="1", + line_code="1100", + line_name="Assets", + year=2025, + period_start=100, + period_end=200, + ) + IndustrialCertificateRecordFactory(inn=shared_inn) + IndustrialProductRecordFactory(inn=shared_inn) + ManufacturerRecordFactory(inn=shared_inn) + InspectionRecordFactory() + ProcurementRecordFactory(customer_inn=shared_inn) + ParserLoadLogFactory( + source=ParserLoadLog.Source.FNS_REPORTS, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.MANUFACTURES, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.PROCUREMENTS, + status="failed", + error_message="download error", + ) + BackgroundJob.objects.create( + task_id="job-inspections", + task_name="apps.parsers.tasks.sync_inspections", + status=JobStatus.STARTED, + progress=55, + progress_message="sync", + user_id=self.user.id, + meta={"source": ParserLoadLog.Source.INSPECTIONS}, + ) + + self.client.force_authenticate(self.user) + cards_response = self.client.get(reverse("api_v1:sources:source-cards-list")) + detail_response = self.client.get( + reverse( + "api_v1:sources:source-cards-detail", + kwargs={"slug": "manufacturers-and-products"}, + ) + ) + statuses_response = self.client.get( + reverse("api_v1:sources:source-cards-statuses") + ) + + self.assertEqual(cards_response.status_code, status.HTTP_200_OK) + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + self.assertEqual(statuses_response.status_code, status.HTTP_200_OK) + + cards = {item["slug"]: item for item in cards_response.data["data"]} + self.assertEqual(cards["financial-indicators"]["records_count"], 1) + self.assertEqual(cards["planned-inspections"]["status"], "in_progress") + self.assertEqual(cards["planned-inspections"]["progress"], 55) + self.assertEqual(cards["public-procurements"]["status"], "error") + self.assertEqual(cards["public-procurements"]["error_message"], "download error") + + minprom_card = detail_response.data["data"] + self.assertEqual(minprom_card["records_count"], 3) + self.assertEqual(minprom_card["organizations_count"], 1) + self.assertEqual(len(minprom_card["source_items"]), 3) + + statuses = {item["slug"]: item for item in statuses_response.data["data"]} + self.assertEqual(statuses["planned-inspections"]["progress"], 55) + self.assertEqual(statuses["public-procurements"]["status"], "error") + + def test_products_endpoint_supports_filter_and_search(self): + target = IndustrialProductRecordFactory( + registry_number="MPP-12345678", + product_name="Laser module", + full_organisation_name="Target Company", + ) + IndustrialProductRecordFactory( + registry_number="MPP-87654321", + product_name="Other product", + full_organisation_name="Another Company", + ) + + self.client.force_authenticate(self.user) + filtered_response = self.client.get( + reverse("api_v1:minpromtorg:industrial-products-list"), + {"registry_number": "MPP-12345678"}, + ) + search_response = self.client.get( + reverse("api_v1:minpromtorg:industrial-products-list"), + {"search": "Laser"}, + ) + detail_response = self.client.get( + reverse("api_v1:minpromtorg:industrial-products-detail", args=[target.id]) + ) + + self.assertEqual(filtered_response.status_code, status.HTTP_200_OK) + self.assertEqual(search_response.status_code, status.HTTP_200_OK) + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + self.assertEqual(len(filtered_response.data["data"]), 1) + self.assertEqual(filtered_response.data["data"][0]["id"], target.id) + self.assertEqual(len(search_response.data["data"]), 1) + self.assertEqual(search_response.data["data"][0]["id"], target.id) + self.assertEqual(detail_response.data["id"], target.id) + + def test_refresh_endpoints_cover_multiple_source_cards(self): + self.client.force_authenticate(self.admin) + + with patch( + "apps.parsers.source_cards.uuid.uuid4", + side_effect=[ + "request-minprom", + "task-industrial", + "task-products", + "task-manufactures", + "request-procurements", + "task-procurements", + ], + ): + with patch( + "apps.parsers.tasks.parse_industrial_production.apply_async", + return_value=SimpleNamespace(id="task-industrial"), + ): + with patch( + "apps.parsers.tasks.parse_industrial_products.apply_async", + return_value=SimpleNamespace(id="task-products"), + ): + with patch( + "apps.parsers.tasks.parse_manufactures.apply_async", + return_value=SimpleNamespace(id="task-manufactures"), + ): + minprom_response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "manufacturers-and-products"}, + ), + {}, + format="json", + ) + + with patch( + "apps.parsers.tasks.sync_procurements.apply_async", + return_value=SimpleNamespace(id="task-procurements"), + ): + procurements_response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "public-procurements"}, + ), + {"params": {"region_code": "77", "current_year": "2025"}}, + format="json", + ) + + self.assertEqual(minprom_response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(procurements_response.status_code, status.HTTP_202_ACCEPTED) + + minprom_tasks = minprom_response.data["data"]["tasks"] + self.assertEqual( + [item["task_id"] for item in minprom_tasks], + ["task-industrial", "task-products", "task-manufactures"], + ) + self.assertEqual( + procurements_response.data["data"]["tasks"][0]["task_id"], + "task-procurements", + ) + self.assertTrue( + BackgroundJob.objects.filter( + task_id="task-procurements", + task_name="apps.parsers.tasks.sync_procurements", + user_id=self.admin.id, + ).exists() + ) diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index f011596..bf60634 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -20,11 +20,16 @@ from apps.parsers.clients.minpromtorg.manufactures import ( ManufacturesClient, ManufacturesClientError, ) +from apps.parsers.clients.minpromtorg.products import ( + IndustrialProductsClient, + IndustrialProductsClientError, +) from apps.parsers.clients.proverki.client import ProverkiClientError from apps.parsers.clients.zakupki import ZakupkiClientError from apps.parsers.models import ( FinancialReport, IndustrialCertificateRecord, + IndustrialProductRecord, InspectionRecord, ManufacturerRecord, ParserLoadLog, @@ -38,6 +43,7 @@ from apps.parsers.tasks import ( parse_all_minpromtorg, parse_all_sources, parse_industrial_production, + parse_industrial_products, parse_inspections, parse_manufactures, parse_procurements, @@ -58,6 +64,7 @@ from tests.utils import TestHTTPServer from tests.utils.fixtures import ( build_minpromtorg_certificates_excel, build_minpromtorg_manufacturers_excel, + build_minpromtorg_products_excel, build_proverki_xml, build_zakupki_xml, build_zip, @@ -360,12 +367,14 @@ class MinpromtorgTasksTestCase(TestCase): def _add_minpromtorg_routes(self, server: TestHTTPServer): certificates_bytes, cert_rows = build_minpromtorg_certificates_excel(count=2) manufacturers_bytes, manuf_rows = build_minpromtorg_manufacturers_excel(count=2) + products_bytes, product_rows = build_minpromtorg_products_excel(count=2) date_str = fake.date_between(start_date="-30d", end_date="today").strftime( "%Y%m%d" ) cert_file = f"data_resolutions_{date_str}.xlsx" manuf_file = f"data_orgs_{date_str}.xlsx" + products_file = f"industrial_products_{date_str}.xlsx" server.add_json( "/api/kss-document-preview", @@ -379,6 +388,15 @@ class MinpromtorgTasksTestCase(TestCase): "name": ManufacturesClient().query, "files": [{"name": manuf_file, "url": f"/files/{manuf_file}"}], }, + { + "name": IndustrialProductsClient().query, + "files": [ + { + "name": products_file, + "url": f"/files/{products_file}", + } + ], + }, ] }, ) @@ -396,24 +414,33 @@ class MinpromtorgTasksTestCase(TestCase): "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ), ) - return cert_rows, manuf_rows + server.add_bytes( + f"/files/{products_file}", + products_bytes, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + return cert_rows, product_rows, manuf_rows def test_parse_all_minpromtorg_success(self): with TestHTTPServer() as server: - cert_rows, manuf_rows = self._add_minpromtorg_routes(server) + cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) result = parse_all_minpromtorg( proxies=[], client_adapter=server.adapter, ) self.assertIn("industrial", result) + self.assertIn("industrial_products", result) self.assertIn("manufactures", result) self.assertEqual(IndustrialCertificateRecord.objects.count(), len(cert_rows)) + self.assertEqual(IndustrialProductRecord.objects.count(), len(product_rows)) self.assertEqual(ManufacturerRecord.objects.count(), len(manuf_rows)) def test_parse_all_sources_success(self): with TestHTTPServer() as server: - cert_rows, manuf_rows = self._add_minpromtorg_routes(server) + cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) result = parse_all_sources( proxies=[], client_adapter=server.adapter, @@ -421,31 +448,43 @@ class MinpromtorgTasksTestCase(TestCase): ) self.assertIn("industrial", result) + self.assertIn("industrial_products", result) self.assertIn("manufactures", result) self.assertIn("inspections", result) self.assertEqual(IndustrialCertificateRecord.objects.count(), len(cert_rows)) + self.assertEqual(IndustrialProductRecord.objects.count(), len(product_rows)) self.assertEqual(ManufacturerRecord.objects.count(), len(manuf_rows)) self.assertEqual(InspectionRecord.objects.count(), 0) def test_parse_all_minpromtorg_without_adapter(self): with TestHTTPServer() as server: - cert_rows, manuf_rows = self._add_minpromtorg_routes(server) + cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) class _LocalIndustrialClient(IndustrialProductionClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) + class _LocalIndustrialProductsClient(IndustrialProductsClient): + def __init__(self, *args, **kwargs): + kwargs.setdefault("http_adapter", server.adapter) + super().__init__(*args, **kwargs) + class _LocalManufacturesClient(ManufacturesClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) original_industrial = parser_tasks.IndustrialProductionClient + original_industrial_products = parser_tasks.IndustrialProductsClient original_manufactures = parser_tasks.ManufacturesClient original_industrial_delay = parser_tasks.parse_industrial_production.delay + original_industrial_products_delay = ( + parser_tasks.parse_industrial_products.delay + ) original_manufactures_delay = parser_tasks.parse_manufactures.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient + parser_tasks.IndustrialProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): @@ -454,6 +493,12 @@ class MinpromtorgTasksTestCase(TestCase): kwargs=kwargs, ) + def _industrial_products_eager_delay(*args, **kwargs): + return parser_tasks.parse_industrial_products.apply( + args=args, + kwargs=kwargs, + ) + def _manufactures_eager_delay(*args, **kwargs): return parser_tasks.parse_manufactures.apply( args=args, @@ -461,42 +506,61 @@ class MinpromtorgTasksTestCase(TestCase): ) parser_tasks.parse_industrial_production.delay = _industrial_eager_delay + parser_tasks.parse_industrial_products.delay = ( + _industrial_products_eager_delay + ) parser_tasks.parse_manufactures.delay = _manufactures_eager_delay try: result = parse_all_minpromtorg(proxies=[]) finally: parser_tasks.IndustrialProductionClient = original_industrial + parser_tasks.IndustrialProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures parser_tasks.parse_industrial_production.delay = ( original_industrial_delay ) + parser_tasks.parse_industrial_products.delay = ( + original_industrial_products_delay + ) parser_tasks.parse_manufactures.delay = original_manufactures_delay self.assertIn("industrial", result) + self.assertIn("industrial_products", result) self.assertIn("manufactures", result) self.assertEqual(IndustrialCertificateRecord.objects.count(), len(cert_rows)) + self.assertEqual(IndustrialProductRecord.objects.count(), len(product_rows)) self.assertEqual(ManufacturerRecord.objects.count(), len(manuf_rows)) def test_parse_all_sources_without_adapter(self): with TestHTTPServer() as server: - cert_rows, manuf_rows = self._add_minpromtorg_routes(server) + cert_rows, product_rows, manuf_rows = self._add_minpromtorg_routes(server) class _LocalIndustrialClient(IndustrialProductionClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) + class _LocalIndustrialProductsClient(IndustrialProductsClient): + def __init__(self, *args, **kwargs): + kwargs.setdefault("http_adapter", server.adapter) + super().__init__(*args, **kwargs) + class _LocalManufacturesClient(ManufacturesClient): def __init__(self, *args, **kwargs): kwargs.setdefault("http_adapter", server.adapter) super().__init__(*args, **kwargs) original_industrial = parser_tasks.IndustrialProductionClient + original_industrial_products = parser_tasks.IndustrialProductsClient original_manufactures = parser_tasks.ManufacturesClient original_industrial_delay = parser_tasks.parse_industrial_production.delay + original_industrial_products_delay = ( + parser_tasks.parse_industrial_products.delay + ) original_manufactures_delay = parser_tasks.parse_manufactures.delay original_inspections_delay = parser_tasks.parse_inspections.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient + parser_tasks.IndustrialProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): @@ -505,6 +569,12 @@ class MinpromtorgTasksTestCase(TestCase): kwargs=kwargs, ) + def _industrial_products_eager_delay(*args, **kwargs): + return parser_tasks.parse_industrial_products.apply( + args=args, + kwargs=kwargs, + ) + def _manufactures_eager_delay(*args, **kwargs): return parser_tasks.parse_manufactures.apply( args=args, @@ -515,23 +585,32 @@ class MinpromtorgTasksTestCase(TestCase): return SimpleNamespace(id="inspections-test-task") parser_tasks.parse_industrial_production.delay = _industrial_eager_delay + parser_tasks.parse_industrial_products.delay = ( + _industrial_products_eager_delay + ) parser_tasks.parse_manufactures.delay = _manufactures_eager_delay parser_tasks.parse_inspections.delay = _inspections_stub_delay try: result = parse_all_sources(proxies=[], inspections_use_playwright=None) finally: parser_tasks.IndustrialProductionClient = original_industrial + parser_tasks.IndustrialProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures parser_tasks.parse_industrial_production.delay = ( original_industrial_delay ) + parser_tasks.parse_industrial_products.delay = ( + original_industrial_products_delay + ) parser_tasks.parse_manufactures.delay = original_manufactures_delay parser_tasks.parse_inspections.delay = original_inspections_delay self.assertIn("industrial", result) + self.assertIn("industrial_products", result) self.assertIn("manufactures", result) self.assertIn("inspections", result) self.assertEqual(IndustrialCertificateRecord.objects.count(), len(cert_rows)) + self.assertEqual(IndustrialProductRecord.objects.count(), len(product_rows)) self.assertEqual(ManufacturerRecord.objects.count(), len(manuf_rows)) def test_parse_industrial_production_failure(self): @@ -563,12 +642,47 @@ class MinpromtorgTasksTestCase(TestCase): def test_parse_industrial_production_with_default_proxies(self): with TestHTTPServer() as server: - cert_rows, _manuf_rows = self._add_minpromtorg_routes(server) + cert_rows, _product_rows, _manuf_rows = self._add_minpromtorg_routes(server) result = parse_industrial_production(client_adapter=server.adapter) self.assertEqual(result["status"], "success") self.assertEqual(IndustrialCertificateRecord.objects.count(), len(cert_rows)) + def test_parse_industrial_products_failure(self): + date_str = fake.date_between(start_date="-30d", end_date="today").strftime( + "%Y%m%d" + ) + products_file = f"industrial_products_{date_str}.xlsx" + + with TestHTTPServer() as server: + server.add_json( + "/api/kss-document-preview", + { + "data": [ + { + "name": IndustrialProductsClient().query, + "files": [ + { + "name": products_file, + "url": f"/files/{products_file}", + } + ], + } + ] + }, + ) + server.add_bytes("/files/" + products_file, b"not-an-excel") + with self.assertRaises(IndustrialProductsClientError): + parse_industrial_products(client_adapter=server.adapter) + + def test_parse_industrial_products_with_default_proxies(self): + with TestHTTPServer() as server: + _cert_rows, product_rows, _manuf_rows = self._add_minpromtorg_routes(server) + result = parse_industrial_products(client_adapter=server.adapter) + + self.assertEqual(result["status"], "success") + self.assertEqual(IndustrialProductRecord.objects.count(), len(product_rows)) + def test_parse_manufactures_failure(self): date_str = fake.date_between(start_date="-30d", end_date="today").strftime( "%Y%m%d" diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index b3d69cd..5ac584b 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -15,6 +15,7 @@ from rest_framework.test import APITestCase from tests.apps.parsers.factories import ( IndustrialCertificateRecordFactory, + IndustrialProductRecordFactory, InspectionRecordFactory, ManufacturerRecordFactory, ParserLoadLogFactory, @@ -87,6 +88,17 @@ class ParsersViewSetTest(APITestCase): ) self.assertEqual(detail.status_code, status.HTTP_200_OK) + def test_products_list_and_retrieve(self): + record = IndustrialProductRecordFactory() + self.client.force_authenticate(self.user) + url = reverse("api_v1:minpromtorg:industrial-products-list") + response = self.client.get(url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + detail = self.client.get( + reverse("api_v1:minpromtorg:industrial-products-detail", args=[record.id]) + ) + self.assertEqual(detail.status_code, status.HTTP_200_OK) + def test_inspections_list_and_retrieve(self): record = InspectionRecordFactory() self.client.force_authenticate(self.user) @@ -168,7 +180,7 @@ class ParsersViewSetTest(APITestCase): self.assertEqual(proxy_detail.status_code, status.HTTP_200_OK) def test_fns_upload_invalid_filename(self): - self.client.force_authenticate(self.user) + self.client.force_authenticate(self.admin) with tempfile.TemporaryDirectory() as tmpdir: watch_dir = os.path.join(tmpdir, "watch") processed_dir = os.path.join(tmpdir, "processed") @@ -191,3 +203,28 @@ class ParsersViewSetTest(APITestCase): url, {"files": [upload]}, format="multipart" ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_fns_upload_forbidden_for_regular_user(self): + self.client.force_authenticate(self.user) + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir = os.path.join(tmpdir, "watch") + processed_dir = os.path.join(tmpdir, "processed") + failed_dir = os.path.join(tmpdir, "failed") + content = _build_fns_excel_bytes() + upload = SimpleUploadedFile( + f"fin_{_digits(5)}_{_digits(13)}.xlsx", + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + with self.settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ): + url = reverse("api_v1:fns:fns-upload") + response = self.client.post( + url, {"files": [upload]}, format="multipart" + ) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) diff --git a/tests/apps/registers/factories.py b/tests/apps/registers/factories.py index 676b55b..55d5e5b 100644 --- a/tests/apps/registers/factories.py +++ b/tests/apps/registers/factories.py @@ -31,9 +31,7 @@ class OrganizationFactory(factory.django.DjangoModelFactory): pn_name = factory.LazyAttribute(lambda _: fake.company()) mn_ogrn = factory.Sequence(lambda n: 10_000_000_000_000 + n) mn_inn = factory.Sequence(lambda n: 1_000_000_000 + n) - in_kpp = factory.LazyAttribute( - lambda _: fake.random_number(digits=9, fix_len=True) - ) + in_kpp = factory.LazyAttribute(lambda _: fake.random_number(digits=9, fix_len=True)) mn_okpo = factory.LazyAttribute( lambda _: str(fake.random_number(digits=8, fix_len=True)) ) @@ -62,5 +60,7 @@ class RegistryMembershipPeriodFactory(factory.django.DjangoModelFactory): organization = factory.SubFactory(OrganizationFactory) started_at = factory.LazyFunction(timezone.localdate) ended_at = None - started_by_upload = factory.SubFactory(RegisterUploadFactory, registry=factory.SelfAttribute("..registry")) + started_by_upload = factory.SubFactory( + RegisterUploadFactory, registry=factory.SelfAttribute("..registry") + ) ended_by_upload = None diff --git a/tests/apps/registers/test_models.py b/tests/apps/registers/test_models.py new file mode 100644 index 0000000..f2e0700 --- /dev/null +++ b/tests/apps/registers/test_models.py @@ -0,0 +1,32 @@ +"""Tests for registers models.""" + +from django.test import TestCase + +from tests.apps.registers.factories import ( + OrganizationFactory, + RegisterFactory, + RegisterUploadFactory, + RegistryMembershipPeriodFactory, +) + + +class RegistersModelsTest(TestCase): + def test_string_representations(self): + registry = RegisterFactory(name="Реестр ОПК") + organization = OrganizationFactory( + pn_name="Очень длинное наименование организации для проверки строкового представления", + mn_ogrn=1027700118984, + mn_inn=7702000406, + ) + upload = RegisterUploadFactory(registry=registry) + period = RegistryMembershipPeriodFactory( + registry=registry, + organization=organization, + started_by_upload=upload, + ) + + self.assertEqual(str(registry), "Реестр ОПК") + self.assertIn("1027700118984/7702000406", str(organization)) + self.assertEqual(str(upload), f"{registry.name} @ {upload.actual_date}") + self.assertIn(registry.name, str(period)) + self.assertIn(str(period.started_at), str(period)) diff --git a/tests/apps/registers/test_serializers.py b/tests/apps/registers/test_serializers.py new file mode 100644 index 0000000..205a5fc --- /dev/null +++ b/tests/apps/registers/test_serializers.py @@ -0,0 +1,59 @@ +"""Tests for registers serializers.""" + +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase + +from apps.registers.serializers import ( + OrganizationListQuerySerializer, + RegisterFileUploadSerializer, + RegistryOrganizationListQuerySerializer, +) +from tests.apps.registers.factories import RegisterFactory + + +class RegisterFileUploadSerializerTest(TestCase): + def test_rejects_non_xlsx_file(self): + registry = RegisterFactory() + serializer = RegisterFileUploadSerializer( + data={ + "registry": str(registry.id), + "file": SimpleUploadedFile("registry.csv", b"csv"), + } + ) + + self.assertFalse(serializer.is_valid()) + self.assertIn("file", serializer.errors) + + +class OrganizationListQuerySerializerTest(TestCase): + def test_actual_date_requires_registry(self): + serializer = OrganizationListQuerySerializer( + data={"actual_date": "2026-03-17"} + ) + + self.assertFalse(serializer.is_valid()) + self.assertIn("actual_date", serializer.errors) + + def test_rejects_non_digit_okpo(self): + serializer = OrganizationListQuerySerializer(data={"mn_okpo": "12AB"}) + + self.assertFalse(serializer.is_valid()) + self.assertIn("mn_okpo", serializer.errors) + + def test_accepts_digit_only_okpo(self): + serializer = OrganizationListQuerySerializer(data={"mn_okpo": "12345678"}) + + self.assertTrue(serializer.is_valid(), serializer.errors) + + +class RegistryOrganizationListQuerySerializerTest(TestCase): + def test_rejects_non_digit_okpo(self): + serializer = RegistryOrganizationListQuerySerializer(data={"mn_okpo": "OKPO-1"}) + + self.assertFalse(serializer.is_valid()) + self.assertIn("mn_okpo", serializer.errors) + + def test_accepts_digit_only_okpo(self): + serializer = RegistryOrganizationListQuerySerializer(data={"mn_okpo": "87654321"}) + + self.assertTrue(serializer.is_valid(), serializer.errors) diff --git a/tests/apps/registers/test_services.py b/tests/apps/registers/test_services.py new file mode 100644 index 0000000..b919910 --- /dev/null +++ b/tests/apps/registers/test_services.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import io +from datetime import date + +from apps.registers.services import ParsedOrganization, RegisterImportError, RegisterImportService +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase +from openpyxl import Workbook + +from tests.apps.registers.factories import ( + OrganizationFactory, + RegisterFactory, + RegisterUploadFactory, + RegistryMembershipPeriodFactory, +) + + +def _build_workbook(rows: list[list[object]]) -> bytes: + workbook = Workbook() + worksheet = workbook.active + for row in rows: + worksheet.append(row) + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + return buffer.getvalue() + + +def _upload(name: str, rows: list[list[object]]) -> SimpleUploadedFile: + return SimpleUploadedFile( + name, + _build_workbook(rows), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + +class RegisterImportServiceTest(TestCase): + def test_update_organization_fields_returns_false_when_nothing_changed(self): + organization = OrganizationFactory( + pn_name="Org", + in_kpp=123456789, + mn_okpo="12345678", + ) + row = ParsedOrganization( + pn_name="Org", + mn_ogrn=organization.mn_ogrn, + mn_inn=organization.mn_inn, + in_kpp=123456789, + mn_okpo="12345678", + ) + + updated = RegisterImportService._update_organization_fields( + organization=organization, + row=row, + ) + + self.assertFalse(updated) + + def test_update_organization_fields_updates_changed_values(self): + organization = OrganizationFactory( + pn_name="Old", + in_kpp=123456789, + mn_okpo="12345678", + ) + row = ParsedOrganization( + pn_name="New", + mn_ogrn=organization.mn_ogrn, + mn_inn=organization.mn_inn, + in_kpp=987654321, + mn_okpo="87654321", + ) + + updated = RegisterImportService._update_organization_fields( + organization=organization, + row=row, + ) + + self.assertTrue(updated) + organization.refresh_from_db() + self.assertEqual(organization.pn_name, "New") + self.assertEqual(organization.in_kpp, 987654321) + self.assertEqual(organization.mn_okpo, "87654321") + + def test_get_active_periods_by_org_returns_mapping(self): + registry = RegisterFactory() + active_period = RegistryMembershipPeriodFactory(registry=registry, ended_at=None) + RegistryMembershipPeriodFactory( + registry=registry, + started_at=date(2026, 1, 1), + ended_at=date(2026, 2, 1), + ) + + active_by_org = RegisterImportService._get_active_periods_by_org(registry) + + self.assertEqual(list(active_by_org.keys()), [active_period.organization_id]) + + def test_close_missing_periods_updates_and_deletes(self): + registry = RegisterFactory() + upload = RegisterUploadFactory(registry=registry, actual_date=date(2026, 3, 1)) + active_period = RegistryMembershipPeriodFactory( + registry=registry, + started_at=date(2026, 2, 1), + ended_at=None, + ) + same_day_period = RegistryMembershipPeriodFactory( + registry=registry, + started_at=date(2026, 3, 1), + ended_at=None, + ) + + closed = RegisterImportService._close_missing_periods( + active_by_org={ + active_period.organization_id: active_period, + same_day_period.organization_id: same_day_period, + }, + snapshot_org_ids=set(), + snapshot_date=date(2026, 3, 1), + upload=upload, + ) + + self.assertEqual(closed, 2) + active_period.refresh_from_db() + self.assertEqual(active_period.ended_at, date(2026, 3, 1)) + self.assertFalse( + active_period.__class__.objects.filter(id=same_day_period.id).exists() + ) + + def test_open_new_periods_returns_zero_when_all_active(self): + registry = RegisterFactory() + upload = RegisterUploadFactory(registry=registry) + organization = OrganizationFactory() + + opened = RegisterImportService._open_new_periods( + registry=registry, + snapshot_org_ids={organization.id}, + active_org_ids={organization.id}, + snapshot_date=date(2026, 3, 1), + upload=upload, + ) + + self.assertEqual(opened, 0) + + def test_open_new_periods_creates_missing_memberships(self): + registry = RegisterFactory() + upload = RegisterUploadFactory(registry=registry) + organization = OrganizationFactory() + + opened = RegisterImportService._open_new_periods( + registry=registry, + snapshot_org_ids={organization.id}, + active_org_ids=set(), + snapshot_date=date(2026, 3, 1), + upload=upload, + ) + + self.assertEqual(opened, 1) + + def test_resolve_actual_date_returns_requested_latest_or_today(self): + registry = RegisterFactory() + requested = RegisterImportService.resolve_actual_date( + registry=registry, + requested_date=date(2026, 3, 10), + ) + self.assertEqual(requested, date(2026, 3, 10)) + + RegisterUploadFactory(registry=registry, actual_date=date(2026, 2, 1)) + latest_upload = RegisterUploadFactory( + registry=registry, + actual_date=date(2026, 3, 5), + ) + latest = RegisterImportService.resolve_actual_date( + registry=registry, + requested_date=None, + ) + self.assertEqual(latest, latest_upload.actual_date) + + empty_registry = RegisterFactory() + today = RegisterImportService.resolve_actual_date( + registry=empty_registry, + requested_date=None, + ) + self.assertEqual(today, date.today()) + + def test_get_organizations_queryset_applies_registry_filters_and_search(self): + registry = RegisterFactory() + upload = RegisterUploadFactory(registry=registry, actual_date=date(2026, 3, 1)) + match = OrganizationFactory(pn_name="Target organization", mn_okpo="11111111") + other = OrganizationFactory(pn_name="Other company", mn_okpo="22222222") + RegistryMembershipPeriodFactory( + registry=registry, + organization=match, + started_at=date(2026, 3, 1), + started_by_upload=upload, + ) + RegistryMembershipPeriodFactory( + registry=registry, + organization=other, + started_at=date(2026, 1, 1), + ended_at=date(2026, 2, 1), + started_by_upload=upload, + ended_by_upload=upload, + ) + + queryset, resolved_date = RegisterImportService.get_organizations_queryset( + registry=registry, + search="Target", + mn_okpo="11111111", + ) + + self.assertEqual(resolved_date, date(2026, 3, 1)) + self.assertEqual(list(queryset.values_list("id", flat=True)), [match.id]) + + def test_get_registry_organizations_queryset_applies_exact_filters(self): + registry = RegisterFactory() + upload = RegisterUploadFactory(registry=registry, actual_date=date(2026, 3, 1)) + organization = OrganizationFactory() + RegistryMembershipPeriodFactory( + registry=registry, + organization=organization, + started_at=date(2026, 3, 1), + started_by_upload=upload, + ) + + queryset, resolved_date = RegisterImportService.get_registry_organizations_queryset( + registry=registry, + mn_ogrn=organization.mn_ogrn, + mn_inn=organization.mn_inn, + in_kpp=organization.in_kpp, + mn_okpo=organization.mn_okpo, + ) + + self.assertEqual(resolved_date, date(2026, 3, 1)) + self.assertEqual(list(queryset.values_list("id", flat=True)), [organization.id]) + + def test_parse_xlsx_parses_rows_and_supports_optional_kpp(self): + upload = _upload( + "register.xlsx", + [ + ["pn_name", "mn_ogrn", "mn_inn", "mn_okpo"], + ["Org", "1027700118984", "7702000000", "12345678"], + ], + ) + + rows = RegisterImportService.parse_xlsx(upload) + + self.assertEqual( + rows, + [ + ParsedOrganization( + pn_name="Org", + mn_ogrn=1027700118984, + mn_inn=7702000000, + in_kpp=None, + mn_okpo="12345678", + ) + ], + ) + + def test_parse_xlsx_raises_on_invalid_workbook_headers_and_empty_rows(self): + broken = SimpleUploadedFile("broken.xlsx", b"not-an-excel") + with self.assertRaisesMessage(RegisterImportError, "Не удалось прочитать Excel файл"): + RegisterImportService.parse_xlsx(broken) + + no_headers = _upload("no-headers.xlsx", []) + with self.assertRaisesMessage(RegisterImportError, "Файл не содержит заголовков"): + RegisterImportService.parse_xlsx(no_headers) + + missing_headers = _upload("missing.xlsx", [["pn_name", "mn_ogrn"]]) + with self.assertRaisesMessage(RegisterImportError, "Отсутствуют обязательные колонки"): + RegisterImportService.parse_xlsx(missing_headers) + + empty_rows = _upload( + "empty.xlsx", + [["pn_name", "mn_ogrn", "mn_inn", "mn_okpo"], ["", "", "", ""]], + ) + with self.assertRaisesMessage( + RegisterImportError, + "Файл не содержит строк с организациями", + ): + RegisterImportService.parse_xlsx(empty_rows) + + def test_validate_snapshot_date_and_unique_identities(self): + registry = RegisterFactory() + RegisterUploadFactory(registry=registry, actual_date=date(2026, 3, 5)) + + with self.assertRaisesMessage( + RegisterImportError, + "Дата актуальности не может быть раньше последней загрузки", + ): + RegisterImportService._validate_snapshot_date( + registry=registry, + snapshot_date=date(2026, 3, 1), + ) + + row = ParsedOrganization( + pn_name="Org", + mn_ogrn=1, + mn_inn=2, + in_kpp=None, + mn_okpo="12345678", + ) + with self.assertRaisesMessage( + RegisterImportError, + "Файл содержит дубли по ключу", + ): + RegisterImportService._ensure_unique_identities([row, row]) + + def test_scalar_parsers_validate_required_and_numeric_values(self): + self.assertEqual(RegisterImportService._normalize_header(" Mn_Inn "), "mn_inn") + self.assertTrue(RegisterImportService._is_empty_row((None, " "))) + self.assertFalse(RegisterImportService._is_empty_row((None, "x"))) + self.assertEqual( + RegisterImportService._as_required_text(" Org ", field_name="pn_name", row_number=2), + "Org", + ) + self.assertEqual( + RegisterImportService._as_required_int("123.0", field_name="mn_inn", row_number=2), + 123, + ) + self.assertIsNone( + RegisterImportService._as_optional_int(" ", field_name="in_kpp", row_number=2) + ) + self.assertEqual( + RegisterImportService._as_numeric_text("123 456.0", field_name="mn_okpo", row_number=2), + "123456", + ) + + with self.assertRaisesMessage(RegisterImportError, "поле pn_name обязательно"): + RegisterImportService._as_required_text("", field_name="pn_name", row_number=2) + with self.assertRaisesMessage(RegisterImportError, "поле mn_inn обязательно"): + RegisterImportService._as_required_int(None, field_name="mn_inn", row_number=2) + with self.assertRaisesMessage(RegisterImportError, "поле in_kpp должно быть числом"): + RegisterImportService._as_optional_int("abc", field_name="in_kpp", row_number=2) + with self.assertRaisesMessage( + RegisterImportError, + "поле mn_okpo должно содержать только цифры", + ): + RegisterImportService._as_numeric_text("ab12", field_name="mn_okpo", row_number=2) diff --git a/tests/apps/registers/test_views.py b/tests/apps/registers/test_views.py index 4330cb8..214bfb0 100644 --- a/tests/apps/registers/test_views.py +++ b/tests/apps/registers/test_views.py @@ -61,6 +61,7 @@ def _extract_results(response_data): class RegistersViewsTest(APITestCase): def setUp(self): self.user = UserFactory.create_user() + self.admin = UserFactory.create_user(is_staff=True) self.client.force_authenticate(self.user) def _post_upload( @@ -72,6 +73,7 @@ class RegistersViewsTest(APITestCase): with_kpp: bool = True, file_name: str = "registry.xlsx", ): + self.client.force_authenticate(self.admin) content = _build_register_excel_bytes(rows, with_kpp=with_kpp) upload = SimpleUploadedFile( file_name, @@ -80,7 +82,7 @@ class RegistersViewsTest(APITestCase): "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ), ) - return self.client.post( + response = self.client.post( reverse("api_v1:registers:register-upload"), { "registry": str(registry.id), @@ -89,6 +91,8 @@ class RegistersViewsTest(APITestCase): }, format="multipart", ) + self.client.force_authenticate(self.user) + return response def test_registries_list_and_retrieve(self): registry = RegisterFactory(name="Росатом") @@ -148,8 +152,12 @@ class RegistersViewsTest(APITestCase): organization_old = OrganizationFactory() organization_current = OrganizationFactory() - upload_start = RegisterUploadFactory(registry=registry, actual_date=date(2026, 1, 1)) - upload_end = RegisterUploadFactory(registry=registry, actual_date=date(2026, 2, 1)) + upload_start = RegisterUploadFactory( + registry=registry, actual_date=date(2026, 1, 1) + ) + upload_end = RegisterUploadFactory( + registry=registry, actual_date=date(2026, 2, 1) + ) RegistryMembershipPeriodFactory( registry=registry, @@ -260,7 +268,9 @@ class RegistersViewsTest(APITestCase): self.assertEqual(third.data["opened_periods"], 1) self.assertEqual(third.data["closed_periods"], 1) - organization_a = Organization.objects.get(mn_ogrn=1027600980990, mn_inn=7601000086) + organization_a = Organization.objects.get( + mn_ogrn=1027600980990, mn_inn=7601000086 + ) periods = list( RegistryMembershipPeriod.objects.filter( registry=registry, @@ -303,10 +313,14 @@ class RegistersViewsTest(APITestCase): self.assertEqual(upload_b.status_code, status.HTTP_201_CREATED) response_a = self.client.get( - reverse("api_v1:registers:registry-organizations-list", args=[registry_a.id]) + reverse( + "api_v1:registers:registry-organizations-list", args=[registry_a.id] + ) ) response_b = self.client.get( - reverse("api_v1:registers:registry-organizations-list", args=[registry_b.id]) + reverse( + "api_v1:registers:registry-organizations-list", args=[registry_b.id] + ) ) self.assertEqual(response_a.status_code, status.HTTP_200_OK) @@ -357,7 +371,9 @@ class RegistersViewsTest(APITestCase): ) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - organization = Organization.objects.get(mn_ogrn=1027600980990, mn_inn=7601000086) + organization = Organization.objects.get( + mn_ogrn=1027600980990, mn_inn=7601000086 + ) self.assertIsNone(organization.in_kpp) def test_upload_rejects_invalid_okpo(self): @@ -414,3 +430,37 @@ class RegistersViewsTest(APITestCase): ) self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_upload_forbidden_for_regular_user(self): + registry = RegisterFactory(name="Только для админа") + content = _build_register_excel_bytes( + [ + { + "pn_name": 'АО "Ограниченный доступ"', + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07506197", + } + ] + ) + upload = SimpleUploadedFile( + "viewer.xlsx", + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + + self.client.force_authenticate(self.user) + response = self.client.post( + reverse("api_v1:registers:register-upload"), + { + "registry": str(registry.id), + "actual_date": "2026-01-01", + "file": upload, + }, + format="multipart", + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) diff --git a/tests/apps/user/test_admin.py b/tests/apps/user/test_admin.py index 0aed11d..e9ec230 100644 --- a/tests/apps/user/test_admin.py +++ b/tests/apps/user/test_admin.py @@ -12,6 +12,10 @@ from django.test import RequestFactory, TestCase from tests.utils.fixtures import fake +def _password() -> str: + return fake.password(length=12, special_chars=False) + + class UserAdminTest(TestCase): def setUp(self): self.site = AdminSite() @@ -23,7 +27,7 @@ class UserAdminTest(TestCase): request.user = User.objects.create_superuser( email=fake.email(), username=fake.user_name(), - password="pass", + password=_password(), ) request.session = {} request._messages = FallbackStorage(request) @@ -33,13 +37,13 @@ class UserAdminTest(TestCase): verified = User.objects.create_user( email=fake.email(), username=fake.user_name(), - password="pass", + password=_password(), is_verified=True, ) unverified = User.objects.create_user( email=fake.email(), username=fake.user_name(), - password="pass", + password=_password(), is_verified=False, ) self.assertIn("span", str(self.admin.is_verified_badge(verified))) @@ -53,7 +57,7 @@ class UserAdminTest(TestCase): def test_actions(self): users = [ User.objects.create_user( - email=fake.email(), username=fake.user_name(), password="pass" + email=fake.email(), username=fake.user_name(), password=_password() ) for _ in range(2) ] @@ -80,7 +84,7 @@ class ProfileAdminTest(TestCase): def test_has_avatar_badge(self): user = User.objects.create_user( - email=fake.email(), username=fake.user_name(), password="pass" + email=fake.email(), username=fake.user_name(), password=_password() ) profile = user.profile self.assertIn("span", str(self.admin.has_avatar(profile))) diff --git a/tests/apps/user/test_serializers.py b/tests/apps/user/test_serializers.py index 86305e1..12c7f9e 100644 --- a/tests/apps/user/test_serializers.py +++ b/tests/apps/user/test_serializers.py @@ -1,8 +1,11 @@ """Tests for user serializers""" from apps.user.serializers import ( + AdminUserCreateSerializer, + AdminUserUpdateSerializer, LoginSerializer, PasswordChangeSerializer, + PasswordResetConfirmSerializer, ProfileUpdateSerializer, TokenSerializer, UserRegistrationSerializer, @@ -110,6 +113,9 @@ class UserSerializerTest(TestCase): self.assertEqual(data["email"], self.user.email) self.assertEqual(data["username"], self.user.username) self.assertEqual(data["phone"], self.user.phone) + self.assertEqual(data["role"], "user") + self.assertEqual(data["role_label"], "Пользователь") + self.assertIn("capabilities", data) self.assertEqual(data["is_verified"], self.user.is_verified) self.assertIn("profile", data) self.assertIn("created_at", data) @@ -117,7 +123,16 @@ class UserSerializerTest(TestCase): def test_read_only_fields(self): """Test that read-only fields are not writable""" - read_only_fields = ["id", "is_verified", "created_at", "updated_at"] + read_only_fields = [ + "id", + "is_active", + "is_verified", + "role", + "role_label", + "capabilities", + "created_at", + "updated_at", + ] serializer = UserSerializer() for field_name in read_only_fields: @@ -152,6 +167,45 @@ class UserUpdateSerializerTest(TestCase): self.assertEqual(set(serializer.Meta.fields), set(allowed_fields)) +class AdminUserCreateSerializerTest(TestCase): + """Tests for AdminUserCreateSerializer.""" + + def test_valid_admin_user_create_data(self): + serializer = AdminUserCreateSerializer( + data={ + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "phone": f"+7{fake.numerify('##########')}", + "password": fake.password(length=12, special_chars=False), + "role": "user", + "is_active": True, + "is_verified": False, + "first_name": fake.first_name(), + "last_name": fake.last_name(), + } + ) + self.assertTrue(serializer.is_valid(), serializer.errors) + + +class AdminUserUpdateSerializerTest(TestCase): + """Tests for AdminUserUpdateSerializer.""" + + def setUp(self): + self.user = UserFactory.create_user() + + def test_valid_admin_user_update_data(self): + serializer = AdminUserUpdateSerializer( + self.user, + data={ + "role": "admin", + "is_active": False, + "first_name": fake.first_name(), + }, + partial=True, + ) + self.assertTrue(serializer.is_valid(), serializer.errors) + + class ProfileUpdateSerializerTest(TestCase): """Tests for ProfileUpdateSerializer""" @@ -289,3 +343,31 @@ class PasswordChangeSerializerTest(TestCase): serializer = PasswordChangeSerializer(data=data) self.assertFalse(serializer.is_valid()) self.assertIn("old_password", serializer.errors) + + +class PasswordResetConfirmSerializerTest(TestCase): + """Tests for PasswordResetConfirmSerializer.""" + + def test_valid_payload(self): + new_password = fake.password(length=12, special_chars=False) + serializer = PasswordResetConfirmSerializer( + data={ + "token": fake.sha1(raw_output=False), + "new_password": new_password, + "new_password_confirm": new_password, + } + ) + + self.assertTrue(serializer.is_valid(), serializer.errors) + + def test_passwords_must_match(self): + serializer = PasswordResetConfirmSerializer( + data={ + "token": fake.sha1(raw_output=False), + "new_password": fake.password(length=12, special_chars=False), + "new_password_confirm": fake.password(length=12, special_chars=False), + } + ) + + self.assertFalse(serializer.is_valid()) + self.assertIn("non_field_errors", serializer.errors) diff --git a/tests/apps/user/test_services.py b/tests/apps/user/test_services.py index 4c1d030..8222725 100644 --- a/tests/apps/user/test_services.py +++ b/tests/apps/user/test_services.py @@ -33,6 +33,8 @@ class UserServiceTest(TestCase): self.assertEqual(user.username, self.user_data["username"]) self.assertTrue(user.check_password(self.user_data["password"])) self.assertFalse(user.is_verified) # Default value + self.assertEqual(UserService.get_user_role(user), UserService.ROLE_USER) + self.assertTrue(user.groups.filter(name=UserService.ROLE_USER).exists()) def test_create_user_with_extra_fields(self): """Test user creation with extra fields""" @@ -47,6 +49,14 @@ class UserServiceTest(TestCase): self.assertEqual(user.phone, extra_data["phone"]) self.assertTrue(user.is_verified) + def test_create_user_with_admin_role(self): + """Test admin role assignment on creation.""" + user = UserService.create_user(role=UserService.ROLE_ADMIN, **self.user_data) + + self.assertTrue(user.is_staff) + self.assertEqual(UserService.get_user_role(user), UserService.ROLE_ADMIN) + self.assertTrue(user.groups.filter(name=UserService.ROLE_ADMIN).exists()) + def test_get_user_by_email_found(self): """Test getting user by existing email""" found_user = UserService.get_user_by_email(self.user.email) @@ -106,6 +116,31 @@ class UserServiceTest(TestCase): with self.assertRaises(NotFoundError): UserService.update_user(nonexistent_id, username=fake.user_name()) + def test_update_managed_user_updates_role_and_profile(self): + """Test admin update flow changes role and profile names.""" + updated_user = UserService.update_managed_user( + self.user.id, + role=UserService.ROLE_ADMIN, + first_name="Иван", + last_name="Иванов", + ) + + self.assertEqual( + UserService.get_user_role(updated_user), UserService.ROLE_ADMIN + ) + self.assertTrue(updated_user.is_staff) + self.assertEqual(updated_user.profile.first_name, "Иван") + self.assertEqual(updated_user.profile.last_name, "Иванов") + + def test_update_managed_user_updates_password(self): + raw_password = fake.password(length=12, special_chars=False) + updated_user = UserService.update_managed_user( + self.user.id, + password=raw_password, + ) + + self.assertTrue(updated_user.check_password(raw_password)) + def test_delete_user_success(self): """Test successful user deletion""" user_id = self.user.id @@ -121,6 +156,47 @@ class UserServiceTest(TestCase): with self.assertRaises(NotFoundError): UserService.delete_user(nonexistent_id) + def test_deactivate_user_success(self): + """Test soft deactivation of user.""" + user = UserService.deactivate_user(self.user.id) + self.assertFalse(user.is_active) + + def test_get_user_capabilities_for_admin(self): + """Test admin capabilities set.""" + admin = UserFactory.create_user(is_staff=True) + capabilities = UserService.get_user_capabilities(admin) + + self.assertTrue(capabilities["can_manage_users"]) + self.assertTrue(capabilities["can_refresh_dashboard"]) + self.assertIn("exchange", capabilities["settings_sections"]) + + def test_get_user_capabilities_for_regular_user(self): + """Test regular user capabilities set.""" + capabilities = UserService.get_user_capabilities(self.user) + + self.assertFalse(capabilities["can_manage_users"]) + self.assertFalse(capabilities["can_refresh_dashboard"]) + self.assertEqual(capabilities["settings_sections"], []) + + def test_get_user_role_uses_admin_group(self): + user = UserFactory.create_user(is_staff=False, is_superuser=False) + admin_group = UserService.ensure_role_groups()[UserService.ROLE_ADMIN] + user.groups.add(admin_group) + + self.assertEqual(UserService.get_user_role(user), UserService.ROLE_ADMIN) + + def test_assign_role_replaces_previous_role_group(self): + user = UserFactory.create_user() + + UserService.assign_role(user, UserService.ROLE_ADMIN) + + self.assertTrue(user.groups.filter(name=UserService.ROLE_ADMIN).exists()) + self.assertFalse(user.groups.filter(name=UserService.ROLE_USER).exists()) + + def test_assign_role_rejects_unknown_role(self): + with self.assertRaisesMessage(ValueError, "Unsupported role: root"): + UserService.assign_role(self.user, "root") + def test_get_tokens_for_user(self): """Test JWT token generation""" tokens = UserService.get_tokens_for_user(self.user) diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index 344641b..a13e063 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -117,6 +117,24 @@ class LoginViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + def test_login_invalid_payload_returns_400(self): + response = self.client.post(self.login_url, {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("username", response.data) + + +class LogoutViewTest(APITestCase): + def test_logout_returns_success_message(self): + user = UserFactory.create_user() + tokens = UserService.get_tokens_for_user(user) + self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {tokens['access']}") + + response = self.client.post(reverse("api_v1:user:logout"), {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["message"], "Успешный выход") + class CurrentUserViewTest(APITestCase): """Tests for CurrentUserView""" @@ -135,6 +153,8 @@ class CurrentUserViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["id"], self.user.id) self.assertEqual(response.data["email"], self.user.email) + self.assertEqual(response.data["role"], "user") + self.assertFalse(response.data["capabilities"]["can_refresh_dashboard"]) self.assertIn("profile", response.data) def test_get_current_user_unauthenticated(self): @@ -178,6 +198,125 @@ class UserUpdateViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + def test_update_user_invalid_returns_400(self): + response = self.client.patch(self.update_url, {"username": ""}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("username", response.data) + + +class AdminUserManagementViewTest(APITestCase): + """Tests for admin-only user management endpoints.""" + + def setUp(self): + self.admin = UserFactory.create_user(is_staff=True) + self.user = UserFactory.create_user() + self.tokens = UserService.get_tokens_for_user(self.admin) + self.user_tokens = UserService.get_tokens_for_user(self.user) + self.list_url = reverse("api_v1:user:admin-users") + self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {self.tokens['access']}") + + def test_admin_can_list_users(self): + response = self.client.get(self.list_url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + usernames = {item["username"] for item in response.data} + self.assertIn(self.admin.username, usernames) + self.assertIn(self.user.username, usernames) + + def test_admin_can_create_user_with_role(self): + password = fake.password(length=12, special_chars=False) + payload = { + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "phone": f"+7{fake.numerify('##########')}", + "password": password, + "role": "admin", + "first_name": "Петр", + "last_name": "Петров", + } + + response = self.client.post(self.list_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + created = User.objects.get(username=payload["username"]) + self.assertTrue(created.is_staff) + self.assertEqual(response.data["role"], "admin") + self.assertEqual(created.profile.first_name, "Петр") + + def test_admin_can_update_user_and_role(self): + url = reverse("api_v1:user:admin-user-detail", args=[self.user.id]) + + response = self.client.patch( + url, + {"role": "admin", "first_name": "Иван", "is_verified": True}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.user.refresh_from_db() + self.assertTrue(self.user.is_staff) + self.assertTrue(self.user.is_verified) + self.assertEqual(self.user.profile.first_name, "Иван") + + def test_admin_can_get_user_detail(self): + url = reverse("api_v1:user:admin-user-detail", args=[self.user.id]) + + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["id"], self.user.id) + + def test_admin_cannot_patch_self_to_inactive(self): + url = reverse("api_v1:user:admin-user-detail", args=[self.admin.id]) + + response = self.client.patch(url, {"is_active": False}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.data["detail"], "Нельзя деактивировать самого себя.") + + def test_admin_cannot_patch_self_to_regular_user(self): + url = reverse("api_v1:user:admin-user-detail", args=[self.admin.id]) + + response = self.client.patch(url, {"role": "user"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.data["detail"], "Нельзя снять у себя роль администратора." + ) + + def test_admin_can_patch_self_with_safe_fields(self): + url = reverse("api_v1:user:admin-user-detail", args=[self.admin.id]) + + response = self.client.patch(url, {"first_name": "Админ"}, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["profile"]["first_name"], "Админ") + + def test_admin_can_deactivate_user(self): + url = reverse("api_v1:user:admin-user-deactivate", args=[self.user.id]) + + response = self.client.post(url, {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.user.refresh_from_db() + self.assertFalse(self.user.is_active) + + def test_admin_cannot_deactivate_self(self): + url = reverse("api_v1:user:admin-user-deactivate", args=[self.admin.id]) + + response = self.client.post(url, {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_regular_user_cannot_access_admin_user_management(self): + self.client.credentials( + HTTP_AUTHORIZATION=f"Bearer {self.user_tokens['access']}" + ) + + response = self.client.get(self.list_url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + class ProfileDetailViewTest(APITestCase): """Tests for ProfileDetailView""" @@ -225,6 +364,22 @@ class ProfileDetailViewTest(APITestCase): # Profile should be created automatically self.assertTrue(Profile.objects.filter(user=self.user).exists()) + def test_update_profile_invalid_returns_400(self): + response = self.client.patch( + self.profile_url, + {"date_of_birth": "not-a-date"}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("date_of_birth", response.data) + + def test_get_full_profile_endpoint(self): + response = self.client.get(reverse("api_v1:user:profile_full")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["id"], self.user.id) + class PasswordChangeViewTest(APITestCase): """Tests for PasswordChangeView""" @@ -316,6 +471,20 @@ class TokenRefreshViewTest(APITestCase): self.assertTrue("errors" in response.data or "detail" in response.data) +class TokenVerifyViewTest(APITestCase): + def test_verify_access_token_success(self): + user = UserFactory.create_user() + tokens = UserService.get_tokens_for_user(user) + + response = self.client.post( + reverse("api_v1:user:token_verify"), + {"token": tokens["access"]}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + class ApiJwtOnlyAuthenticationTest(APITestCase): """Tests that API auth flow is JWT-only and not session-cookie based.""" diff --git a/tests/utils/fixtures.py b/tests/utils/fixtures.py index 3bcbd6b..6e6dd8a 100644 --- a/tests/utils/fixtures.py +++ b/tests/utils/fixtures.py @@ -32,6 +32,19 @@ class ManufacturerRow: address: str +@dataclass(frozen=True) +class IndustrialProductRow: + full_organisation_name: str + ogrn: str + inn: str + registry_number: str + product_name: str + product_model: str + okpd2_code: str + tnved_code: str + regulatory_document: str + + @dataclass(frozen=True) class InspectionRow: registration_number: str @@ -138,6 +151,64 @@ def build_minpromtorg_manufacturers_excel( return buf.getvalue(), rows +def build_minpromtorg_products_excel( + count: int = 5, +) -> tuple[bytes, list[IndustrialProductRow]]: + wb = Workbook() + ws = wb.active + ws.append( + [ + "Реестр промышленной продукции, произведенной на территории РФ", + ] + ) + ws.append( + [ + "Полное наименование организации", + "ОГРН", + "ИНН", + "Регистрационный номер записи", + "Наименование продукции", + "Модель или модификация", + "Код по ОКПД2", + "Код по ТН ВЭД", + "Наименование нормативного документа", + ] + ) + + rows: list[IndustrialProductRow] = [] + for _ in range(count): + row = IndustrialProductRow( + full_organisation_name=fake.company(), + ogrn=_digits(13), + inn=_digits(10), + registry_number=f"MPP-{_digits(8)}", + product_name=fake.sentence(nb_words=4), + product_model=fake.bothify(text="MODEL-###"), + okpd2_code=f"{fake.random_int(min=10, max=99)}.{fake.random_int(min=10, max=99)}", + tnved_code=_digits(10), + regulatory_document=fake.sentence(nb_words=5), + ) + rows.append(row) + ws.append( + [ + row.full_organisation_name, + row.ogrn, + row.inn, + row.registry_number, + row.product_name, + row.product_model, + row.okpd2_code, + row.tnved_code, + row.regulatory_document, + ] + ) + + buf = io.BytesIO() + wb.save(buf) + wb.close() + return buf.getvalue(), rows + + def build_proverki_xml(count: int = 3) -> tuple[bytes, list[InspectionRow]]: rows: list[InspectionRow] = [] parts = ["", ""]