diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 39bc1fc..04d38d1 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -18,6 +18,7 @@ jobs: lint: name: Code Quality Checks runs-on: ubuntu-latest + timeout-minutes: 15 if: ${{ !contains(github.event.head_commit.message, '#no_lint') }} env: TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} @@ -34,18 +35,33 @@ jobs: - name: Install Python and uv run: | set -euo pipefail - apt-get update - apt-get install -y software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.11 python3.11-venv - curl -LsSf https://astral.sh/uv/install.sh | sh + if command -v python3.11 >/dev/null 2>&1; then + BOOTSTRAP_PYTHON=python3.11 + elif command -v python3 >/dev/null 2>&1; then + BOOTSTRAP_PYTHON=python3 + else + echo "python3 is not available on the runner" >&2 + exit 1 + fi + + timeout 180s "${BOOTSTRAP_PYTHON}" -m pip install --user --break-system-packages --upgrade pip uv + export PATH="$HOME/.local/bin:$PATH" + + if "${BOOTSTRAP_PYTHON}" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)'; then + PYTHON_BIN="${BOOTSTRAP_PYTHON}" + else + timeout 300s uv python install "${PYTHON_VERSION}" + PYTHON_BIN="${PYTHON_VERSION}" + fi + + printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env - name: Create virtual environment and install dependencies run: | set -euo pipefail export PATH="$HOME/.local/bin:$PATH" - uv venv --python python3.11 + . ./.ci-python-env + uv venv --python "${PYTHON_BIN}" . .venv/bin/activate uv sync --dev --frozen @@ -93,6 +109,7 @@ jobs: test: name: Run Tests runs-on: ubuntu-latest + timeout-minutes: 20 if: ${{ !contains(github.event.head_commit.message, '#no_test') }} env: TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} @@ -109,18 +126,33 @@ jobs: - name: Install Python and uv run: | set -euo pipefail - apt-get update - apt-get install -y software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - apt-get install -y python3.11 python3.11-venv - curl -LsSf https://astral.sh/uv/install.sh | sh + if command -v python3.11 >/dev/null 2>&1; then + BOOTSTRAP_PYTHON=python3.11 + elif command -v python3 >/dev/null 2>&1; then + BOOTSTRAP_PYTHON=python3 + else + echo "python3 is not available on the runner" >&2 + exit 1 + fi + + timeout 180s "${BOOTSTRAP_PYTHON}" -m pip install --user --break-system-packages --upgrade pip uv + export PATH="$HOME/.local/bin:$PATH" + + if "${BOOTSTRAP_PYTHON}" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)'; then + PYTHON_BIN="${BOOTSTRAP_PYTHON}" + else + timeout 300s uv python install "${PYTHON_VERSION}" + PYTHON_BIN="${PYTHON_VERSION}" + fi + + printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env - name: Create virtual environment and install dependencies run: | set -euo pipefail export PATH="$HOME/.local/bin:$PATH" - uv venv --python python3.11 + . ./.ci-python-env + uv venv --python "${PYTHON_BIN}" . .venv/bin/activate uv sync --dev --frozen diff --git a/src/apps/core/serializers.py b/src/apps/core/serializers.py index fbc8300..9357614 100644 --- a/src/apps/core/serializers.py +++ b/src/apps/core/serializers.py @@ -8,6 +8,22 @@ from rest_framework import serializers +def _get_frontend_job_status(raw_status: str) -> str: + mapping = { + "pending": "running", + "started": "running", + "retry": "running", + "success": "success", + "failure": "error", + "revoked": "error", + } + return mapping.get(raw_status, raw_status) + + +def _get_task_short_name(task_name: str) -> str: + return task_name.rsplit(".", 1)[-1] if task_name else task_name + + class BackgroundJobSerializer(serializers.Serializer): """ Сериализатор для отображения статуса фоновой задачи. @@ -15,22 +31,21 @@ class BackgroundJobSerializer(serializers.Serializer): Используется для API ответов о статусе задач. """ - id = serializers.UUIDField(read_only=True) task_id = serializers.CharField(read_only=True) - task_name = serializers.CharField(read_only=True) - status = serializers.CharField(read_only=True) + status = serializers.SerializerMethodField() progress = serializers.IntegerField(read_only=True) - progress_message = serializers.CharField(read_only=True) + message = serializers.SerializerMethodField() result = serializers.JSONField(read_only=True) - error = serializers.CharField(read_only=True) - started_at = serializers.DateTimeField(read_only=True) - completed_at = serializers.DateTimeField(read_only=True) - created_at = serializers.DateTimeField(read_only=True) - duration = serializers.FloatField(read_only=True) + error = serializers.SerializerMethodField() - # Вычисляемые поля - is_finished = serializers.BooleanField(read_only=True) - is_successful = serializers.BooleanField(read_only=True) + def get_message(self, obj): + return obj.progress_message or None + + def get_error(self, obj): + return obj.error or None + + def get_status(self, obj): + return _get_frontend_job_status(obj.status) class BackgroundJobListSerializer(serializers.Serializer): @@ -38,10 +53,13 @@ class BackgroundJobListSerializer(serializers.Serializer): Краткий сериализатор для списка задач. """ - id = serializers.UUIDField(read_only=True) task_id = serializers.CharField(read_only=True) - task_name = serializers.CharField(read_only=True) - status = serializers.CharField(read_only=True) + name = serializers.SerializerMethodField() + status = serializers.SerializerMethodField() progress = serializers.IntegerField(read_only=True) - created_at = serializers.DateTimeField(read_only=True) - is_finished = serializers.BooleanField(read_only=True) + + def get_name(self, obj): + return _get_task_short_name(obj.task_name) + + def get_status(self, obj): + return _get_frontend_job_status(obj.status) diff --git a/src/apps/core/views.py b/src/apps/core/views.py index 1842f6e..7ae8a0f 100644 --- a/src/apps/core/views.py +++ b/src/apps/core/views.py @@ -7,6 +7,7 @@ Provides endpoints for: - Detailed health check (DB, Redis, Celery status) """ +import json import logging import time from typing import Any @@ -15,6 +16,7 @@ from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from apps.core.serializers import BackgroundJobListSerializer, BackgroundJobSerializer from django.conf import settings from django.db import connection +from django.http import StreamingHttpResponse from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.exceptions import ValidationError @@ -212,6 +214,15 @@ class BackgroundJobStatusView(APIView): permission_classes = [IsAuthenticated] + @staticmethod + def _check_access(request: Request, job) -> Response | None: + if not request.user.is_staff and job.user_id != request.user.id: + return Response( + {"detail": "Нет доступа к этой задаче"}, + status=status.HTTP_403_FORBIDDEN, + ) + return None + @swagger_auto_schema( tags=[JOBS_TAG], operation_summary="Статус задачи", @@ -231,19 +242,114 @@ class BackgroundJobStatusView(APIView): from apps.core.services import BackgroundJobService job = BackgroundJobService.get_by_task_id(task_id) - - # Проверка доступа: только владелец или админ. - # Задачи без владельца считаем системными и не показываем обычным пользователям. - if not request.user.is_staff and job.user_id != request.user.id: - return Response( - {"detail": "Нет доступа к этой задаче"}, - status=status.HTTP_403_FORBIDDEN, - ) + access_error = self._check_access(request, job) + if access_error is not None: + return access_error serializer = BackgroundJobSerializer(job) return Response(serializer.data) +class BackgroundJobStreamView(BackgroundJobStatusView): + """SSE stream with job progress updates until completion.""" + + poll_interval_seconds = 1.0 + + @staticmethod + def _build_sse_message(*, event: str, payload: dict[str, Any]) -> str: + return f"event: {event}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n" + + def _build_progress_payload(self, job) -> dict[str, Any]: + return { + "task_id": job.task_id, + "status": "running", + "progress": job.progress, + "message": job.progress_message, + } + + def _build_final_payload(self, job) -> tuple[str, dict[str, Any]]: + if job.is_successful: + return ( + "completed", + { + "task_id": job.task_id, + "status": "success", + "progress": job.progress, + "result": job.result, + }, + ) + + return ( + "failed", + { + "task_id": job.task_id, + "status": "error", + "progress": job.progress, + "message": job.error + or job.progress_message + or "Задача завершилась с ошибкой", + }, + ) + + def _event_stream(self, task_id: str): + from apps.core.services import BackgroundJobService + + last_snapshot: tuple[str, int, str, str] | None = None + + while True: + job = BackgroundJobService.get_by_task_id(task_id) + snapshot = ( + job.status, + job.progress, + job.progress_message, + job.error, + ) + + if snapshot != last_snapshot: + last_snapshot = snapshot + if job.is_finished: + event, payload = self._build_final_payload(job) + yield self._build_sse_message(event=event, payload=payload) + break + + yield self._build_sse_message( + event="progress", + payload=self._build_progress_payload(job), + ) + + time.sleep(self.poll_interval_seconds) + + @swagger_auto_schema( + tags=[JOBS_TAG], + operation_summary="Поток статуса задачи", + operation_description=( + "Открывает SSE stream со статусом фоновой задачи до её завершения.\n" + "Доступно только владельцу задачи или администратору." + ), + responses={ + 200: "SSE stream", + 403: CommonResponses.FORBIDDEN, + 404: CommonResponses.NOT_FOUND, + **ErrorResponses.AUTHENTICATED, + }, + ) + def get(self, request: Request, task_id: str) -> StreamingHttpResponse | Response: + from apps.core.services import BackgroundJobService + + job = BackgroundJobService.get_by_task_id(task_id) + access_error = self._check_access(request, job) + if access_error is not None: + return access_error + + response = StreamingHttpResponse( + self._event_stream(task_id), + content_type="text/event-stream", + ) + response["Cache-Control"] = "no-cache" + response["X-Accel-Buffering"] = "no" + return response + + class BackgroundJobListView(APIView): """ Список фоновых задач пользователя. @@ -290,4 +396,4 @@ class BackgroundJobListView(APIView): ) serializer = BackgroundJobListSerializer(jobs, many=True) - return Response(serializer.data) + return Response({"results": serializer.data}) diff --git a/src/apps/exchange/serializers.py b/src/apps/exchange/serializers.py index 3f0b333..4430531 100644 --- a/src/apps/exchange/serializers.py +++ b/src/apps/exchange/serializers.py @@ -61,10 +61,6 @@ class ExchangeConnectionSerializer(serializers.ModelSerializer): "database_name", "schema_name", "is_active", - "last_checked_at", - "last_error", - "created_at", - "updated_at", ] read_only_fields = fields @@ -119,19 +115,12 @@ class ExchangePeriodicTaskSerializer(serializers.ModelSerializer): crontab_day_of_week = serializers.SerializerMethodField() crontab_day_of_month = serializers.SerializerMethodField() crontab_month_of_year = serializers.SerializerMethodField() - crontab_timezone = serializers.SerializerMethodField() - mode = serializers.SerializerMethodField() - table = serializers.SerializerMethodField() - tables = serializers.SerializerMethodField() - truncate_before_copy = serializers.SerializerMethodField() + notify_on_error = serializers.SerializerMethodField() class Meta: model = PeriodicTask fields = [ "id", - "name", - "description", - "enabled", "schedule_type", "interval_every", "interval_period", @@ -140,14 +129,7 @@ class ExchangePeriodicTaskSerializer(serializers.ModelSerializer): "crontab_day_of_week", "crontab_day_of_month", "crontab_month_of_year", - "crontab_timezone", - "mode", - "table", - "tables", - "truncate_before_copy", - "last_run_at", - "total_run_count", - "date_changed", + "notify_on_error", ] read_only_fields = fields @@ -179,23 +161,8 @@ class ExchangePeriodicTaskSerializer(serializers.ModelSerializer): def get_crontab_month_of_year(self, obj: PeriodicTask) -> str | None: return obj.crontab.month_of_year if obj.crontab_id else None - def get_crontab_timezone(self, obj: PeriodicTask) -> str | None: - if not obj.crontab_id: - return None - timezone = obj.crontab.timezone - return str(timezone) if timezone is not None else None - - def get_mode(self, obj: PeriodicTask) -> str | None: - return get_periodic_task_payload(obj).get("mode") - - def get_table(self, obj: PeriodicTask) -> str | None: - return get_periodic_task_payload(obj).get("table") - - def get_tables(self, obj: PeriodicTask) -> list[str] | None: - return get_periodic_task_payload(obj).get("tables") - - def get_truncate_before_copy(self, obj: PeriodicTask) -> bool | None: - return get_periodic_task_payload(obj).get("truncate_before_copy") + def get_notify_on_error(self, obj: PeriodicTask) -> bool: + return bool(get_periodic_task_payload(obj).get("notify_on_error", False)) class ExchangePeriodicTaskUpsertSerializer(serializers.Serializer): @@ -231,6 +198,7 @@ class ExchangePeriodicTaskUpsertSerializer(serializers.Serializer): allow_empty=False, ) truncate_before_copy = serializers.BooleanField(required=False) + notify_on_error = serializers.BooleanField(required=False) def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: if not self.instance and "name" not in attrs: @@ -270,6 +238,10 @@ class ExchangePeriodicTaskUpsertSerializer(serializers.Serializer): "truncate_before_copy", current_payload.get("truncate_before_copy", True), ) + notify_on_error = attrs.get( + "notify_on_error", + current_payload.get("notify_on_error", False), + ) if "mode" in attrs and attrs["mode"] != "single" and "table" not in attrs: table = None @@ -286,6 +258,7 @@ class ExchangePeriodicTaskUpsertSerializer(serializers.Serializer): "table": table, "tables": tables, "truncate_before_copy": truncate_before_copy, + "notify_on_error": notify_on_error, } def _build_schedule( diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py index 09ab6e0..40ddae6 100644 --- a/src/apps/exchange/services.py +++ b/src/apps/exchange/services.py @@ -51,7 +51,7 @@ class ExchangeConnectionService: return { "status": "success", - "message": "Подключение и структура целевой БД валидны.", + "message": "Подключение проверено. Соединение и структура БД валидны.", } @classmethod diff --git a/src/apps/exchange/views.py b/src/apps/exchange/views.py index 55c5d6e..3a91bff 100644 --- a/src/apps/exchange/views.py +++ b/src/apps/exchange/views.py @@ -3,7 +3,7 @@ from contextlib import suppress from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag -from apps.core.response import api_created_response, api_response +from apps.core.response import api_response from apps.core.services import BackgroundJobService from apps.exchange.models import ExchangeConnection from apps.exchange.serializers import ( @@ -26,6 +26,7 @@ from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response from rest_framework.views import APIView EXCHANGE_TAG = swagger_tag("Обмен данными", "exchange") @@ -53,7 +54,7 @@ class ExchangeConnectionListCreateView(APIView): "-is_active", "-created_at" ) serializer = ExchangeConnectionSerializer(queryset, many=True) - return api_response(serializer.data, status_code=status.HTTP_200_OK) + return Response({"results": serializer.data}, status=status.HTTP_200_OK) @swagger_auto_schema( tags=[EXCHANGE_TAG], @@ -82,7 +83,7 @@ class ExchangeConnectionListCreateView(APIView): raise ValidationError({"connection": str(exc)}) from exc output = ExchangeConnectionSerializer(connection) - return api_created_response(output.data) + return Response(output.data, status=status.HTTP_201_CREATED) class ExchangeConnectionTestView(APIView): @@ -124,7 +125,13 @@ class ExchangeConnectionTestView(APIView): except ExchangeServiceError as exc: raise ValidationError({"connection": str(exc)}) from exc - return api_response(result, status_code=status.HTTP_200_OK) + return Response( + { + "success": result.get("status") == "success", + "message": result["message"], + }, + status=status.HTTP_200_OK, + ) class ExchangeCopyDataView(APIView): @@ -232,7 +239,7 @@ class ExchangePeriodicTaskListCreateView(APIView): def get(self, request): queryset = ExchangePeriodicTaskService.get_queryset() serializer = ExchangePeriodicTaskSerializer(queryset, many=True) - return api_response(serializer.data, status_code=status.HTTP_200_OK) + return Response({"results": serializer.data}, status=status.HTTP_200_OK) @swagger_auto_schema( tags=[EXCHANGE_TAG], @@ -265,7 +272,7 @@ class ExchangePeriodicTaskListCreateView(APIView): raise ValidationError({"periodic_task": str(exc)}) from exc output = ExchangePeriodicTaskSerializer(task) - return api_created_response(output.data) + return Response(output.data, status=status.HTTP_201_CREATED) class ExchangePeriodicTaskDetailView(APIView): @@ -288,7 +295,7 @@ class ExchangePeriodicTaskDetailView(APIView): id=task_id, ) output = ExchangePeriodicTaskSerializer(task) - return api_response(output.data, status_code=status.HTTP_200_OK) + return Response(output.data, status=status.HTTP_200_OK) @swagger_auto_schema( tags=[EXCHANGE_TAG], @@ -330,4 +337,4 @@ class ExchangePeriodicTaskDetailView(APIView): raise ValidationError({"periodic_task": str(exc)}) from exc output = ExchangePeriodicTaskSerializer(task) - return api_response(output.data, status_code=status.HTTP_200_OK) + return Response(output.data, status=status.HTTP_200_OK) diff --git a/src/apps/parsers/migrations/0014_parsingsettings.py b/src/apps/parsers/migrations/0014_parsingsettings.py new file mode 100644 index 0000000..038ed73 --- /dev/null +++ b/src/apps/parsers/migrations/0014_parsingsettings.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.25 on 2026-03-22 11:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0013_auto_20260320_1010'), + ] + + operations = [ + migrations.CreateModel( + name='ParsingSettings', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')), + ('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')), + ('singleton_key', models.PositiveSmallIntegerField(default=1, editable=False, unique=True, verbose_name='ключ singleton-записи')), + ('manufacturers_and_products', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='производители и продукция')), + ('public_procurements', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='госзакупки')), + ('defense_unreliable_suppliers', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='weekly', max_length=20, verbose_name='недобросовестные поставщики ОПК')), + ('planned_inspections', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='monthly', max_length=20, verbose_name='плановые проверки')), + ('arbitration_cases', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='арбитражные дела')), + ('bankruptcy_procedures', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='daily', max_length=20, verbose_name='банкротные процедуры')), + ('information_security_registries', models.CharField(choices=[('daily', 'Ежедневно'), ('weekly', 'Еженедельно'), ('monthly', 'Ежемесячно'), ('yearly', 'Ежегодно')], default='yearly', max_length=20, verbose_name='реестры информационной безопасности')), + ], + options={ + 'verbose_name': 'настройки парсинга', + 'verbose_name_plural': 'настройки парсинга', + 'db_table': 'parsers_parsing_settings', + }, + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 81181f7..d14f01f 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -846,3 +846,70 @@ class FinancialReportLine(models.Model): def __str__(self) -> str: return f"{self.line_code} ({self.line_name[:30]}) - {self.year}" + + +class ParsingSettings(TimestampMixin, models.Model): + """Singleton-настройки периодичности обновления источников парсинга.""" + + class Frequency(models.TextChoices): + DAILY = "daily", _("Ежедневно") + WEEKLY = "weekly", _("Еженедельно") + MONTHLY = "monthly", _("Ежемесячно") + YEARLY = "yearly", _("Ежегодно") + + singleton_key = models.PositiveSmallIntegerField( + _("ключ singleton-записи"), + default=1, + unique=True, + editable=False, + ) + manufacturers_and_products = models.CharField( + _("производители и продукция"), + max_length=20, + choices=Frequency.choices, + default=Frequency.DAILY, + ) + public_procurements = models.CharField( + _("госзакупки"), + max_length=20, + choices=Frequency.choices, + default=Frequency.DAILY, + ) + defense_unreliable_suppliers = models.CharField( + _("недобросовестные поставщики ОПК"), + max_length=20, + choices=Frequency.choices, + default=Frequency.WEEKLY, + ) + planned_inspections = models.CharField( + _("плановые проверки"), + max_length=20, + choices=Frequency.choices, + default=Frequency.MONTHLY, + ) + arbitration_cases = models.CharField( + _("арбитражные дела"), + max_length=20, + choices=Frequency.choices, + default=Frequency.DAILY, + ) + bankruptcy_procedures = models.CharField( + _("банкротные процедуры"), + max_length=20, + choices=Frequency.choices, + default=Frequency.DAILY, + ) + information_security_registries = models.CharField( + _("реестры информационной безопасности"), + max_length=20, + choices=Frequency.choices, + default=Frequency.YEARLY, + ) + + class Meta: + db_table = "parsers_parsing_settings" + verbose_name = _("настройки парсинга") + verbose_name_plural = _("настройки парсинга") + + def __str__(self) -> str: + return "Настройки парсинга" diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index 458d32f..e27fd5f 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -13,6 +13,7 @@ from apps.parsers.models import ( InspectionRecord, ManufacturerRecord, ParserLoadLog, + ParsingSettings, ProcurementRecord, Proxy, ) @@ -262,22 +263,45 @@ class FNSFileUploadSerializer(serializers.Serializer): Принимает список Excel файлов в формате fin_{id}_{ogrn}.xlsx """ + file = serializers.FileField( + required=False, + help_text="Одиночный файл для загрузки (fin_*.xlsx)", + ) files = serializers.ListField( child=serializers.FileField(), + required=False, allow_empty=False, help_text="Список файлов для загрузки (fin_*.xlsx)", ) - def validate_files(self, value): + def validate(self, attrs): + files = attrs.get("files") + single_file = attrs.get("file") + + if single_file and files: + raise serializers.ValidationError( + {"file": "Используйте либо file, либо files."} + ) + if single_file: + files = [single_file] + if not files: + raise serializers.ValidationError( + {"file": "Нужно передать file или files."} + ) + + attrs["files"] = self._validate_uploaded_files(files) + return attrs + + def _validate_uploaded_files(self, files): """Валидация файлов.""" - for file in value: + for file in files: if not FNS_XLSX_FILENAME_RE.match(file.name): raise serializers.ValidationError( f"Неверный формат имени файла: {file.name}. " "Ожидается: fin_{{id}}_{{ogrn}}.xlsx" ) - return value + return files class FNSZipUploadSerializer(serializers.Serializer): @@ -291,6 +315,22 @@ class FNSZipUploadSerializer(serializers.Serializer): return value +class ParsingSettingsSerializer(serializers.ModelSerializer): + """Настройки периодичности обновления источников парсинга.""" + + class Meta: + model = ParsingSettings + fields = [ + "manufacturers_and_products", + "public_procurements", + "defense_unreliable_suppliers", + "planned_inspections", + "arbitration_cases", + "bankruptcy_procedures", + "information_security_registries", + ] + + # ============================================================================= # Служебные модели # ============================================================================= @@ -374,6 +414,22 @@ class ParserLoadLogSerializer(serializers.ModelSerializer): return 0 +class ParserLoadLogListSerializer(serializers.Serializer): + """Строка списка логов в frontend-friendly формате.""" + + id = serializers.IntegerField(read_only=True) + batch_id = serializers.IntegerField(read_only=True) + source = serializers.CharField(read_only=True) + source_label = serializers.CharField(read_only=True, allow_null=True) + records_count = serializers.IntegerField(read_only=True) + organizations_count = serializers.IntegerField(read_only=True) + status = serializers.CharField(read_only=True) + status_label = serializers.CharField(read_only=True) + error_message = serializers.CharField(read_only=True, allow_blank=True) + created_at = serializers.DateTimeField(read_only=True) + updated_at = serializers.DateTimeField(read_only=True) + + class ProxySerializer(serializers.ModelSerializer): """ Прокси-сервер для парсеров. diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index 983f1a7..44be505 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -195,11 +195,36 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( ) SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS} +SOURCE_CARD_BY_PARSER_SOURCE = { + source_item.parser_source: definition + for definition in SOURCE_CARD_DEFINITIONS + for source_item in definition.source_items + if source_item.parser_source +} class SourceCardService: """Builds aggregated source cards for frontend pages.""" + @classmethod + def get_card_slug_by_parser_source(cls, parser_source: str | None) -> str | None: + definition = SOURCE_CARD_BY_PARSER_SOURCE.get(parser_source) + return definition.slug if definition else None + + @classmethod + def get_card_title_by_parser_source(cls, parser_source: str | None) -> str | None: + definition = SOURCE_CARD_BY_PARSER_SOURCE.get(parser_source) + return definition.title if definition else None + + @classmethod + def get_parser_sources_by_card_slug(cls, slug: str) -> list[str]: + definition = SOURCE_CARD_BY_SLUG.get(slug) + if definition is None: + return [] + return [ + item.parser_source for item in definition.source_items if item.parser_source + ] + @classmethod def list_cards(cls) -> list[dict[str, Any]]: return [cls.get_card(definition.slug) for definition in SOURCE_CARD_DEFINITIONS] diff --git a/src/apps/parsers/urls.py b/src/apps/parsers/urls.py index 4851f00..a20946c 100644 --- a/src/apps/parsers/urls.py +++ b/src/apps/parsers/urls.py @@ -13,6 +13,7 @@ from apps.parsers.views import ( ManufacturerViewSet, ParserLoadLogExportView, ParserLoadLogViewSet, + ParsingSettingsView, ProcurementViewSet, ProxyViewSet, SourceCardDetailView, @@ -78,6 +79,14 @@ fns_urlpatterns = [ path("", include(fns_router.urls)), ] +# ============================================================================= +# Parsing settings: /api/v1/parsing/ +# ============================================================================= + +parsing_urlpatterns = [ + path("settings/", ParsingSettingsView.as_view(), name="parsing-settings"), +] + # ============================================================================= # Frontend sources: /api/v1/sources/ # ============================================================================= diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 3b2fd77..e62687f 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -17,6 +17,7 @@ from apps.parsers.models import ( InspectionRecord, ManufacturerRecord, ParserLoadLog, + ParsingSettings, ProcurementRecord, Proxy, ) @@ -28,7 +29,9 @@ from apps.parsers.serializers import ( IndustrialProductSerializer, InspectionSerializer, ManufacturerSerializer, + ParserLoadLogListSerializer, ParserLoadLogSerializer, + ParsingSettingsSerializer, ProcurementSerializer, ProxySerializer, SourceCardDetailSerializer, @@ -38,12 +41,14 @@ from apps.parsers.serializers import ( SourceTaskStatusSerializer, ) from apps.parsers.source_cards import SourceCardService +from django.core.paginator import Paginator from django.db.models import CharField, Count, Q from django.db.models.functions import Cast from django.http import HttpResponse from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework import status +from rest_framework.exceptions import ValidationError from rest_framework.parsers import MultiPartParser from rest_framework.permissions import IsAdminUser, IsAuthenticated from rest_framework.response import Response @@ -60,6 +65,7 @@ ZAKUPKI_TAG = swagger_tag("Государственные закупки", "publ FNS_TAG = swagger_tag("ФНС - Бухгалтерская отчетность", "fns_financial_reports") SOURCES_TAG = swagger_tag("Источники для фронта", "frontend_sources") SYSTEM_TAG = swagger_tag("Системные", "system") +PARSING_TAG = swagger_tag("Настройки парсинга", "parsing_settings") PARSER_LOG_ORDERING_FIELDS = { "id", @@ -71,6 +77,18 @@ PARSER_LOG_ORDERING_FIELDS = { "updated_at", } +PARSER_LOG_STATUS_LABELS = { + "success": "Успешно", + "failed": "Ошибка", + "failure": "Ошибка", + "error": "Ошибка", + "in_progress": "В процессе", + "pending": "В очереди", + "started": "В процессе", + "retry": "Повтор", + "skipped": "Пропущено", +} + def _get_parser_logs_queryset(*, search_query: str = ""): queryset = ParserLoadLog.objects.all().order_by("-created_at") @@ -102,6 +120,185 @@ def _apply_safe_ordering(queryset, ordering: str, allowed_fields: set[str]): return queryset.order_by(*order_by_fields) +def _matches_parser_log_search(row: dict, search_term: str) -> bool: + normalized_search = search_term.casefold() + for value in row.values(): + if value is None: + continue + if normalized_search in str(value).casefold(): + return True + return False + + +def _sort_parser_log_rows(rows: list[dict], ordering: str) -> list[dict]: + allowed_fields = { + "id", + "batch_id", + "source", + "source_label", + "status", + "status_label", + "records_count", + "organizations_count", + "created_at", + "updated_at", + } + sorted_rows = list(rows) + order_by_fields = [item.strip() for item in ordering.split(",") if item.strip()] + for raw_field in reversed(order_by_fields): + field_name = raw_field[1:] if raw_field.startswith("-") else raw_field + if field_name not in allowed_fields: + continue + reverse = raw_field.startswith("-") + sorted_rows.sort( + key=lambda row: (row.get(field_name) is None, row.get(field_name)), + reverse=reverse, + ) + return sorted_rows + + +def _build_page_url(request, page_number: int) -> str: + query_params = request.query_params.copy() + query_params["page"] = page_number + return request.build_absolute_uri(f"{request.path}?{query_params.urlencode()}") + + +def _paginate_results(request, rows: list[dict]): + page_size_raw = request.query_params.get("page_size", "20") + page_raw = request.query_params.get("page", "1") + try: + page_size = max(1, min(int(page_size_raw), 100)) + page_number = max(1, int(page_raw)) + except (TypeError, ValueError) as exc: + raise ValidationError( + { + "detail": "Параметры page и page_size должны быть положительными целыми числами." + } + ) from exc + + paginator = Paginator(rows, page_size) + page_obj = paginator.get_page(page_number) + return { + "count": paginator.count, + "next": ( + _build_page_url(request, page_obj.next_page_number()) + if page_obj.has_next() + else None + ), + "previous": ( + _build_page_url(request, page_obj.previous_page_number()) + if page_obj.has_previous() + else None + ), + "results": list(page_obj.object_list), + } + + +def _get_parser_log_status_label(status_value: str) -> str: + return PARSER_LOG_STATUS_LABELS.get(status_value, status_value) + + +def _get_parser_log_source_label(log: ParserLoadLog) -> str: + return ( + SourceCardService.get_card_title_by_parser_source(log.source) + or log.get_source_display() + ) + + +def _get_parser_log_source_value(log: ParserLoadLog) -> str: + return SourceCardService.get_card_slug_by_parser_source(log.source) or log.source + + +def _get_parser_log_organizations_count(log: ParserLoadLog) -> int: + if log.source == ParserLoadLog.Source.FNS_REPORTS: + return ( + FinancialReport.objects.filter(load_batch=log.batch_id) + .exclude(ogrn="") + .values("ogrn") + .distinct() + .count() + ) + if log.source == ParserLoadLog.Source.INDUSTRIAL: + return ( + IndustrialCertificateRecord.objects.filter(load_batch=log.batch_id) + .exclude(inn="") + .values("inn") + .distinct() + .count() + ) + if log.source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS: + return ( + IndustrialProductRecord.objects.filter(load_batch=log.batch_id) + .exclude(inn="") + .values("inn") + .distinct() + .count() + ) + if log.source == ParserLoadLog.Source.MANUFACTURES: + return ( + ManufacturerRecord.objects.filter(load_batch=log.batch_id) + .exclude(inn="") + .values("inn") + .distinct() + .count() + ) + if log.source == ParserLoadLog.Source.INSPECTIONS: + return ( + InspectionRecord.objects.filter(load_batch=log.batch_id) + .exclude(inn="") + .values("inn") + .distinct() + .count() + ) + if log.source == ParserLoadLog.Source.PROCUREMENTS: + return ( + ProcurementRecord.objects.filter(load_batch=log.batch_id) + .exclude(customer_inn="") + .values("customer_inn") + .distinct() + .count() + ) + return 0 + + +def _serialize_parser_log_row(log: ParserLoadLog) -> dict: + return { + "id": log.id, + "batch_id": log.batch_id, + "source": _get_parser_log_source_value(log), + "source_label": _get_parser_log_source_label(log), + "records_count": log.records_count, + "organizations_count": _get_parser_log_organizations_count(log), + "status": log.status, + "status_label": _get_parser_log_status_label(log.status), + "error_message": log.error_message, + "created_at": log.created_at, + "updated_at": log.updated_at, + } + + +def _apply_parser_log_filters(request): + queryset = _get_parser_logs_queryset(search_query="") + + source_value = request.query_params.get("source", "").strip() + if source_value: + card_sources = SourceCardService.get_parser_sources_by_card_slug(source_value) + if card_sources: + queryset = queryset.filter(source__in=card_sources) + else: + queryset = queryset.filter(source=source_value) + + status_value = request.query_params.get("status") + if status_value: + queryset = queryset.filter(status=status_value) + + batch_id = request.query_params.get("batch_id") + if batch_id: + queryset = queryset.filter(batch_id=batch_id) + + return queryset + + # ============================================================================= # Минпромторг - Сертификаты промышленного производства # ============================================================================= @@ -531,6 +728,17 @@ class FNSReportUploadView(APIView): requested_by_id=request.user.id, ) + if "file" in request.data and "files" not in request.data: + message = ( + "Файл загружен" + if result.queued + else "Файл уже обрабатывается или был загружен ранее" + ) + return Response( + {"success": True, "message": message}, + status=status.HTTP_200_OK, + ) + return Response( { "queued": result.queued, @@ -541,6 +749,50 @@ class FNSReportUploadView(APIView): ) +class ParsingSettingsView(APIView): + """Получение и изменение singleton-настроек парсинга.""" + + permission_classes = [IsAdminUser] + + @staticmethod + def _get_settings() -> ParsingSettings: + settings_obj, _ = ParsingSettings.objects.get_or_create(singleton_key=1) + return settings_obj + + @swagger_auto_schema( + tags=[PARSING_TAG], + operation_summary="Получить настройки парсинга", + responses={ + 200: ParsingSettingsSerializer, + **ErrorResponses.ADMIN, + }, + ) + def get(self, request): + serializer = ParsingSettingsSerializer(self._get_settings()) + return Response(serializer.data, status=status.HTTP_200_OK) + + @swagger_auto_schema( + tags=[PARSING_TAG], + operation_summary="Изменить настройки парсинга", + request_body=ParsingSettingsSerializer, + responses={ + 200: ParsingSettingsSerializer, + 400: CommonResponses.BAD_REQUEST, + **ErrorResponses.ADMIN, + }, + ) + def patch(self, request): + settings_obj = self._get_settings() + serializer = ParsingSettingsSerializer( + settings_obj, + data=request.data, + partial=True, + ) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data, status=status.HTTP_200_OK) + + # ============================================================================= # Frontend-oriented source cards # ============================================================================= @@ -643,7 +895,20 @@ class SourceCardRefreshView(APIView): params=serializer.validated_data.get("params", {}), ) output = SourceCardRefreshResponseSerializer(payload) - return api_response(output.data, status_code=status.HTTP_202_ACCEPTED) + serialized_payload = output.data + tasks = serialized_payload.get("tasks", []) + task_id = tasks[0]["task_id"] if tasks else None + response_payload = { + "task_id": task_id, + "status": "accepted", + } + if len(tasks) > 1: + response_payload["tasks"] = tasks + response_payload["source_card"] = serialized_payload.get("source_card") + return Response( + response_payload, + status=status.HTTP_202_ACCEPTED, + ) # ============================================================================= @@ -665,9 +930,7 @@ class ParserLoadLogViewSet(ReadOnlyModelViewSet): ordering_fields = list(PARSER_LOG_ORDERING_FIELDS) def get_queryset(self): - return _get_parser_logs_queryset( - search_query=self.request.query_params.get("search", "") - ) + return _apply_parser_log_filters(self.request) @swagger_auto_schema( tags=[SYSTEM_TAG], @@ -705,7 +968,13 @@ class ParserLoadLogViewSet(ReadOnlyModelViewSet): }, ) def list(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) + rows = [_serialize_parser_log_row(item) for item in self.get_queryset()] + search_term = request.query_params.get("search", "").strip() + if search_term: + rows = [row for row in rows if _matches_parser_log_search(row, search_term)] + rows = _sort_parser_log_rows(rows, request.query_params.get("ordering", "")) + serializer = ParserLoadLogListSerializer(rows, many=True) + return Response(_paginate_results(request, serializer.data)) @swagger_auto_schema( tags=[SYSTEM_TAG], @@ -775,28 +1044,15 @@ class ParserLoadLogExportView(APIView): }, ) def get(self, request): - queryset = _get_parser_logs_queryset( - search_query=request.query_params.get("search", "") - ) + rows = [ + _serialize_parser_log_row(item) + for item in _apply_parser_log_filters(request) + ] + search_term = request.query_params.get("search", "").strip() + if search_term: + rows = [row for row in rows if _matches_parser_log_search(row, search_term)] + rows = _sort_parser_log_rows(rows, request.query_params.get("ordering", "")) - source = request.query_params.get("source") - status_value = request.query_params.get("status") - batch_id = request.query_params.get("batch_id") - - if source: - queryset = queryset.filter(source=source) - if status_value: - queryset = queryset.filter(status=status_value) - if batch_id: - queryset = queryset.filter(batch_id=batch_id) - - queryset = _apply_safe_ordering( - queryset, - request.query_params.get("ordering", ""), - PARSER_LOG_ORDERING_FIELDS, - ) - - serializer = ParserLoadLogSerializer(queryset, many=True) response = HttpResponse(content_type="text/csv; charset=utf-8") response["Content-Disposition"] = 'attachment; filename="parser-load-logs.csv"' @@ -806,26 +1062,28 @@ class ParserLoadLogExportView(APIView): "id", "batch_id", "source", - "source_display", + "source_label", "records_count", "organizations_count", "status", + "status_label", "error_message", "created_at", "updated_at", ] ) - for row in serializer.data: + for row in rows: writer.writerow( [ row["id"], row["batch_id"], row["source"], - row["source_display"], + row["source_label"], row["records_count"], row["organizations_count"], row["status"], + row["status_label"], row["error_message"], row["created_at"], row["updated_at"], diff --git a/src/apps/registers/apps.py b/src/apps/registers/apps.py index a05bc64..869b5b6 100644 --- a/src/apps/registers/apps.py +++ b/src/apps/registers/apps.py @@ -7,3 +7,6 @@ class RegistersConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.registers" verbose_name = "Реестры организаций" + + def ready(self): + from . import signals # noqa: F401 diff --git a/src/apps/registers/signals.py b/src/apps/registers/signals.py new file mode 100644 index 0000000..cb22d4e --- /dev/null +++ b/src/apps/registers/signals.py @@ -0,0 +1,22 @@ +"""Signals for registers app.""" + +from django.apps import apps +from django.db.models.signals import post_migrate +from django.dispatch import receiver + +DEFAULT_REGISTER_NAMES = ( + "Реестр предприятий ОПК", + "Реестр госкорпорации Роскосмос", + "Реестр госкорпорации Росатом", +) + + +@receiver(post_migrate) +def seed_default_registers(sender, **kwargs): + """Create default registries on fresh environments.""" + if sender.name != "apps.registers": + return + + Register = apps.get_model("registers", "Register") + for name in DEFAULT_REGISTER_NAMES: + Register.objects.get_or_create(name=name) diff --git a/src/apps/registers/views.py b/src/apps/registers/views.py index 1dbf72f..89d76c9 100644 --- a/src/apps/registers/views.py +++ b/src/apps/registers/views.py @@ -49,7 +49,9 @@ class RegisterViewSet(ReadOnlyModelViewSet): }, ) def list(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) + queryset = self.filter_queryset(self.get_queryset()) + serializer = self.get_serializer(queryset, many=True) + return Response({"results": serializer.data}, status=status.HTTP_200_OK) @swagger_auto_schema( tags=[REGISTERS_TAG], @@ -363,7 +365,7 @@ class RegisterUploadView(APIView): actual_date = serializer.validated_data.get("actual_date") try: - result = RegisterImportService.sync_registry_memberships( + RegisterImportService.sync_registry_memberships( registry=registry, uploaded_file=uploaded_file, actual_date=actual_date, @@ -372,4 +374,7 @@ class RegisterUploadView(APIView): except RegisterImportError as exc: raise ValidationError({"file": str(exc)}) from exc - return Response(result, status=status.HTTP_201_CREATED) + return Response( + {"success": True, "message": "Файл успешно загружен"}, + status=status.HTTP_201_CREATED, + ) diff --git a/src/apps/user/migrations/0010_profile_names_required.py b/src/apps/user/migrations/0010_profile_names_required.py new file mode 100644 index 0000000..c643071 --- /dev/null +++ b/src/apps/user/migrations/0010_profile_names_required.py @@ -0,0 +1,57 @@ +from django.db import migrations, models + + +def backfill_profile_names(apps, schema_editor): + Profile = apps.get_model("user", "Profile") + + for profile in Profile.objects.select_related("user").all().iterator(): + username = profile.user.username + first_name = (profile.first_name or "").strip() or username + middle_name = (profile.middle_name or "").strip() + last_name = (profile.last_name or "").strip() or username + + updates = [] + if profile.first_name != first_name: + profile.first_name = first_name + updates.append("first_name") + if (profile.middle_name or "") != middle_name: + profile.middle_name = middle_name + updates.append("middle_name") + if profile.last_name != last_name: + profile.last_name = last_name + updates.append("last_name") + + if updates: + profile.save(update_fields=updates) + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ("user", "0009_alter_user_groups"), + ] + + operations = [ + migrations.RunPython(backfill_profile_names, migrations.RunPython.noop), + migrations.AlterField( + model_name="profile", + name="first_name", + field=models.CharField(default="", max_length=50, verbose_name="first name"), + ), + migrations.AlterField( + model_name="profile", + name="middle_name", + field=models.CharField( + blank=True, + default="", + max_length=50, + verbose_name="middle name", + ), + ), + migrations.AlterField( + model_name="profile", + name="last_name", + field=models.CharField(default="", max_length=50, verbose_name="last name"), + ), + ] diff --git a/src/apps/user/models.py b/src/apps/user/models.py index 14fc8ee..f3efcd6 100644 --- a/src/apps/user/models.py +++ b/src/apps/user/models.py @@ -69,16 +69,16 @@ class Profile(models.Model): User, on_delete=models.CASCADE, related_name="profile", verbose_name=_("user") ) - first_name = models.CharField(_("first name"), max_length=50, blank=True, null=True) + first_name = models.CharField(_("first name"), max_length=50, default="") middle_name = models.CharField( _("middle name"), max_length=50, blank=True, - null=True, + default="", ) - last_name = models.CharField(_("last name"), max_length=50, blank=True, null=True) + last_name = models.CharField(_("last name"), max_length=50, default="") bio = models.TextField( _("bio"), blank=True, null=True, help_text=_("Short biography or description") @@ -110,7 +110,7 @@ class Profile(models.Model): @property def full_name(self): """Полное имя пользователя""" - parts = [self.first_name, self.middle_name, self.last_name] + parts = [self.last_name, self.first_name, self.middle_name] full_name = " ".join(part for part in parts if part) if full_name: return full_name diff --git a/src/apps/user/serializers.py b/src/apps/user/serializers.py index 8ea92b0..0238843 100644 --- a/src/apps/user/serializers.py +++ b/src/apps/user/serializers.py @@ -21,10 +21,26 @@ class UserRegistrationSerializer(serializers.ModelSerializer): password_confirm = serializers.CharField( write_only=True, min_length=8, help_text="Подтверждение пароля" ) + first_name = serializers.CharField(help_text="Имя пользователя") + middle_name = serializers.CharField( + required=False, + allow_blank=True, + help_text="Отчество пользователя", + ) + last_name = serializers.CharField(help_text="Фамилия пользователя") class Meta: model = User - fields = ("email", "username", "password", "password_confirm", "phone") + fields = ( + "email", + "username", + "password", + "password_confirm", + "phone", + "first_name", + "middle_name", + "last_name", + ) extra_kwargs = { "username": { "validators": [UniqueValidator(queryset=User.objects.all())], @@ -64,6 +80,21 @@ class UserProfileSerializer(serializers.ModelSerializer): read_only_fields = ("id",) +class FrontendUserProfileSerializer(serializers.ModelSerializer): + """Профиль пользователя в shape, который ожидает frontend.""" + + full_name = serializers.ReadOnlyField(help_text="Полное имя") + + class Meta: + model = Profile + fields = ( + "first_name", + "middle_name", + "last_name", + "full_name", + ) + + class UserSerializer(serializers.ModelSerializer): """Сериализатор для пользователя""" @@ -109,6 +140,60 @@ class UserSerializer(serializers.ModelSerializer): return UserService.get_user_capabilities(obj) +class FrontendUserWithProfileSerializer(serializers.ModelSerializer): + """Короткий сериализатор пользователя с профилем для frontend.""" + + profile = FrontendUserProfileSerializer(read_only=True) + role = serializers.SerializerMethodField() + role_label = serializers.SerializerMethodField() + + class Meta: + model = User + fields = ( + "id", + "username", + "email", + "phone", + "is_active", + "role", + "role_label", + "profile", + ) + read_only_fields = fields + + def get_role(self, obj) -> str: + return UserService.get_user_role(obj) + + def get_role_label(self, obj) -> str: + return UserService.get_role_label(self.get_role(obj)) + + +class FrontendManagedUserSerializer(serializers.ModelSerializer): + """Короткий сериализатор пользователя без вложенного профиля.""" + + role = serializers.SerializerMethodField() + role_label = serializers.SerializerMethodField() + + class Meta: + model = User + fields = ( + "id", + "username", + "email", + "phone", + "role", + "role_label", + "is_active", + ) + read_only_fields = fields + + def get_role(self, obj) -> str: + return UserService.get_user_role(obj) + + def get_role_label(self, obj) -> str: + return UserService.get_role_label(self.get_role(obj)) + + class UserUpdateSerializer(serializers.ModelSerializer): """Сериализатор для обновления данных пользователя""" @@ -132,7 +217,6 @@ class AdminUserCreateSerializer(serializers.ModelSerializer): middle_name = serializers.CharField( required=False, allow_blank=True, - allow_null=True, ) last_name = serializers.CharField(allow_blank=False) @@ -174,15 +258,12 @@ class AdminUserUpdateSerializer(serializers.ModelSerializer): required=False, help_text="Прикладная роль пользователя", ) - first_name = serializers.CharField( - required=False, allow_blank=True, allow_null=True - ) + first_name = serializers.CharField(required=False, allow_blank=True) middle_name = serializers.CharField( required=False, allow_blank=True, - allow_null=True, ) - last_name = serializers.CharField(required=False, allow_blank=True, allow_null=True) + last_name = serializers.CharField(required=False, allow_blank=True) class Meta: model = User @@ -199,6 +280,32 @@ class AdminUserUpdateSerializer(serializers.ModelSerializer): "last_name", ) + def validate(self, attrs): + profile_fields = {"first_name", "middle_name", "last_name"} + if not any(field in attrs for field in profile_fields): + return attrs + + profile = getattr(self.instance, "profile", None) if self.instance else None + first_name = attrs.get( + "first_name", + profile.first_name if profile is not None else None, + ) + last_name = attrs.get( + "last_name", + profile.last_name if profile is not None else None, + ) + + errors = {} + if not str(first_name or "").strip(): + errors["first_name"] = "Обязательное поле." + if not str(last_name or "").strip(): + errors["last_name"] = "Обязательное поле." + + if errors: + raise serializers.ValidationError(errors) + + return attrs + class ProfileUpdateSerializer(serializers.ModelSerializer): """Сериализатор для обновления профиля""" @@ -214,6 +321,27 @@ class ProfileUpdateSerializer(serializers.ModelSerializer): "date_of_birth", ) + def validate(self, attrs): + first_name = attrs.get( + "first_name", + self.instance.first_name if self.instance is not None else None, + ) + last_name = attrs.get( + "last_name", + self.instance.last_name if self.instance is not None else None, + ) + + errors = {} + if not str(first_name or "").strip(): + errors["first_name"] = "Обязательное поле." + if not str(last_name or "").strip(): + errors["last_name"] = "Обязательное поле." + + if errors: + raise serializers.ValidationError(errors) + + return attrs + class LoginSerializer(serializers.Serializer): """Сериализатор для входа""" diff --git a/src/apps/user/services.py b/src/apps/user/services.py index 8187756..b52cd91 100644 --- a/src/apps/user/services.py +++ b/src/apps/user/services.py @@ -5,6 +5,10 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.db import transaction from django.db.models import F, Q +from rest_framework_simplejwt.token_blacklist.models import ( + BlacklistedToken, + OutstandingToken, +) from rest_framework_simplejwt.tokens import RefreshToken from .models import Profile @@ -53,12 +57,21 @@ class UserService: ValidationError: При некорректных данных """ role = role or cls.ROLE_USER + first_name = extra_fields.pop("first_name", None) + middle_name = extra_fields.pop("middle_name", None) + last_name = extra_fields.pop("last_name", None) with transaction.atomic(): user = User.objects.create_user( email=email, username=username, password=password, **extra_fields ) cls.assign_role(user, role) - return user + cls._update_or_create_profile( + user=user, + first_name=first_name or user.username, + middle_name=middle_name or "", + last_name=last_name or user.username, + ) + return cls.get_users_queryset().get(id=user.id) @classmethod def get_users_queryset(cls): @@ -218,13 +231,10 @@ class UserService: username=username, password=password, role=role, - **extra_fields, - ) - cls._update_or_create_profile( - user=user, first_name=first_name, middle_name=middle_name, last_name=last_name, + **extra_fields, ) return cls.get_users_queryset().get(id=user.id) @@ -304,6 +314,16 @@ class UserService: "access": str(refresh.access_token), } + @classmethod + def logout_user(cls, user: User) -> int: + """Отзывает все активные refresh-токены пользователя.""" + outstanding_tokens = OutstandingToken.objects.filter(user=user) + revoked_count = 0 + for token in outstanding_tokens: + _, created = BlacklistedToken.objects.get_or_create(token=token) + revoked_count += int(created) + return revoked_count + @classmethod def ensure_role_groups(cls) -> dict[str, Group]: """Гарантирует существование системных role-групп.""" diff --git a/src/apps/user/signals.py b/src/apps/user/signals.py index 7959fc5..d89c166 100644 --- a/src/apps/user/signals.py +++ b/src/apps/user/signals.py @@ -13,7 +13,12 @@ def create_user_profile(sender, instance, created, **kwargs): Автоматически создает профиль при создании пользователя """ if created: - Profile.objects.create(user=instance) + Profile.objects.create( + user=instance, + first_name=instance.username, + middle_name="", + last_name=instance.username, + ) @receiver(post_save, sender=User) diff --git a/src/apps/user/views.py b/src/apps/user/views.py index 30d684d..c1470dc 100644 --- a/src/apps/user/views.py +++ b/src/apps/user/views.py @@ -1,6 +1,7 @@ from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from django.contrib.auth import authenticate from django.contrib.auth.hashers import check_password +from django.core.paginator import Paginator from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -15,6 +16,8 @@ from rest_framework_simplejwt.views import TokenVerifyView as SimpleJWTTokenVeri from .serializers import ( AdminUserCreateSerializer, AdminUserUpdateSerializer, + FrontendManagedUserSerializer, + FrontendUserWithProfileSerializer, LoginSerializer, PasswordChangeSerializer, ProfileUpdateSerializer, @@ -131,9 +134,8 @@ class LogoutView(APIView): }, ) def post(self, request): - # Для JWT логаут означает удаление токенов на клиенте. - # Сервер не хранит сессию и ничего не инвалидирует. - return Response({"message": "Успешный выход"}, status=status.HTTP_200_OK) + UserService.logout_user(request.user) + return Response({}, status=status.HTTP_200_OK) class CurrentUserView(APIView): @@ -146,12 +148,12 @@ class CurrentUserView(APIView): operation_summary="Текущий пользователь", operation_description="Возвращает данные авторизованного пользователя.", responses={ - 200: UserSerializer, + 200: FrontendUserWithProfileSerializer, **ErrorResponses.AUTHENTICATED, }, ) def get(self, request): - serializer = UserSerializer(request.user) + serializer = FrontendUserWithProfileSerializer(request.user) return Response(serializer.data) @@ -160,6 +162,24 @@ class AdminUserListCreateView(APIView): permission_classes = [IsAdminUser] + @staticmethod + def _get_positive_int(value, *, default: int, minimum: int = 1) -> int: + if value in (None, ""): + return default + try: + parsed = int(value) + except (TypeError, ValueError): + raise ValueError("must be integer") from None + if parsed < minimum: + raise ValueError("must be positive") + return parsed + + @staticmethod + def _build_page_url(request, page_number: int) -> str: + query_params = request.query_params.copy() + query_params["page"] = page_number + return request.build_absolute_uri(f"{request.path}?{query_params.urlencode()}") + @swagger_auto_schema( tags=[USER_ADMIN_TAG], operation_summary="Список пользователей", @@ -189,7 +209,7 @@ class AdminUserListCreateView(APIView): ), ], responses={ - 200: UserSerializer(many=True), + 200: FrontendUserWithProfileSerializer(many=True), **ErrorResponses.ADMIN, }, ) @@ -198,8 +218,47 @@ class AdminUserListCreateView(APIView): search=request.query_params.get("search", ""), ordering=request.query_params.get("ordering", ""), ) - serializer = UserSerializer(queryset, many=True) - return Response(serializer.data) + try: + page_number = self._get_positive_int( + request.query_params.get("page"), + default=1, + ) + page_size = self._get_positive_int( + request.query_params.get("page_size"), + default=20, + ) + except ValueError: + return Response( + { + "detail": ( + "Параметры page и page_size должны быть положительными " + "целыми числами." + ) + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + page_size = min(page_size, 100) + paginator = Paginator(queryset, page_size) + page_obj = paginator.get_page(page_number) + serializer = FrontendUserWithProfileSerializer(page_obj.object_list, many=True) + + return Response( + { + "count": paginator.count, + "next": ( + self._build_page_url(request, page_obj.next_page_number()) + if page_obj.has_next() + else None + ), + "previous": ( + self._build_page_url(request, page_obj.previous_page_number()) + if page_obj.has_previous() + else None + ), + "results": serializer.data, + } + ) @swagger_auto_schema( tags=[USER_ADMIN_TAG], @@ -209,7 +268,7 @@ class AdminUserListCreateView(APIView): ), request_body=AdminUserCreateSerializer, responses={ - 201: UserSerializer, + 201: FrontendManagedUserSerializer, 400: CommonResponses.BAD_REQUEST, **ErrorResponses.ADMIN, }, @@ -218,7 +277,10 @@ class AdminUserListCreateView(APIView): serializer = AdminUserCreateSerializer(data=request.data) serializer.is_valid(raise_exception=True) user = UserService.create_managed_user(**serializer.validated_data) - return Response(UserSerializer(user).data, status=status.HTTP_201_CREATED) + return Response( + FrontendManagedUserSerializer(user).data, + status=status.HTTP_201_CREATED, + ) class AdminUserDetailView(APIView): @@ -234,13 +296,13 @@ class AdminUserDetailView(APIView): operation_summary="Детали пользователя", operation_description="Возвращает данные конкретного пользователя.", responses={ - 200: UserSerializer, + 200: FrontendUserWithProfileSerializer, **ErrorResponses.ADMIN_NOT_FOUND, }, ) def get(self, request, user_id: int): user = self._get_user(user_id) - return Response(UserSerializer(user).data) + return Response(FrontendUserWithProfileSerializer(user).data) @swagger_auto_schema( tags=[USER_ADMIN_TAG], @@ -251,7 +313,7 @@ class AdminUserDetailView(APIView): ), request_body=AdminUserUpdateSerializer, responses={ - 200: UserSerializer, + 200: FrontendManagedUserSerializer, 400: CommonResponses.BAD_REQUEST, **ErrorResponses.ADMIN_NOT_FOUND, }, @@ -277,7 +339,7 @@ class AdminUserDetailView(APIView): user_id=user.id, **serializer.validated_data, ) - return Response(UserSerializer(updated_user).data) + return Response(FrontendManagedUserSerializer(updated_user).data) class AdminUserDeactivateView(APIView): @@ -302,8 +364,8 @@ class AdminUserDeactivateView(APIView): status=status.HTTP_400_BAD_REQUEST, ) - user = UserService.deactivate_user(user_id) - return Response(UserSerializer(user).data) + UserService.deactivate_user(user_id) + return Response({"success": True}) class AdminUserActivateView(APIView): @@ -316,13 +378,13 @@ class AdminUserActivateView(APIView): operation_summary="Активировать пользователя", operation_description="Возвращает пользователя в активное состояние.", responses={ - 200: UserSerializer, + 200: FrontendManagedUserSerializer, **ErrorResponses.ADMIN_NOT_FOUND, }, ) def post(self, request, user_id: int): user = UserService.activate_user(user_id) - return Response(UserSerializer(user).data) + return Response(FrontendManagedUserSerializer(user).data) class UserUpdateView(APIView): @@ -362,7 +424,12 @@ class ProfileDetailView(generics.RetrieveUpdateAPIView): # Если профиль не существует, создаем его from .models import Profile - profile = Profile.objects.create(user=self.request.user) + profile = Profile.objects.create( + user=self.request.user, + first_name=self.request.user.username, + middle_name="", + last_name=self.request.user.username, + ) return profile @swagger_auto_schema( diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index 69b1068..62e7b1c 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -18,11 +18,16 @@ API v1 URL configuration. """ from apps.backups.urls import backups_urlpatterns -from apps.core.views import BackgroundJobListView, BackgroundJobStatusView +from apps.core.views import ( + BackgroundJobListView, + BackgroundJobStatusView, + BackgroundJobStreamView, +) from apps.exchange.urls import exchange_urlpatterns from apps.parsers.urls import ( fns_urlpatterns, minpromtorg_urlpatterns, + parsing_urlpatterns, proverki_urlpatterns, sources_urlpatterns, system_urlpatterns, @@ -36,6 +41,7 @@ app_name = "api_v1" # Фоновые задачи jobs_urlpatterns = [ path("", BackgroundJobListView.as_view(), name="job-list"), + path("/stream/", BackgroundJobStreamView.as_view(), name="job-stream"), path("/", BackgroundJobStatusView.as_view(), name="job-status"), ] @@ -54,6 +60,8 @@ urlpatterns = [ path("fns/", include((fns_urlpatterns, "fns"))), # Агрегированные карточки источников для фронтенда path("sources/", include((sources_urlpatterns, "sources"))), + # Настройки периодичности парсинга + path("parsing/", include((parsing_urlpatterns, "parsing"))), # Реестры организаций path("registers/", include((registers_urlpatterns, "registers"))), # Обмен с внешней БД diff --git a/tests/apps/core/test_views.py b/tests/apps/core/test_views.py index 26e97e6..2d688a0 100644 --- a/tests/apps/core/test_views.py +++ b/tests/apps/core/test_views.py @@ -332,7 +332,12 @@ class BackgroundJobsViewTest(APITestCase): url = reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + set(response.data.keys()), + {"task_id", "status", "progress", "message", "result", "error"}, + ) self.assertEqual(response.data["task_id"], job.task_id) + self.assertEqual(response.data["status"], "success") def test_job_status_forbidden_for_other_user(self): job = self._create_job( @@ -361,12 +366,17 @@ class BackgroundJobsViewTest(APITestCase): def test_job_list_filters_status(self): self._create_job(task_id="job-1", user_id=self.user.id, status="success") - self._create_job(task_id="job-2", user_id=self.user.id, status="pending") + self._create_job(task_id="job-2", user_id=self.user.id, status="started") self.client.force_authenticate(self.user) url = reverse("api_v1:jobs:job-list") response = self.client.get(url, {"status": "success", "limit": 10}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(len(response.data), 1) + self.assertEqual(len(response.data["results"]), 1) + self.assertEqual( + set(response.data["results"][0].keys()), + {"task_id", "status", "progress", "name"}, + ) + self.assertEqual(response.data["results"][0]["name"], "task") def test_job_list_limit(self): for idx in range(5): @@ -379,7 +389,7 @@ class BackgroundJobsViewTest(APITestCase): url = reverse("api_v1:jobs:job-list") response = self.client.get(url, {"limit": 2}) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertLessEqual(len(response.data), 2) + self.assertLessEqual(len(response.data["results"]), 2) def test_job_list_invalid_limit_returns_400(self): self._create_job( @@ -389,3 +399,46 @@ class BackgroundJobsViewTest(APITestCase): url = reverse("api_v1:jobs:job-list") response = self.client.get(url, {"limit": "abc"}) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_job_status_maps_started_to_running(self): + job = self._create_job( + task_id="job-running", + user_id=self.user.id, + status="started", + ) + self.client.force_authenticate(self.user) + url = reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) + + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["status"], "running") + + def test_job_stream_returns_completed_sse_event(self): + job = self._create_job( + task_id="job-stream-complete", + user_id=self.user.id, + status="success", + ) + self.client.force_authenticate(self.user) + url = reverse("api_v1:jobs:job-stream", kwargs={"task_id": job.task_id}) + + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + chunks = b"".join(response.streaming_content).decode("utf-8") + self.assertIn("event: completed", chunks) + self.assertIn('"task_id": "job-stream-complete"', chunks) + + def test_job_stream_forbidden_for_other_user(self): + job = self._create_job( + task_id="job-stream-forbidden", + user_id=self.user.id, + status="success", + ) + self.client.force_authenticate(self.other) + url = reverse("api_v1:jobs:job-stream", kwargs={"task_id": job.task_id}) + + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) diff --git a/tests/apps/exchange/test_serializers.py b/tests/apps/exchange/test_serializers.py index b206c39..5cbdc22 100644 --- a/tests/apps/exchange/test_serializers.py +++ b/tests/apps/exchange/test_serializers.py @@ -74,6 +74,7 @@ class ExchangePeriodicTaskUpsertSerializerTest(SimpleTestCase): "table": None, "tables": None, "truncate_before_copy": True, + "notify_on_error": False, }, ) @@ -90,3 +91,18 @@ class ExchangePeriodicTaskUpsertSerializerTest(SimpleTestCase): self.assertFalse(serializer.is_valid()) self.assertIn("table", serializer.errors) + + def test_notify_on_error_is_added_to_payload(self): + serializer = ExchangePeriodicTaskUpsertSerializer( + data={ + "name": "copy-job", + "schedule_type": "interval", + "interval_every": 1, + "interval_period": "hours", + "mode": "all", + "notify_on_error": True, + } + ) + + self.assertTrue(serializer.is_valid(), serializer.errors) + self.assertTrue(serializer.validated_data["payload"]["notify_on_error"]) diff --git a/tests/apps/exchange/test_service_units.py b/tests/apps/exchange/test_service_units.py index dde0442..be42a7d 100644 --- a/tests/apps/exchange/test_service_units.py +++ b/tests/apps/exchange/test_service_units.py @@ -99,7 +99,8 @@ class ExchangeConnectionServiceUnitTest(TestCase): self.assertEqual(result["status"], "success") self.assertEqual( - result["message"], "Подключение и структура целевой БД валидны." + result["message"], + "Подключение проверено. Соединение и структура БД валидны.", ) self.assertEqual(ExchangeConnection.objects.count(), 0) test_connection_mock.assert_called_once() diff --git a/tests/apps/exchange/test_views.py b/tests/apps/exchange/test_views.py index 9071a43..2cf2e9d 100644 --- a/tests/apps/exchange/test_views.py +++ b/tests/apps/exchange/test_views.py @@ -36,8 +36,7 @@ class ExchangeViewsTest(APITestCase): self.client.force_authenticate(self.admin) response = self.client.get(self.connections_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertTrue(response.data["success"]) - self.assertIsInstance(response.data["data"], list) + self.assertIsInstance(response.data["results"], list) @patch("apps.exchange.services.ExchangeConnectionService.validate_target_structure") @patch("apps.exchange.services.ExchangeConnectionService.test_connection") @@ -58,8 +57,20 @@ class ExchangeViewsTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(ExchangeConnection.objects.filter(is_active=True).count(), 1) + self.assertEqual( + set(response.data.keys()), + { + "id", + "server", + "port", + "username", + "database_name", + "schema_name", + "is_active", + }, + ) - new_connection = ExchangeConnection.objects.get(id=response.data["data"]["id"]) + new_connection = ExchangeConnection.objects.get(id=response.data["id"]) self.assertTrue(new_connection.is_active) self.assertNotEqual(new_connection.password, payload["password"]) self.assertEqual(new_connection.get_decrypted_password(), payload["password"]) @@ -89,7 +100,8 @@ class ExchangeViewsTest(APITestCase): response = self.client.post(self.test_connection_url, payload, format="json") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data["data"]["status"], "success") + self.assertTrue(response.data["success"]) + self.assertEqual(response.data["message"], "ok") self.assertEqual(ExchangeConnection.objects.count(), 0) test_connection_mock.assert_called_once_with(**payload) @@ -180,8 +192,7 @@ class ExchangeViewsTest(APITestCase): self.client.force_authenticate(self.admin) response = self.client.get(self.periodic_tasks_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertTrue(response.data["success"]) - self.assertEqual(response.data["data"], []) + self.assertEqual(response.data["results"], []) def test_create_periodic_interval_task_success(self): payload = { @@ -192,17 +203,34 @@ class ExchangeViewsTest(APITestCase): "interval_every": 1, "interval_period": "hours", "mode": "all", + "notify_on_error": True, } self.client.force_authenticate(self.admin) response = self.client.post(self.periodic_tasks_url, payload, format="json") self.assertEqual(response.status_code, status.HTTP_201_CREATED) - task = PeriodicTask.objects.get(id=response.data["data"]["id"]) + task = PeriodicTask.objects.get(id=response.data["id"]) self.assertEqual(task.task, ExchangePeriodicTaskService.TASK_NAME) - self.assertEqual(response.data["data"]["schedule_type"], "interval") - self.assertEqual(response.data["data"]["interval_every"], 1) - self.assertEqual(response.data["data"]["interval_period"], "hours") + self.assertEqual( + set(response.data.keys()), + { + "id", + "schedule_type", + "interval_every", + "interval_period", + "crontab_minute", + "crontab_hour", + "crontab_day_of_week", + "crontab_day_of_month", + "crontab_month_of_year", + "notify_on_error", + }, + ) + self.assertEqual(response.data["schedule_type"], "interval") + self.assertEqual(response.data["interval_every"], 1) + self.assertEqual(response.data["interval_period"], "hours") + self.assertTrue(response.data["notify_on_error"]) self.assertEqual( json.loads(task.kwargs), { @@ -211,6 +239,7 @@ class ExchangeViewsTest(APITestCase): "table": None, "tables": None, "truncate_before_copy": True, + "notify_on_error": True, } }, ) @@ -236,8 +265,8 @@ class ExchangeViewsTest(APITestCase): response = self.client.get(self.periodic_tasks_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(len(response.data["data"]), 1) - self.assertEqual(response.data["data"][0]["name"], "exchange-copy-hourly") + self.assertEqual(len(response.data["results"]), 1) + self.assertEqual(response.data["results"][0]["schedule_type"], "interval") def test_update_periodic_task_switches_to_crontab(self): interval = IntervalSchedule.objects.create(every=1, period="hours") @@ -272,6 +301,7 @@ class ExchangeViewsTest(APITestCase): "mode": "single", "table": "parsers_proxy", "enabled": False, + "notify_on_error": True, } self.client.force_authenticate(self.admin) @@ -283,10 +313,9 @@ class ExchangeViewsTest(APITestCase): self.assertIsNotNone(task.crontab) self.assertEqual(str(task.crontab.timezone), settings.TIME_ZONE) self.assertFalse(task.enabled) - self.assertEqual(response.data["data"]["schedule_type"], "crontab") - self.assertEqual(response.data["data"]["crontab_hour"], "4") - self.assertEqual(response.data["data"]["mode"], "single") - self.assertEqual(response.data["data"]["table"], "parsers_proxy") + self.assertEqual(response.data["schedule_type"], "crontab") + self.assertEqual(response.data["crontab_hour"], "4") + self.assertTrue(response.data["notify_on_error"]) self.assertFalse(IntervalSchedule.objects.filter(id=interval.id).exists()) def test_periodic_task_detail_returns_404_for_non_exchange_task(self): diff --git a/tests/apps/parsers/test_fns_upload.py b/tests/apps/parsers/test_fns_upload.py index 820480f..0ea7f39 100644 --- a/tests/apps/parsers/test_fns_upload.py +++ b/tests/apps/parsers/test_fns_upload.py @@ -181,8 +181,8 @@ class FNSUploadIntegrationTest(APITestCase): jobs_response = self.client.get(reverse("api_v1:jobs:job-list")) self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) - self.assertEqual(len(jobs_response.data), 1) - self.assertEqual(jobs_response.data[0]["task_id"], task_id) + self.assertEqual(len(jobs_response.data["results"]), 1) + self.assertEqual(jobs_response.data["results"][0]["task_id"], task_id) other_client = self.client_class() other_client.force_authenticate(self.other) diff --git a/tests/apps/parsers/test_source_cards_views.py b/tests/apps/parsers/test_source_cards_views.py index a19cdfd..ce715eb 100644 --- a/tests/apps/parsers/test_source_cards_views.py +++ b/tests/apps/parsers/test_source_cards_views.py @@ -181,12 +181,10 @@ class SourceCardsApiTestCase(APITestCase): ) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - self.assertTrue(response.data["success"]) - payload = response.data["data"] - self.assertEqual(payload["source_card"], "financial-indicators") - self.assertEqual(len(payload["tasks"]), 1) + self.assertEqual(response.data["status"], "accepted") + self.assertEqual(set(response.data.keys()), {"task_id", "status"}) - task_id = payload["tasks"][0]["task_id"] + task_id = response.data["task_id"] self.assertTrue( BackgroundJob.objects.filter( task_id=task_id, diff --git a/tests/apps/parsers/test_sources_api_e2e.py b/tests/apps/parsers/test_sources_api_e2e.py index ff9d22b..509a5e7 100644 --- a/tests/apps/parsers/test_sources_api_e2e.py +++ b/tests/apps/parsers/test_sources_api_e2e.py @@ -206,15 +206,19 @@ class SourcesApiE2ETest(APITestCase): self.assertEqual(minprom_response.status_code, status.HTTP_202_ACCEPTED) self.assertEqual(procurements_response.status_code, status.HTTP_202_ACCEPTED) - minprom_tasks = minprom_response.data["data"]["tasks"] + self.assertEqual(minprom_response.data["status"], "accepted") + self.assertEqual(procurements_response.data["status"], "accepted") + + minprom_tasks = minprom_response.data["tasks"] self.assertEqual( [item["task_id"] for item in minprom_tasks], ["task-industrial", "task-products", "task-manufactures"], ) self.assertEqual( - procurements_response.data["data"]["tasks"][0]["task_id"], - "task-procurements", + set(procurements_response.data.keys()), + {"task_id", "status"}, ) + self.assertEqual(procurements_response.data["task_id"], "task-procurements") self.assertTrue( BackgroundJob.objects.filter( task_id="task-procurements", diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 4b531e5..7843da4 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -202,7 +202,7 @@ class ParsersViewSetTest(APITestCase): ) self.assertEqual(response.status_code, status.HTTP_200_OK) - rows = response.data["data"] + rows = response.data["results"] self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["id"], first_log.id) self.assertEqual(rows[0]["organizations_count"], 2) @@ -228,6 +228,23 @@ class ParsersViewSetTest(APITestCase): self.assertIn("organizations_count", content) self.assertIn("333", content) + def test_system_logs_support_search_by_source_label(self): + ParserLoadLogFactory( + source="fns_reports", + batch_id=909, + status="success", + ) + + self.client.force_authenticate(self.admin) + response = self.client.get( + reverse("api_v1:system:parser-logs-list"), + {"search": "финансово"}, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]), 1) + self.assertEqual(response.data["results"][0]["source"], "financial-indicators") + def test_fns_upload_invalid_filename(self): self.client.force_authenticate(self.admin) with tempfile.TemporaryDirectory() as tmpdir: @@ -277,3 +294,46 @@ class ParsersViewSetTest(APITestCase): url, {"files": [upload]}, format="multipart" ) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_fns_upload_accepts_single_file_payload(self): + self.client.force_authenticate(self.admin) + with tempfile.TemporaryDirectory() as tmpdir: + watch_dir = os.path.join(tmpdir, "watch") + processed_dir = os.path.join(tmpdir, "processed") + failed_dir = os.path.join(tmpdir, "failed") + content = _build_fns_excel_bytes() + upload = SimpleUploadedFile( + f"fin_{_digits(5)}_{_digits(13)}.xlsx", + content, + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + with self.settings( + FNS_WATCH_DIRECTORY=watch_dir, + FNS_PROCESSED_DIRECTORY=processed_dir, + FNS_FAILED_DIRECTORY=failed_dir, + ): + url = reverse("api_v1:fns:fns-upload") + response = self.client.post(url, {"file": upload}, format="multipart") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["success"], True) + self.assertIn("message", response.data) + + def test_parsing_settings_get_and_patch(self): + self.client.force_authenticate(self.admin) + url = reverse("api_v1:parsing:parsing-settings") + + initial = self.client.get(url) + self.assertEqual(initial.status_code, status.HTTP_200_OK) + self.assertEqual(initial.data["planned_inspections"], "monthly") + + updated = self.client.patch( + url, + {"planned_inspections": "weekly"}, + format="json", + ) + + self.assertEqual(updated.status_code, status.HTTP_200_OK) + self.assertEqual(updated.data["planned_inspections"], "weekly") diff --git a/tests/apps/registers/test_generate_test_data_command.py b/tests/apps/registers/test_generate_test_data_command.py index 36d8554..89547d7 100644 --- a/tests/apps/registers/test_generate_test_data_command.py +++ b/tests/apps/registers/test_generate_test_data_command.py @@ -24,6 +24,7 @@ class GenerateTestDataCommandTest(TestCase): def test_command_creates_data_for_every_registry(self): """Команда создаёт данные по всем целевым реестрам.""" + initial_registers_count = Register.objects.count() call_command( "generate_test_data", count=2, @@ -32,7 +33,7 @@ class GenerateTestDataCommandTest(TestCase): verbosity=0, ) - self.assertEqual(Register.objects.count(), 3) + self.assertEqual(Register.objects.count(), initial_registers_count + 3) self.assertEqual(RegisterUpload.objects.count(), 6) self.assertEqual(RegistryMembershipPeriod.objects.count(), 6) self.assertEqual(Organization.objects.count(), 6) @@ -51,6 +52,7 @@ class GenerateTestDataCommandTest(TestCase): def test_command_dry_run_rolls_back_everything(self): """В dry-run режиме все изменения откатываются.""" + initial_registers_count = Register.objects.count() call_command( "generate_test_data", count=1, @@ -60,7 +62,7 @@ class GenerateTestDataCommandTest(TestCase): verbosity=0, ) - self.assertEqual(Register.objects.count(), 0) + self.assertEqual(Register.objects.count(), initial_registers_count) self.assertEqual(RegisterUpload.objects.count(), 0) self.assertEqual(RegistryMembershipPeriod.objects.count(), 0) self.assertEqual(Organization.objects.count(), 0) diff --git a/tests/apps/registers/test_views.py b/tests/apps/registers/test_views.py index 214bfb0..ebe444a 100644 --- a/tests/apps/registers/test_views.py +++ b/tests/apps/registers/test_views.py @@ -106,6 +106,15 @@ class RegistersViewsTest(APITestCase): self.assertEqual(detail_response.status_code, status.HTTP_200_OK) self.assertEqual(detail_response.data["name"], "Росатом") + def test_default_registries_are_seeded(self): + response = self.client.get(reverse("api_v1:registers:registries-list")) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + names = {item["name"] for item in _extract_results(response.data)} + self.assertIn("Реестр предприятий ОПК", names) + self.assertIn("Реестр госкорпорации Роскосмос", names) + self.assertIn("Реестр госкорпорации Росатом", names) + def test_organizations_list_and_retrieve(self): organization = OrganizationFactory() @@ -245,8 +254,8 @@ class RegistersViewsTest(APITestCase): file_name="first.xlsx", ) self.assertEqual(first.status_code, status.HTTP_201_CREATED) - self.assertEqual(first.data["opened_periods"], 1) - self.assertEqual(first.data["closed_periods"], 0) + self.assertTrue(first.data["success"]) + self.assertEqual(first.data["message"], "Файл успешно загружен") second = self._post_upload( registry=registry, @@ -255,8 +264,8 @@ class RegistersViewsTest(APITestCase): file_name="second.xlsx", ) self.assertEqual(second.status_code, status.HTTP_201_CREATED) - self.assertEqual(second.data["opened_periods"], 1) - self.assertEqual(second.data["closed_periods"], 1) + self.assertTrue(second.data["success"]) + self.assertEqual(second.data["message"], "Файл успешно загружен") third = self._post_upload( registry=registry, @@ -265,8 +274,8 @@ class RegistersViewsTest(APITestCase): file_name="third.xlsx", ) self.assertEqual(third.status_code, status.HTTP_201_CREATED) - self.assertEqual(third.data["opened_periods"], 1) - self.assertEqual(third.data["closed_periods"], 1) + self.assertTrue(third.data["success"]) + self.assertEqual(third.data["message"], "Файл успешно загружен") organization_a = Organization.objects.get( mn_ogrn=1027600980990, mn_inn=7601000086 @@ -371,6 +380,8 @@ class RegistersViewsTest(APITestCase): ) self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertTrue(response.data["success"]) + self.assertEqual(response.data["message"], "Файл успешно загружен") organization = Organization.objects.get( mn_ogrn=1027600980990, mn_inn=7601000086 ) diff --git a/tests/apps/user/test_models.py b/tests/apps/user/test_models.py index 47ff02b..6abd2bd 100644 --- a/tests/apps/user/test_models.py +++ b/tests/apps/user/test_models.py @@ -80,23 +80,23 @@ class ProfileModelTest(TestCase): """Test OneToOne relationship with User""" self.assertIsNotNone(self.profile.user) - def test_profile_first_name_optional(self): - """Test first_name field is optional""" + def test_profile_first_name_required(self): + """Test first_name field is required and stored without NULL.""" field = self.profile._meta.get_field("first_name") - self.assertTrue(field.blank) - self.assertTrue(field.null) + self.assertFalse(field.blank) + self.assertFalse(field.null) - def test_profile_last_name_optional(self): - """Test last_name field is optional""" + def test_profile_last_name_required(self): + """Test last_name field is required and stored without NULL.""" field = self.profile._meta.get_field("last_name") - self.assertTrue(field.blank) - self.assertTrue(field.null) + self.assertFalse(field.blank) + self.assertFalse(field.null) - def test_profile_middle_name_optional(self): - """Test middle_name field is optional""" + def test_profile_middle_name_optional_but_not_null(self): + """Test middle_name remains optional but stored without NULL.""" field = self.profile._meta.get_field("middle_name") self.assertTrue(field.blank) - self.assertTrue(field.null) + self.assertFalse(field.null) def test_profile_bio_optional(self): """Test bio field is optional""" @@ -126,7 +126,7 @@ class ProfileModelTest(TestCase): self.profile.middle_name = middle_name self.profile.last_name = last_name self.assertEqual( - self.profile.full_name, f"{first_name} {middle_name} {last_name}" + self.profile.full_name, f"{last_name} {first_name} {middle_name}" ) # Test with only first name diff --git a/tests/apps/user/test_serializers.py b/tests/apps/user/test_serializers.py index cbfada9..b6d850c 100644 --- a/tests/apps/user/test_serializers.py +++ b/tests/apps/user/test_serializers.py @@ -33,6 +33,9 @@ class UserRegistrationSerializerTest(TestCase): "password": self.password, "password_confirm": self.password, "phone": f"+7{fake.numerify('##########')}", + "first_name": fake.first_name(), + "middle_name": fake.first_name(), + "last_name": fake.last_name(), } def test_valid_registration_data(self): @@ -95,6 +98,20 @@ class UserRegistrationSerializerTest(TestCase): self.assertEqual(user.email, self.user_data["email"]) self.assertEqual(user.username, self.user_data["username"]) self.assertTrue(user.check_password(self.user_data["password"])) + self.assertEqual(user.profile.first_name, self.user_data["first_name"]) + self.assertEqual(user.profile.middle_name, self.user_data["middle_name"]) + self.assertEqual(user.profile.last_name, self.user_data["last_name"]) + + def test_registration_requires_first_and_last_name(self): + data = self.user_data.copy() + data.pop("first_name") + data.pop("last_name") + + serializer = UserRegistrationSerializer(data=data) + + self.assertFalse(serializer.is_valid()) + self.assertIn("first_name", serializer.errors) + self.assertIn("last_name", serializer.errors) class UserSerializerTest(TestCase): @@ -217,6 +234,7 @@ class AdminUserUpdateSerializerTest(TestCase): "is_active": False, "first_name": fake.first_name(), "middle_name": fake.first_name(), + "last_name": fake.last_name(), }, partial=True, ) diff --git a/tests/apps/user/test_services.py b/tests/apps/user/test_services.py index f819922..0d3833e 100644 --- a/tests/apps/user/test_services.py +++ b/tests/apps/user/test_services.py @@ -57,6 +57,18 @@ class UserServiceTest(TestCase): self.assertEqual(UserService.get_user_role(user), UserService.ROLE_ADMIN) self.assertTrue(user.groups.filter(name=UserService.ROLE_ADMIN).exists()) + def test_create_user_with_profile_names(self): + user = UserService.create_user( + **self.user_data, + first_name="Иван", + middle_name="Иванович", + last_name="Иванов", + ) + + self.assertEqual(user.profile.first_name, "Иван") + self.assertEqual(user.profile.middle_name, "Иванович") + self.assertEqual(user.profile.last_name, "Иванов") + def test_get_user_by_email_found(self): """Test getting user by existing email""" found_user = UserService.get_user_by_email(self.user.email) @@ -191,8 +203,8 @@ class UserServiceTest(TestCase): queryset = UserService.get_filtered_users_queryset(ordering="first_name") - ids = list(queryset.values_list("id", flat=True)[:2]) - self.assertEqual(ids, [second.id, first.id]) + ids = list(queryset.values_list("id", flat=True)) + self.assertLess(ids.index(second.id), ids.index(first.id)) def test_get_user_capabilities_for_admin(self): """Test admin capabilities set.""" diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index cca4230..6dabd2b 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -7,6 +7,7 @@ from django.urls import reverse from faker import Faker from rest_framework import status from rest_framework.test import APIClient, APITestCase +from rest_framework_simplejwt.token_blacklist.models import BlacklistedToken from .factories import ProfileFactory, UserFactory @@ -26,6 +27,9 @@ class RegisterViewTest(APITestCase): "password": self.password, "password_confirm": self.password, "phone": f"+7{fake.numerify('##########')}", + "first_name": fake.first_name(), + "middle_name": fake.first_name(), + "last_name": fake.last_name(), } def test_register_success(self): @@ -39,7 +43,9 @@ class RegisterViewTest(APITestCase): self.assertIn("access", response.data["tokens"]) # Verify user was created - self.assertTrue(User.objects.filter(email=self.user_data["email"]).exists()) + created_user = User.objects.get(email=self.user_data["email"]) + self.assertEqual(created_user.profile.first_name, self.user_data["first_name"]) + self.assertEqual(created_user.profile.last_name, self.user_data["last_name"]) def test_register_passwords_do_not_match(self): """Test registration fails when passwords don't match""" @@ -77,6 +83,17 @@ class RegisterViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn("password", response.data) + def test_register_requires_first_and_last_name(self): + data = self.user_data.copy() + data.pop("first_name") + data.pop("last_name") + + response = self.client.post(self.register_url, data, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("first_name", response.data) + self.assertIn("last_name", response.data) + class LoginViewTest(APITestCase): """Tests for LoginView""" @@ -125,7 +142,7 @@ class LoginViewTest(APITestCase): class LogoutViewTest(APITestCase): - def test_logout_returns_success_message(self): + def test_logout_blacklists_refresh_tokens_and_returns_empty_payload(self): user = UserFactory.create_user() tokens = UserService.get_tokens_for_user(user) self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {tokens['access']}") @@ -133,7 +150,8 @@ class LogoutViewTest(APITestCase): response = self.client.post(reverse("api_v1:user:logout"), {}, format="json") self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data["message"], "Успешный выход") + self.assertEqual(response.data, {}) + self.assertEqual(BlacklistedToken.objects.filter(token__user=user).count(), 1) class CurrentUserViewTest(APITestCase): @@ -151,11 +169,26 @@ class CurrentUserViewTest(APITestCase): response = self.client.get(self.current_user_url) self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + set(response.data.keys()), + { + "id", + "username", + "email", + "phone", + "is_active", + "role", + "role_label", + "profile", + }, + ) self.assertEqual(response.data["id"], self.user.id) self.assertEqual(response.data["email"], self.user.email) self.assertEqual(response.data["role"], "user") - self.assertFalse(response.data["capabilities"]["can_refresh_dashboard"]) - self.assertIn("profile", response.data) + self.assertEqual( + set(response.data["profile"].keys()), + {"first_name", "middle_name", "last_name", "full_name"}, + ) def test_get_current_user_unauthenticated(self): """Test getting current user when unauthenticated""" @@ -220,7 +253,24 @@ class AdminUserManagementViewTest(APITestCase): response = self.client.get(self.list_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - usernames = {item["username"] for item in response.data} + self.assertEqual( + set(response.data["results"][0].keys()), + { + "id", + "username", + "email", + "phone", + "is_active", + "role", + "role_label", + "profile", + }, + ) + self.assertEqual( + set(response.data["results"][0]["profile"].keys()), + {"first_name", "middle_name", "last_name", "full_name"}, + ) + usernames = {item["username"] for item in response.data["results"]} self.assertIn(self.admin.username, usernames) self.assertIn(self.user.username, usernames) @@ -237,7 +287,7 @@ class AdminUserManagementViewTest(APITestCase): response = self.client.get(self.list_url, {"search": "Петрович"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - usernames = [item["username"] for item in response.data] + usernames = [item["username"] for item in response.data["results"]] self.assertEqual(usernames, [self.user.username]) def test_admin_can_order_users(self): @@ -249,7 +299,7 @@ class AdminUserManagementViewTest(APITestCase): response = self.client.get(self.list_url, {"ordering": "first_name"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - ordered_ids = [item["id"] for item in response.data] + ordered_ids = [item["id"] for item in response.data["results"]] self.assertLess(ordered_ids.index(second.id), ordered_ids.index(first.id)) def test_admin_can_create_user_with_role(self): @@ -270,6 +320,18 @@ class AdminUserManagementViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_201_CREATED) created = User.objects.get(username=payload["username"]) self.assertTrue(created.is_staff) + self.assertEqual( + set(response.data.keys()), + { + "id", + "username", + "email", + "phone", + "role", + "role_label", + "is_active", + }, + ) self.assertEqual(response.data["role"], "admin") self.assertEqual(created.profile.first_name, "Петр") self.assertEqual(created.profile.middle_name, "Петрович") @@ -283,6 +345,7 @@ class AdminUserManagementViewTest(APITestCase): "role": "admin", "first_name": "Иван", "middle_name": "Иванович", + "last_name": "Иванов", "is_verified": True, }, format="json", @@ -292,6 +355,18 @@ class AdminUserManagementViewTest(APITestCase): self.user.refresh_from_db() self.assertTrue(self.user.is_staff) self.assertTrue(self.user.is_verified) + self.assertEqual( + set(response.data.keys()), + { + "id", + "username", + "email", + "phone", + "role", + "role_label", + "is_active", + }, + ) self.assertEqual(self.user.profile.first_name, "Иван") self.assertEqual(self.user.profile.middle_name, "Иванович") @@ -302,6 +377,10 @@ class AdminUserManagementViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["id"], self.user.id) + self.assertEqual( + set(response.data["profile"].keys()), + {"first_name", "middle_name", "last_name", "full_name"}, + ) def test_admin_cannot_patch_self_to_inactive(self): url = reverse("api_v1:user:admin-user-detail", args=[self.admin.id]) @@ -324,10 +403,15 @@ class AdminUserManagementViewTest(APITestCase): def test_admin_can_patch_self_with_safe_fields(self): url = reverse("api_v1:user:admin-user-detail", args=[self.admin.id]) - response = self.client.patch(url, {"first_name": "Админ"}, format="json") + response = self.client.patch( + url, + {"first_name": "Админ", "last_name": "Админов"}, + format="json", + ) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data["profile"]["first_name"], "Админ") + self.assertEqual(response.data["role"], "admin") + self.assertNotIn("profile", response.data) def test_admin_can_deactivate_user(self): url = reverse("api_v1:user:admin-user-deactivate", args=[self.user.id]) @@ -348,6 +432,18 @@ class AdminUserManagementViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.user.refresh_from_db() self.assertTrue(self.user.is_active) + self.assertEqual( + set(response.data.keys()), + { + "id", + "username", + "email", + "phone", + "role", + "role_label", + "is_active", + }, + ) def test_admin_cannot_deactivate_self(self): url = reverse("api_v1:user:admin-user-deactivate", args=[self.admin.id])