diff --git a/.gitea/workflows/ci-cd.yml b/.gitea/workflows/ci-cd.yml index 04d38d1..fb5c484 100644 --- a/.gitea/workflows/ci-cd.yml +++ b/.gitea/workflows/ci-cd.yml @@ -46,13 +46,9 @@ jobs: timeout 180s "${BOOTSTRAP_PYTHON}" -m pip install --user --break-system-packages --upgrade pip uv export PATH="$HOME/.local/bin:$PATH" - - if "${BOOTSTRAP_PYTHON}" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)'; then - PYTHON_BIN="${BOOTSTRAP_PYTHON}" - else - timeout 300s uv python install "${PYTHON_VERSION}" - PYTHON_BIN="${PYTHON_VERSION}" - fi + PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" + timeout 300s uv python install "${PROJECT_PYTHON_VERSION}" + PYTHON_BIN="$(uv python find --managed-python "${PROJECT_PYTHON_VERSION}")" printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env @@ -137,13 +133,9 @@ jobs: timeout 180s "${BOOTSTRAP_PYTHON}" -m pip install --user --break-system-packages --upgrade pip uv export PATH="$HOME/.local/bin:$PATH" - - if "${BOOTSTRAP_PYTHON}" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)'; then - PYTHON_BIN="${BOOTSTRAP_PYTHON}" - else - timeout 300s uv python install "${PYTHON_VERSION}" - PYTHON_BIN="${PYTHON_VERSION}" - fi + PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" + timeout 300s uv python install "${PROJECT_PYTHON_VERSION}" + PYTHON_BIN="$(uv python find --managed-python "${PROJECT_PYTHON_VERSION}")" printf 'PYTHON_BIN=%s\n' "${PYTHON_BIN}" > .ci-python-env @@ -156,15 +148,36 @@ jobs: . .venv/bin/activate uv sync --dev --frozen - - name: Run Django tests + - name: Run regular pytest suite env: DJANGO_SETTINGS_MODULE: settings.test SECRET_KEY: test-secret-key-for-ci run: | set -euo pipefail - . .venv/bin/activate export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" - python src/manage.py test tests --verbosity=2 + .venv/bin/python -m pytest tests --ignore=tests/test_api_inventory_e2e.py -q + + - name: Pack prepared test workspace + if: success() + run: | + set -euo pipefail + WORKSPACE_ARCHIVE="/tmp/ci-test-workspace.tar.gz" + tar \ + --exclude='.git' \ + --exclude='.pytest_cache' \ + --exclude='htmlcov' \ + --exclude='__pycache__' \ + -czf "${WORKSPACE_ARCHIVE}" \ + . + + - name: Upload prepared test workspace + if: success() + uses: actions/upload-artifact@v3 + with: + name: ci-test-workspace + path: /tmp/ci-test-workspace.tar.gz + if-no-files-found: error + retention-days: 1 - name: Telegram notify (test failed) if: failure() @@ -195,20 +208,99 @@ jobs: --data-urlencode "text=${MSG}" \ || echo "Telegram notification failed; continue pipeline" + test_api_inventory_e2e: + name: Run API Inventory E2E Tests + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: [test] + if: ${{ needs.test.result == 'success' }} + env: + TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} + TG_CHANNEL: ${{ secrets.TG_CHANNEL }} + + steps: + - name: Download prepared test workspace + uses: actions/download-artifact@v3 + with: + name: ci-test-workspace + + - name: Extract prepared test workspace + run: | + set -euo pipefail + ARCHIVE_PATH="$(find . -maxdepth 2 -name 'ci-test-workspace.tar.gz' -print -quit)" + if [ -z "${ARCHIVE_PATH}" ]; then + echo "ci-test-workspace.tar.gz not found after artifact download" >&2 + exit 1 + fi + tar -xzf "${ARCHIVE_PATH}" + + - name: Install Python and uv for artifact environment + run: | + set -euo pipefail + if command -v python3.11 >/dev/null 2>&1; then + BOOTSTRAP_PYTHON=python3.11 + elif command -v python3 >/dev/null 2>&1; then + BOOTSTRAP_PYTHON=python3 + else + echo "python3 is not available on the runner" >&2 + exit 1 + fi + + timeout 180s "${BOOTSTRAP_PYTHON}" -m pip install --user --break-system-packages --upgrade pip uv + export PATH="$HOME/.local/bin:$PATH" + PROJECT_PYTHON_VERSION="$(cat .python-version 2>/dev/null || printf '%s' "${PYTHON_VERSION}")" + timeout 300s uv python install "${PROJECT_PYTHON_VERSION}" + + - name: Run API inventory pytest suite + env: + DJANGO_SETTINGS_MODULE: settings.test + SECRET_KEY: test-secret-key-for-ci + run: | + set -euo pipefail + export PYTHONPATH="${PWD}/src:${PYTHONPATH:-}" + .venv/bin/python -m pytest tests/test_api_inventory_e2e.py -q + + - name: Telegram notify (api inventory e2e failed) + if: failure() + continue-on-error: true + run: | + set -euo pipefail + if [ -z "${TG_BOT_KEY:-}" ] || [ -z "${TG_CHANNEL:-}" ]; then + echo "TG_BOT_KEY or TG_CHANNEL is not set; skip telegram notification" + exit 0 + fi + + MSG="❌ [mostovik-backend] api inventory e2e failed + branch=${GITHUB_REF_NAME} + sha=${GITHUB_SHA} + actor=${GITHUB_ACTOR}" + + curl -fsS \ + --connect-timeout 5 \ + --max-time 15 \ + --retry 2 \ + --retry-delay 2 \ + --retry-all-errors \ + -X POST "https://api.telegram.org/bot${TG_BOT_KEY}/sendMessage" \ + -d "chat_id=${TG_CHANNEL}" \ + --data-urlencode "text=${MSG}" \ + || echo "Telegram notification failed; continue pipeline" + notify_success: name: Telegram Notify Success runs-on: ubuntu-latest timeout-minutes: 1 - needs: [lint, test] + needs: [lint, test, test_api_inventory_e2e] if: | always() && needs.lint.result == 'success' && - needs.test.result == 'success' + needs.test.result == 'success' && + needs.test_api_inventory_e2e.result == 'success' env: TG_BOT_KEY: ${{ secrets.TG_BOT_KEY }} TG_CHANNEL: ${{ secrets.TG_CHANNEL }} steps: - - name: Telegram notify (lint+test success) + - name: Telegram notify (lint+tests+e2e success) continue-on-error: true env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} @@ -219,7 +311,7 @@ jobs: exit 0 fi - MSG="✅ [mostovik-backend] lint + tests passed + MSG="✅ [mostovik-backend] lint + tests + api inventory e2e passed branch=${GITHUB_REF_NAME} sha=${GITHUB_SHA} actor=${GITHUB_ACTOR} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4fa0ef6..cd6db37 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -84,11 +84,16 @@ services: target: runtime-celery container_name: mostovik_celery_worker restart: unless-stopped + environment: + CELERY_WORKER_CONCURRENCY: "1" + CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "3145728" env_file: - .env.dev depends_on: migrate: condition: service_completed_successfully + mem_limit: 3g + memswap_limit: 3g volumes: - ./src:/app/src - ./logs:/app/logs diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 73e2e4b..02df776 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -48,11 +48,16 @@ services: image: ${CELERY_IMAGE:-mostovik/celery:latest} container_name: mostovik_celery_worker restart: unless-stopped + environment: + CELERY_WORKER_CONCURRENCY: "1" + CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "3145728" env_file: - .env.prod depends_on: migrate: condition: service_completed_successfully + mem_limit: 3g + memswap_limit: 3g volumes: - ./logs:/app/logs - ./input:/app/input diff --git a/docker/scripts/start-celery-worker.sh b/docker/scripts/start-celery-worker.sh index 757bdab..68e394a 100755 --- a/docker/scripts/start-celery-worker.sh +++ b/docker/scripts/start-celery-worker.sh @@ -17,4 +17,5 @@ esac exec celery -A core worker \ --loglevel="${CELERY_LOG_LEVEL:-INFO}" \ - --concurrency="${CELERY_WORKER_CONCURRENCY:-2}" + --concurrency="${CELERY_WORKER_CONCURRENCY:-1}" \ + --max-memory-per-child="${CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB:-3145728}" diff --git a/src/apps/parsers/admin.py b/src/apps/parsers/admin.py index 5e3523b..ecf0bb3 100644 --- a/src/apps/parsers/admin.py +++ b/src/apps/parsers/admin.py @@ -28,19 +28,32 @@ class ProxyAdmin(admin.ModelAdmin): list_display = [ "address", + "country_code", + "source", "is_active_badge", "fail_count", "last_used_at", "created_at", ] - list_filter = ["is_active", "created_at"] - search_fields = ["address"] + list_filter = ["is_active", "country_code", "source", "created_at"] + search_fields = ["address", "country_code", "source", "description"] readonly_fields = ["created_at", "updated_at", "last_used_at"] ordering = ["-is_active", "-last_used_at"] list_per_page = 50 fieldsets = ( - ("Основное", {"fields": ("address", "is_active")}), + ( + "Основное", + { + "fields": ( + "address", + "country_code", + "source", + "description", + "is_active", + ) + }, + ), ("Статистика", {"fields": ("fail_count", "last_used_at")}), ("Даты", {"fields": ("created_at", "updated_at"), "classes": ("collapse",)}), ) diff --git a/src/apps/parsers/clients/proxy_tools.py b/src/apps/parsers/clients/proxy_tools.py new file mode 100644 index 0000000..97b58b0 --- /dev/null +++ b/src/apps/parsers/clients/proxy_tools.py @@ -0,0 +1,83 @@ +""" +Клиент Proxy-Tools JSON API. + +Документация: +https://proxy-tools.com/pages/proxy-api +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import Any + +from apps.parsers.clients.base import BaseHTTPClient, HTTPClientError +from requests.adapters import BaseAdapter + +logger = logging.getLogger(__name__) + +DEFAULT_API_URL = "https://proxy-tools.com/api/v1/proxies" + + +class ProxyToolsClientError(HTTPClientError): + """Ошибка клиента Proxy-Tools.""" + + +@dataclass +class ProxyToolsClient: + """ + Клиент для загрузки списка прокси из Proxy-Tools. + + Использует Bearer token и возвращает сырой JSON payload, потому что + внешний сервис документирует фильтры, но не фиксирует shape ответа. + """ + + api_key: str + api_url: str = DEFAULT_API_URL + timeout: int = 30 + http_adapter: BaseAdapter | None = None + _http_client: BaseHTTPClient | None = field(default=None, repr=False) + + @property + def http_client(self) -> BaseHTTPClient: + """Ленивая инициализация HTTP клиента.""" + if self._http_client is None: + self._http_client = BaseHTTPClient( + base_url="https://proxy-tools.com", + timeout=self.timeout, + adapter=self.http_adapter, + headers={ + "Accept": "application/json", + "Authorization": f"Bearer {self.api_key}", + }, + ) + return self._http_client + + def fetch_proxies( + self, + *, + country_code: str, + page: int = 1, + limit: int = 100, + ) -> Any: + """Получить страницу прокси по коду страны.""" + params = { + "geo": country_code.lower(), + "page": str(page), + "limit": str(limit), + } + logger.info( + "Fetching proxies from Proxy-Tools (country=%s, page=%s, limit=%s)", + country_code, + page, + limit, + ) + try: + response = self.http_client.get(self.api_url, params=params) + return response.json() + except HTTPClientError: + raise + except Exception as exc: # noqa: BLE001 + raise ProxyToolsClientError( + f"Failed to fetch proxies from Proxy-Tools: {exc}" + ) from exc diff --git a/src/apps/parsers/migrations/0015_add_proxy_metadata.py b/src/apps/parsers/migrations/0015_add_proxy_metadata.py new file mode 100644 index 0000000..883dba6 --- /dev/null +++ b/src/apps/parsers/migrations/0015_add_proxy_metadata.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2.25 on 2026-03-23 10:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("parsers", "0014_parsingsettings"), + ] + + operations = [ + migrations.AddField( + model_name="proxy", + name="country_code", + field=models.CharField( + db_index=True, + default="RU", + help_text="ISO-3166 код страны прокси, например RU", + max_length=2, + verbose_name="код страны", + ), + ), + migrations.AddField( + model_name="proxy", + name="source", + field=models.CharField( + db_index=True, + default="manual", + help_text="Источник прокси (например: manual, proxy-tools)", + max_length=50, + verbose_name="источник", + ), + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index d14f01f..1a20815 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -351,6 +351,20 @@ class Proxy(TimestampMixin, models.Model): blank=True, help_text=_("Описание прокси (провайдер, локация и т.д.)"), ) + source = models.CharField( + _("источник"), + max_length=50, + default="manual", + db_index=True, + help_text=_("Источник прокси (например: manual, proxy-tools)"), + ) + country_code = models.CharField( + _("код страны"), + max_length=2, + default="RU", + db_index=True, + help_text=_("ISO-3166 код страны прокси, например RU"), + ) class Meta: db_table = "parsers_proxy" diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index e27fd5f..a8fb6e4 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -15,7 +15,6 @@ from apps.parsers.models import ( ParserLoadLog, ParsingSettings, ProcurementRecord, - Proxy, ) from rest_framework import serializers @@ -430,28 +429,6 @@ class ParserLoadLogListSerializer(serializers.Serializer): updated_at = serializers.DateTimeField(read_only=True) -class ProxySerializer(serializers.ModelSerializer): - """ - Прокси-сервер для парсеров. - - Используется для обхода блокировок при парсинге. - """ - - class Meta: - model = Proxy - fields = [ - "id", - "address", - "is_active", - "last_used_at", - "fail_count", - "description", - "created_at", - "updated_at", - ] - read_only_fields = fields - - class SourceCardRefreshParamSerializer(serializers.Serializer): """Описание параметра ручного обновления карточки источника.""" diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index 6f689b5..87bd745 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -11,6 +11,8 @@ from contextlib import suppress from dataclasses import dataclass from datetime import date, datetime from decimal import Decimal, InvalidOperation +from typing import Any +from urllib.parse import urlparse from apps.core.services import BaseService, BulkOperationsMixin from apps.parsers.clients.minpromtorg.schemas import ( @@ -19,6 +21,7 @@ from apps.parsers.clients.minpromtorg.schemas import ( Manufacturer, ) from apps.parsers.clients.proverki.schemas import Inspection +from apps.parsers.clients.proxy_tools import ProxyToolsClient, ProxyToolsClientError from apps.parsers.clients.zakupki.schemas import Procurement from apps.parsers.models import ( FinancialReport, @@ -32,6 +35,7 @@ from apps.parsers.models import ( Proxy, ) from apps.registers.models import Organization +from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import Q from django.utils import timezone @@ -639,29 +643,68 @@ class ProxyService(BaseService[Proxy]): """ model = Proxy + RUNTIME_COUNTRY_CODE = "RU" + MANUAL_SOURCE = "manual" + PROXY_TOOLS_SOURCE = "proxy-tools" @classmethod - def get_active_proxies(cls) -> list[str]: + def get_active_proxies( + cls, + *, + country_code: str | None = None, + source: str | None = None, + ) -> list[str]: """ Получить список адресов активных прокси. Returns: Список адресов прокси (может быть пустым) """ - proxies = cls.model.objects.filter(is_active=True).values_list( - "address", flat=True - ) + proxies = cls.model.objects.filter(is_active=True) + if country_code: + proxies = proxies.filter(country_code=country_code.upper()) + if source: + proxies = proxies.filter(source=source) + proxies = proxies.values_list("address", flat=True) return list(proxies) @classmethod - def get_active_proxies_or_none(cls) -> list[str] | None: + def get_active_proxies_or_none( + cls, + *, + country_code: str | None = None, + source: str | None = None, + ) -> list[str] | None: """ Получить список активных прокси или None, если их нет. Returns: Список адресов прокси или None """ - proxies = cls.get_active_proxies() + proxies = cls.get_active_proxies(country_code=country_code, source=source) + return proxies if proxies else None + + @classmethod + def get_runtime_proxies(cls) -> list[str]: + """ + Получить прокси для рантайма парсеров. + + Приоритет: + 1. RU прокси, загруженные из Proxy-Tools + 2. Любые активные RU прокси + """ + proxies = cls.get_active_proxies( + country_code=cls.RUNTIME_COUNTRY_CODE, + source=cls.PROXY_TOOLS_SOURCE, + ) + if proxies: + return proxies + return cls.get_active_proxies(country_code=cls.RUNTIME_COUNTRY_CODE) + + @classmethod + def get_runtime_proxies_or_none(cls) -> list[str] | None: + """Получить runtime-прокси или None, если их нет.""" + proxies = cls.get_runtime_proxies() return proxies if proxies else None @classmethod @@ -698,31 +741,53 @@ class ProxyService(BaseService[Proxy]): @classmethod @transaction.atomic - def add_proxy(cls, address: str, description: str = "") -> Proxy: + def add_proxy( + cls, + address: str, + description: str = "", + *, + source: str = MANUAL_SOURCE, + country_code: str = RUNTIME_COUNTRY_CODE, + ) -> Proxy: """ Добавить новый прокси. Args: address: Адрес прокси (например: http://proxy:8080) description: Описание прокси + source: Источник прокси + country_code: ISO-код страны Returns: Созданный объект Proxy """ proxy, _ = cls.model.objects.get_or_create( address=address, - defaults={"description": description, "is_active": True}, + defaults={ + "description": description, + "is_active": True, + "source": source, + "country_code": country_code.upper(), + }, ) return proxy @classmethod @transaction.atomic - def add_proxies(cls, addresses: list[str]) -> int: + def add_proxies( + cls, + addresses: list[str], + *, + source: str = MANUAL_SOURCE, + country_code: str = RUNTIME_COUNTRY_CODE, + ) -> int: """ Добавить список прокси. Args: addresses: Список адресов прокси + source: Источник прокси + country_code: ISO-код страны Returns: Количество добавленных прокси @@ -731,13 +796,300 @@ class ProxyService(BaseService[Proxy]): for address in addresses: _, created = cls.model.objects.get_or_create( address=address, - defaults={"is_active": True}, + defaults={ + "is_active": True, + "source": source, + "country_code": country_code.upper(), + }, ) if created: created_count += 1 return created_count +class ProxyToolsSyncError(Exception): + """Ошибка синхронизации прокси из Proxy-Tools.""" + + +class ProxyToolsSyncService: + """Сервис синхронизации RU-прокси из Proxy-Tools.""" + + COUNTRY_CODE = ProxyService.RUNTIME_COUNTRY_CODE + SOURCE = ProxyService.PROXY_TOOLS_SOURCE + + @classmethod + def sync_ru_proxies(cls) -> dict[str, int | str]: + """Загрузить RU-прокси из Proxy-Tools и синхронизировать таблицу.""" + api_key = getattr(settings, "PROXY_TOOLS_API_KEY", "").strip() + if not api_key: + logger.warning("Proxy-Tools sync skipped: PROXY_TOOLS_API_KEY is empty") + return { + "status": "skipped", + "reason": "missing_api_key", + "fetched": 0, + "created": 0, + "updated": 0, + "deactivated": 0, + } + + client = ProxyToolsClient( + api_key=api_key, + api_url=settings.PROXY_TOOLS_API_URL, + timeout=int(getattr(settings, "PROXY_TOOLS_TIMEOUT_SECONDS", 30)), + ) + limit = int(getattr(settings, "PROXY_TOOLS_LIMIT", 100)) + max_pages = max(int(getattr(settings, "PROXY_TOOLS_MAX_PAGES", 3)), 1) + + try: + items = cls._fetch_all_pages( + client=client, limit=limit, max_pages=max_pages + ) + addresses = cls._extract_addresses(items) + except ProxyToolsClientError as exc: + raise ProxyToolsSyncError(str(exc)) from exc + + result = cls._sync_addresses(addresses) + return { + "status": "success", + "fetched": len(addresses), + **result, + } + + @classmethod + def _fetch_all_pages( + cls, + *, + client: ProxyToolsClient, + limit: int, + max_pages: int, + ) -> list[Any]: + """Собрать прокси с нескольких страниц, если API их отдаёт.""" + items: list[Any] = [] + for page in range(1, max_pages + 1): + payload = client.fetch_proxies( + country_code=cls.COUNTRY_CODE, + page=page, + limit=limit, + ) + batch = cls._extract_items(payload) + items.extend(batch) + if not cls._has_more_pages( + payload, page=page, batch_size=len(batch), limit=limit + ): + break + return items + + @classmethod + def _extract_items(cls, payload: Any) -> list[Any]: + """Извлечь список элементов прокси из внешнего payload.""" + if isinstance(payload, list): + return payload + if isinstance(payload, dict): + for key in ("proxies", "data", "results", "items"): + value = payload.get(key) + if isinstance(value, list): + return value + if isinstance(value, dict): + with suppress(ProxyToolsSyncError): + return cls._extract_items(value) + for value in payload.values(): + if isinstance(value, list): + return value + raise ProxyToolsSyncError("Unexpected Proxy-Tools response shape") + + @classmethod + def _has_more_pages( + cls, + payload: Any, + *, + page: int, + batch_size: int, + limit: int, + ) -> bool: + """Определить, нужно ли запросить следующую страницу.""" + if batch_size == 0: + return False + if isinstance(payload, dict): + next_value = payload.get("next") or payload.get("next_page") + if next_value not in (None, "", False): + return True + meta = payload.get("meta") + if isinstance(meta, dict): + pagination = meta.get("pagination") + if isinstance(pagination, dict): + current_page = cls._to_int( + pagination.get("current_page") or pagination.get("page") + ) + total_pages = cls._to_int(pagination.get("total_pages")) + if current_page is not None and total_pages is not None: + return current_page < total_pages + total_pages = cls._to_int(meta.get("total_pages")) + if total_pages is not None: + return page < total_pages + return batch_size >= limit + + @classmethod + def _extract_addresses(cls, items: list[Any]) -> list[str]: + """Нормализовать и дедуплицировать адреса прокси.""" + addresses: list[str] = [] + seen: set[str] = set() + for item in items: + address = cls._extract_address(item) + if address and address not in seen: + seen.add(address) + addresses.append(address) + return addresses + + @classmethod + def _extract_address(cls, item: Any) -> str | None: + """Извлечь адрес прокси из одного элемента payload.""" + if isinstance(item, str): + return cls._normalize_address(item) + if not isinstance(item, dict): + return None + + for key in ("proxy", "proxy_url", "url", "address", "addr"): + value = item.get(key) + if isinstance(value, str): + normalized = cls._normalize_address( + value, + scheme_hint=item.get("scheme") + or item.get("protocol") + or item.get("type") + or item.get("proxy_type"), + ) + if normalized: + return normalized + + host = item.get("host") or item.get("ip") + port = item.get("port") + if host and port: + return cls._normalize_address( + f"{host}:{port}", + scheme_hint=item.get("scheme") + or item.get("protocol") + or item.get("type") + or item.get("proxy_type"), + ) + + return None + + @classmethod + def _normalize_address( + cls, + value: str, + *, + scheme_hint: Any = None, + ) -> str | None: + """Привести адрес прокси к нормализованному URL.""" + candidate = str(value).strip() + if not candidate: + return None + + scheme = cls._normalize_scheme(scheme_hint) + if "://" not in candidate: + candidate = f"{scheme or 'http'}://{candidate}" + + parsed = urlparse(candidate) + if not parsed.hostname or parsed.port is None: + return None + + final_scheme = cls._normalize_scheme(parsed.scheme) or scheme or "http" + credentials = "" + if parsed.username: + credentials = parsed.username + if parsed.password: + credentials = f"{credentials}:{parsed.password}" + credentials = f"{credentials}@" + + host = parsed.hostname + if ":" in host and not host.startswith("["): + host = f"[{host}]" + return f"{final_scheme}://{credentials}{host}:{parsed.port}" + + @classmethod + def _normalize_scheme(cls, value: Any) -> str | None: + """Нормализовать схему прокси.""" + if value is None: + return None + mapping = { + "1": "socks4", + "2": "socks5", + "3": "https", + "4": "http", + 1: "socks4", + 2: "socks5", + 3: "https", + 4: "http", + } + candidate = mapping.get(value, str(value).strip().lower()) + if candidate in {"http", "https", "socks4", "socks5"}: + return candidate + return None + + @classmethod + def _to_int(cls, value: Any) -> int | None: + """Безопасно привести значение к int.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + @classmethod + @transaction.atomic + def _sync_addresses(cls, addresses: list[str]) -> dict[str, int]: + """Синхронизировать импортированные адреса с таблицей Proxy.""" + existing_qs = Proxy.objects.filter( + source=cls.SOURCE, + country_code=cls.COUNTRY_CODE, + ) + existing_by_address = { + proxy.address: proxy + for proxy in existing_qs.only("id", "address", "is_active") + } + + created = 0 + updated = 0 + for address in addresses: + proxy = existing_by_address.get(address) + if proxy is None: + Proxy.objects.create( + address=address, + is_active=True, + description="Imported from Proxy-Tools", + source=cls.SOURCE, + country_code=cls.COUNTRY_CODE, + ) + created += 1 + continue + + changed_fields: list[str] = [] + if not proxy.is_active: + proxy.is_active = True + changed_fields.append("is_active") + if proxy.description != "Imported from Proxy-Tools": + proxy.description = "Imported from Proxy-Tools" + changed_fields.append("description") + if changed_fields: + proxy.save(update_fields=[*changed_fields, "updated_at"]) + updated += 1 + + deactivated = 0 + active_imported = existing_qs.filter(is_active=True) + if addresses: + deactivated = active_imported.exclude(address__in=addresses).update( + is_active=False + ) + else: + deactivated = active_imported.update(is_active=False) + + return { + "created": created, + "updated": updated, + "deactivated": deactivated, + } + + class InspectionService(BulkOperationsMixin, BaseService[InspectionRecord]): """ Сервис для управления данными о проверках. diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 436d49b..d0ccf03 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -13,6 +13,7 @@ from datetime import datetime from pathlib import Path from apps.core.services import BackgroundJobService +from apps.core.tasks import PeriodicTask as CorePeriodicTask from apps.parsers.clients.minpromtorg import ( IndustrialProductionClient, IndustrialProductsClient, @@ -30,9 +31,9 @@ from apps.parsers.services import ( ParserLoadLogService, ProcurementService, ProxyService, + ProxyToolsSyncService, ) from celery import shared_task -from django.conf import settings from requests.adapters import BaseAdapter logger = logging.getLogger(__name__) @@ -48,18 +49,16 @@ def _resolve_proxies(proxies: list[str] | None) -> list[str] | None: Приоритет: 1. Явно переданные в задачу `proxies` - 2. Активные прокси из БД - 3. `settings.PARSER_PROXIES` (например, из ENV) + 2. Runtime-прокси из БД (с приоритетом Proxy-Tools RU) """ if proxies is not None: return proxies - db_proxies = ProxyService.get_active_proxies_or_none() + db_proxies = ProxyService.get_runtime_proxies_or_none() if db_proxies: return db_proxies - configured_proxies = getattr(settings, "PARSER_PROXIES", []) or [] - return configured_proxies or None + return None def _get_or_create_background_job( @@ -89,6 +88,14 @@ def _get_or_create_background_job( return job +@shared_task(bind=True, base=CorePeriodicTask) +def sync_ru_proxies(self) -> dict[str, int | str]: # noqa: ARG001 + """Периодически загружать RU-прокси из Proxy-Tools.""" + result = ProxyToolsSyncService.sync_ru_proxies() + logger.info("RU proxy sync finished: %s", result) + return result + + def _lock_path_for(file_path: Path) -> Path: return Path(f"{file_path}.lock") diff --git a/src/apps/parsers/urls.py b/src/apps/parsers/urls.py index a20946c..e0e84b5 100644 --- a/src/apps/parsers/urls.py +++ b/src/apps/parsers/urls.py @@ -15,7 +15,6 @@ from apps.parsers.views import ( ParserLoadLogViewSet, ParsingSettingsView, ProcurementViewSet, - ProxyViewSet, SourceCardDetailView, SourceCardListView, SourceCardRefreshView, @@ -108,7 +107,6 @@ sources_urlpatterns = [ system_router = DefaultRouter() system_router.register(r"logs", ParserLoadLogViewSet, basename="parser-logs") -system_router.register(r"proxies", ProxyViewSet, basename="proxies") system_urlpatterns = [ path("logs/export/", ParserLoadLogExportView.as_view(), name="parser-logs-export"), diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index e62687f..56f2183 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -19,7 +19,6 @@ from apps.parsers.models import ( ParserLoadLog, ParsingSettings, ProcurementRecord, - Proxy, ) from apps.parsers.serializers import ( FinancialReportDetailSerializer, @@ -33,7 +32,6 @@ from apps.parsers.serializers import ( ParserLoadLogSerializer, ParsingSettingsSerializer, ProcurementSerializer, - ProxySerializer, SourceCardDetailSerializer, SourceCardRefreshRequestSerializer, SourceCardRefreshResponseSerializer, @@ -1091,45 +1089,3 @@ class ParserLoadLogExportView(APIView): ) return response - - -class ProxyViewSet(ReadOnlyModelViewSet): - """ - API для просмотра списка прокси-серверов. - - Используется для отладки и мониторинга парсеров. - Только для администраторов. - """ - - queryset = Proxy.objects.all().order_by("-last_used_at") - serializer_class = ProxySerializer - permission_classes = [IsAdminUser] - filterset_fields = ["is_active"] - - @swagger_auto_schema( - tags=[SYSTEM_TAG], - operation_summary="Список прокси", - operation_description=( - "Возвращает список прокси-серверов для парсеров.\n" - "Доступно только администраторам.\n" - "Поддерживает фильтрацию по: is_active." - ), - responses={ - 200: ProxySerializer(many=True), - **ErrorResponses.ADMIN, - }, - ) - def list(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) - - @swagger_auto_schema( - tags=[SYSTEM_TAG], - operation_summary="Детали прокси", - operation_description="Возвращает информацию о конкретном прокси.", - responses={ - 200: ProxySerializer, - **ErrorResponses.ADMIN_NOT_FOUND, - }, - ) - def retrieve(self, request, *args, **kwargs): - return super().retrieve(request, *args, **kwargs) diff --git a/src/apps/registers/migrations/0004_seed_default_registers.py b/src/apps/registers/migrations/0004_seed_default_registers.py new file mode 100644 index 0000000..8adf553 --- /dev/null +++ b/src/apps/registers/migrations/0004_seed_default_registers.py @@ -0,0 +1,28 @@ +from django.db import migrations + +DEFAULT_REGISTER_NAMES = ( + "Реестр предприятий ОПК", + "Реестр госкорпорации Роскосмос", + "Реестр госкорпорации Росатом", +) + + +def seed_default_registers(apps, schema_editor): + Register = apps.get_model("registers", "Register") + db_alias = schema_editor.connection.alias + + for name in DEFAULT_REGISTER_NAMES: + Register.objects.using(db_alias).get_or_create(name=name) + + +class Migration(migrations.Migration): + dependencies = [ + ("registers", "0003_add_unique_active_membership_period"), + ] + + operations = [ + migrations.RunPython( + seed_default_registers, + migrations.RunPython.noop, + ), + ] diff --git a/src/apps/user/views.py b/src/apps/user/views.py index c1470dc..7c31d2a 100644 --- a/src/apps/user/views.py +++ b/src/apps/user/views.py @@ -1,7 +1,6 @@ from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from django.contrib.auth import authenticate from django.contrib.auth.hashers import check_password -from django.core.paginator import Paginator from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -33,6 +32,79 @@ AUTH_TAG = swagger_tag("Аутентификация", "authentication") USER_TAG = swagger_tag("Пользователь", "user") USER_ADMIN_TAG = swagger_tag("Управление пользователями", "user_management") +ADMIN_USER_VALIDATION_ERROR_RESPONSE = openapi.Response( + description="Ошибка валидации пользователя", + schema=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "success": openapi.Schema(type=openapi.TYPE_BOOLEAN, default=False), + "data": openapi.Schema(type=openapi.TYPE_OBJECT, nullable=True), + "errors": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "code": openapi.Schema( + type=openapi.TYPE_STRING, + default="validation_error", + ), + "message": openapi.Schema( + type=openapi.TYPE_STRING, + default="Validation failed", + ), + "details": openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "fields": openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "email": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema( + type=openapi.TYPE_STRING + ), + ), + "username": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema( + type=openapi.TYPE_STRING + ), + ), + "password": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema( + type=openapi.TYPE_STRING + ), + ), + "first_name": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema( + type=openapi.TYPE_STRING + ), + ), + "last_name": openapi.Schema( + type=openapi.TYPE_ARRAY, + items=openapi.Schema( + type=openapi.TYPE_STRING + ), + ), + }, + ) + }, + ), + }, + ), + ), + "meta": openapi.Schema( + type=openapi.TYPE_OBJECT, + properties={ + "request_id": openapi.Schema(type=openapi.TYPE_STRING), + }, + ), + }, + ), +) + class RegisterView(APIView): """ @@ -162,24 +234,6 @@ class AdminUserListCreateView(APIView): permission_classes = [IsAdminUser] - @staticmethod - def _get_positive_int(value, *, default: int, minimum: int = 1) -> int: - if value in (None, ""): - return default - try: - parsed = int(value) - except (TypeError, ValueError): - raise ValueError("must be integer") from None - if parsed < minimum: - raise ValueError("must be positive") - return parsed - - @staticmethod - def _build_page_url(request, page_number: int) -> str: - query_params = request.query_params.copy() - query_params["page"] = page_number - return request.build_absolute_uri(f"{request.path}?{query_params.urlencode()}") - @swagger_auto_schema( tags=[USER_ADMIN_TAG], operation_summary="Список пользователей", @@ -209,7 +263,7 @@ class AdminUserListCreateView(APIView): ), ], responses={ - 200: FrontendUserWithProfileSerializer(many=True), + 200: FrontendManagedUserSerializer(many=True), **ErrorResponses.ADMIN, }, ) @@ -218,47 +272,8 @@ class AdminUserListCreateView(APIView): search=request.query_params.get("search", ""), ordering=request.query_params.get("ordering", ""), ) - try: - page_number = self._get_positive_int( - request.query_params.get("page"), - default=1, - ) - page_size = self._get_positive_int( - request.query_params.get("page_size"), - default=20, - ) - except ValueError: - return Response( - { - "detail": ( - "Параметры page и page_size должны быть положительными " - "целыми числами." - ) - }, - status=status.HTTP_400_BAD_REQUEST, - ) - - page_size = min(page_size, 100) - paginator = Paginator(queryset, page_size) - page_obj = paginator.get_page(page_number) - serializer = FrontendUserWithProfileSerializer(page_obj.object_list, many=True) - - return Response( - { - "count": paginator.count, - "next": ( - self._build_page_url(request, page_obj.next_page_number()) - if page_obj.has_next() - else None - ), - "previous": ( - self._build_page_url(request, page_obj.previous_page_number()) - if page_obj.has_previous() - else None - ), - "results": serializer.data, - } - ) + serializer = FrontendManagedUserSerializer(queryset, many=True) + return Response(serializer.data) @swagger_auto_schema( tags=[USER_ADMIN_TAG], @@ -269,7 +284,7 @@ class AdminUserListCreateView(APIView): request_body=AdminUserCreateSerializer, responses={ 201: FrontendManagedUserSerializer, - 400: CommonResponses.BAD_REQUEST, + 400: ADMIN_USER_VALIDATION_ERROR_RESPONSE, **ErrorResponses.ADMIN, }, ) @@ -314,7 +329,7 @@ class AdminUserDetailView(APIView): request_body=AdminUserUpdateSerializer, responses={ 200: FrontendManagedUserSerializer, - 400: CommonResponses.BAD_REQUEST, + 400: ADMIN_USER_VALIDATION_ERROR_RESPONSE, **ErrorResponses.ADMIN_NOT_FOUND, }, ) diff --git a/src/core/celery.py b/src/core/celery.py index 1cc45eb..7f477b6 100644 --- a/src/core/celery.py +++ b/src/core/celery.py @@ -74,6 +74,10 @@ app.conf.beat_schedule = { "task": "apps.parsers.tasks.parse_inspections", "schedule": 7 * 24 * 60 * 60, # Every 7 days }, + "sync-ru-proxies-hourly": { + "task": "apps.parsers.tasks.sync_ru_proxies", + "schedule": getattr(settings, "PROXY_TOOLS_SYNC_INTERVAL_SECONDS", 3600), + }, # Сканирование папки FNS - каждые 5 минут "scan-fns-directory": { "task": "apps.parsers.tasks.scan_fns_directory", diff --git a/src/settings/base.py b/src/settings/base.py index 28e0348..9423df1 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -200,6 +200,16 @@ FNS_LOCK_TTL_SECONDS = 3600 PARSER_PROXIES = [ item.strip() for item in os.getenv("PARSER_PROXIES", "").split(",") if item.strip() ] +PROXY_TOOLS_API_KEY = os.getenv("PROXY_TOOLS_API_KEY", "").strip() +PROXY_TOOLS_API_URL = os.getenv( + "PROXY_TOOLS_API_URL", "https://proxy-tools.com/api/v1/proxies" +).strip() +PROXY_TOOLS_TIMEOUT_SECONDS = int(os.getenv("PROXY_TOOLS_TIMEOUT_SECONDS", "30")) +PROXY_TOOLS_LIMIT = int(os.getenv("PROXY_TOOLS_LIMIT", "100")) +PROXY_TOOLS_MAX_PAGES = int(os.getenv("PROXY_TOOLS_MAX_PAGES", "3")) +PROXY_TOOLS_SYNC_INTERVAL_SECONDS = int( + os.getenv("PROXY_TOOLS_SYNC_INTERVAL_SECONDS", "3600") +) BACKUP_ENCRYPTION_KEY = os.getenv("BACKUP_ENCRYPTION_KEY", "") BACKUP_KEY_ID = os.getenv("BACKUP_KEY_ID", "default") BACKUP_EXPORT_DIRECTORY = os.getenv( diff --git a/tests/apps/core/test_celery_module.py b/tests/apps/core/test_celery_module.py index 8671906..23eebf3 100644 --- a/tests/apps/core/test_celery_module.py +++ b/tests/apps/core/test_celery_module.py @@ -52,6 +52,7 @@ class CeleryModuleTest(SimpleTestCase): self.assertIn("parse-manufactures-daily", module.app.conf.beat_schedule) self.assertIn("parse-industrial-products-daily", module.app.conf.beat_schedule) self.assertIn("parse-inspections-weekly", module.app.conf.beat_schedule) + self.assertIn("sync-ru-proxies-hourly", module.app.conf.beat_schedule) def test_startup_refresh_queues_when_lock_acquired(self): with patch.dict( diff --git a/tests/apps/parsers/factories.py b/tests/apps/parsers/factories.py index 3238667..1fb5788 100644 --- a/tests/apps/parsers/factories.py +++ b/tests/apps/parsers/factories.py @@ -83,6 +83,8 @@ class ProxyFactory(factory.django.DjangoModelFactory): address = factory.LazyFunction(generate_proxy_address) description = factory.LazyAttribute(lambda _: fake.sentence(nb_words=3)) + source = "manual" + country_code = "RU" is_active = True fail_count = 0 last_used_at = factory.LazyAttribute( diff --git a/tests/apps/parsers/test_services.py b/tests/apps/parsers/test_services.py index f91235e..69e773c 100644 --- a/tests/apps/parsers/test_services.py +++ b/tests/apps/parsers/test_services.py @@ -1,5 +1,6 @@ """Tests for parsers services.""" +from unittest.mock import patch from urllib.parse import urlparse from apps.parsers.clients.minpromtorg.industrial import IndustrialProductionClient @@ -27,9 +28,10 @@ from apps.parsers.services import ( ParserLoadLogService, ProcurementService, ProxyService, + ProxyToolsSyncService, ) from apps.registers.models import Organization -from django.test import TestCase, tag +from django.test import TestCase, override_settings, tag from tests.utils import TestHTTPServer from tests.utils.fixtures import build_minpromtorg_certificates_excel, fake @@ -173,6 +175,127 @@ class ProxyServiceTest(TestCase): self.assertEqual(created, 1) self.assertEqual(Proxy.objects.count(), 2) + def test_get_runtime_proxies_prefers_proxy_tools_ru(self): + """Runtime should prefer RU proxies imported from Proxy-Tools.""" + manual_ru = ProxyFactory( + source=ProxyService.MANUAL_SOURCE, + country_code="RU", + ) + imported_ru = ProxyFactory( + source=ProxyService.PROXY_TOOLS_SOURCE, + country_code="RU", + ) + ProxyFactory( + source=ProxyService.PROXY_TOOLS_SOURCE, + country_code="US", + ) + + result = ProxyService.get_runtime_proxies() + + self.assertEqual(result, [imported_ru.address]) + self.assertNotIn(manual_ru.address, result) + + def test_get_runtime_proxies_falls_back_to_any_ru_proxy(self): + """Runtime should fall back to any RU proxy when imported list is empty.""" + manual_ru = ProxyFactory( + source=ProxyService.MANUAL_SOURCE, + country_code="RU", + ) + ProxyFactory( + source=ProxyService.MANUAL_SOURCE, + country_code="US", + ) + + result = ProxyService.get_runtime_proxies() + + self.assertEqual(result, [manual_ru.address]) + + +class ProxyToolsSyncServiceTest(TestCase): + """Tests for ProxyToolsSyncService.""" + + def test_sync_ru_proxies_skips_without_api_key(self): + """Sync should be skipped when API key is missing.""" + result = ProxyToolsSyncService.sync_ru_proxies() + + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "missing_api_key") + + @override_settings( + PROXY_TOOLS_API_KEY="test-token", + PROXY_TOOLS_LIMIT=2, + PROXY_TOOLS_MAX_PAGES=2, + ) + @patch("apps.parsers.services.ProxyToolsClient.fetch_proxies") + def test_sync_ru_proxies_upserts_and_deactivates(self, fetch_proxies_mock): + """Sync should create, reactivate and deactivate imported proxies.""" + active_stale = ProxyFactory( + address="http://10.0.0.10:8000", + source=ProxyService.PROXY_TOOLS_SOURCE, + country_code="RU", + is_active=True, + ) + inactive_existing = ProxyFactory( + address="http://10.0.0.20:8000", + source=ProxyService.PROXY_TOOLS_SOURCE, + country_code="RU", + is_active=False, + ) + manual_ru = ProxyFactory( + address="http://10.0.0.30:8000", + source=ProxyService.MANUAL_SOURCE, + country_code="RU", + is_active=True, + ) + + fetch_proxies_mock.side_effect = [ + { + "data": [ + {"host": "10.0.0.20", "port": 8000, "type": "4"}, + {"proxy": "socks5://10.0.0.40:1080"}, + ], + "meta": {"total_pages": 2}, + }, + { + "data": [ + "https://10.0.0.50:8443", + ], + "meta": {"total_pages": 2}, + }, + ] + + result = ProxyToolsSyncService.sync_ru_proxies() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["fetched"], 3) + self.assertEqual(result["created"], 2) + self.assertEqual(result["updated"], 1) + self.assertEqual(result["deactivated"], 1) + + active_stale.refresh_from_db() + inactive_existing.refresh_from_db() + manual_ru.refresh_from_db() + + self.assertFalse(active_stale.is_active) + self.assertTrue(inactive_existing.is_active) + self.assertTrue(manual_ru.is_active) + + imported_addresses = set( + Proxy.objects.filter( + source=ProxyService.PROXY_TOOLS_SOURCE, + country_code="RU", + is_active=True, + ).values_list("address", flat=True) + ) + self.assertSetEqual( + imported_addresses, + { + "http://10.0.0.20:8000", + "socks5://10.0.0.40:1080", + "https://10.0.0.50:8443", + }, + ) + class ParserLoadLogServiceTest(TestCase): """Tests for ParserLoadLogService.""" diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index bf60634..f8e3f5e 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -9,6 +9,7 @@ import tempfile import threading from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch from urllib.parse import urlparse from apps.parsers import tasks as parser_tasks @@ -39,6 +40,7 @@ from apps.parsers.tasks import ( _move_to_dir, _process_fns_file_sync, _remove_lock, + _resolve_proxies, _try_create_lock, parse_all_minpromtorg, parse_all_sources, @@ -51,6 +53,7 @@ from apps.parsers.tasks import ( scan_fns_directory, sync_inspections, sync_procurements, + sync_ru_proxies, ) from django.test import TestCase, override_settings from openpyxl import Workbook @@ -59,6 +62,7 @@ from tests.apps.parsers.factories import ( InspectionRecordFactory, ParserLoadLogFactory, ProcurementRecordFactory, + ProxyFactory, ) from tests.utils import TestHTTPServer from tests.utils.fixtures import ( @@ -102,6 +106,55 @@ def _portal_path(year: int, month: int) -> str: return f"/portal/public-open-data/check/{year}/{month}" +class ProxyResolutionTestCase(TestCase): + """Tests for proxy resolution in parser tasks.""" + + @override_settings(PARSER_PROXIES=["http://env-proxy:8080"]) + def test_resolve_proxies_prefers_runtime_db_proxies(self): + imported_proxy = ProxyFactory( + address="http://10.0.0.2:8000", + source="proxy-tools", + country_code="RU", + is_active=True, + ) + ProxyFactory( + address="http://10.0.0.3:8000", + source="manual", + country_code="RU", + is_active=True, + ) + + result = _resolve_proxies(None) + + self.assertEqual(result, [imported_proxy.address]) + + @override_settings(PARSER_PROXIES=["http://env-proxy:8080"]) + def test_resolve_proxies_does_not_use_unclassified_env_fallback(self): + result = _resolve_proxies(None) + + self.assertIsNone(result) + + +class SyncRuProxiesTaskTestCase(TestCase): + """Tests for periodic RU proxy sync task.""" + + @patch("apps.parsers.tasks.ProxyToolsSyncService.sync_ru_proxies") + def test_sync_ru_proxies_returns_service_payload(self, sync_mock): + sync_mock.return_value = { + "status": "success", + "fetched": 3, + "created": 2, + "updated": 1, + "deactivated": 0, + } + + result = sync_ru_proxies.run() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["fetched"], 3) + sync_mock.assert_called_once_with() + + @override_settings( CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPAGATES=True, diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 7843da4..6e245f8 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -19,7 +19,6 @@ from tests.apps.parsers.factories import ( InspectionRecordFactory, ManufacturerRecordFactory, ParserLoadLogFactory, - ProxyFactory, ) from tests.apps.user.factories import UserFactory from tests.utils.fixtures import fake @@ -151,11 +150,9 @@ class ParsersViewSetTest(APITestCase): self.assertEqual(detail.status_code, status.HTTP_200_OK) self.assertIn("lines", detail.data) - def test_system_logs_and_proxies_admin_only(self): + def test_system_logs_admin_only(self): log = ParserLoadLogFactory() - proxy = ProxyFactory() url_logs = reverse("api_v1:system:parser-logs-list") - url_proxy = reverse("api_v1:system:proxies-list") response = self.client.get(url_logs) self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) @@ -172,13 +169,6 @@ class ParsersViewSetTest(APITestCase): ) self.assertEqual(detail.status_code, status.HTTP_200_OK) - proxy_response = self.client.get(url_proxy) - self.assertEqual(proxy_response.status_code, status.HTTP_200_OK) - proxy_detail = self.client.get( - reverse("api_v1:system:proxies-detail", args=[proxy.id]) - ) - self.assertEqual(proxy_detail.status_code, status.HTTP_200_OK) - def test_system_logs_support_search_and_organizations_count(self): first_log = ParserLoadLogFactory( source="manufactures", diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index 6dabd2b..d15cff7 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -254,7 +254,7 @@ class AdminUserManagementViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual( - set(response.data["results"][0].keys()), + set(response.data[0].keys()), { "id", "username", @@ -263,14 +263,9 @@ class AdminUserManagementViewTest(APITestCase): "is_active", "role", "role_label", - "profile", }, ) - self.assertEqual( - set(response.data["results"][0]["profile"].keys()), - {"first_name", "middle_name", "last_name", "full_name"}, - ) - usernames = {item["username"] for item in response.data["results"]} + usernames = {item["username"] for item in response.data} self.assertIn(self.admin.username, usernames) self.assertIn(self.user.username, usernames) @@ -287,7 +282,7 @@ class AdminUserManagementViewTest(APITestCase): response = self.client.get(self.list_url, {"search": "Петрович"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - usernames = [item["username"] for item in response.data["results"]] + usernames = [item["username"] for item in response.data] self.assertEqual(usernames, [self.user.username]) def test_admin_can_order_users(self): @@ -299,7 +294,7 @@ class AdminUserManagementViewTest(APITestCase): response = self.client.get(self.list_url, {"ordering": "first_name"}) self.assertEqual(response.status_code, status.HTTP_200_OK) - ordered_ids = [item["id"] for item in response.data["results"]] + ordered_ids = [item["id"] for item in response.data] self.assertLess(ordered_ids.index(second.id), ordered_ids.index(first.id)) def test_admin_can_create_user_with_role(self): @@ -336,6 +331,51 @@ class AdminUserManagementViewTest(APITestCase): self.assertEqual(created.profile.first_name, "Петр") self.assertEqual(created.profile.middle_name, "Петрович") + def test_admin_create_user_returns_duplicate_email_error(self): + payload = { + "email": self.user.email, + "username": fake.unique.user_name(), + "password": fake.password(length=12, special_chars=False), + "role": "user", + "first_name": "Петр", + "last_name": "Петров", + } + + response = self.client.post(self.list_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("email", response.data["errors"][0]["details"]["fields"]) + + def test_admin_create_user_returns_duplicate_username_error(self): + payload = { + "email": fake.unique.email(), + "username": self.user.username, + "password": fake.password(length=12, special_chars=False), + "role": "user", + "first_name": "Петр", + "last_name": "Петров", + } + + response = self.client.post(self.list_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("username", response.data["errors"][0]["details"]["fields"]) + + def test_admin_create_user_returns_password_validation_error(self): + payload = { + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "password": "123", + "role": "user", + "first_name": "Петр", + "last_name": "Петров", + } + + response = self.client.post(self.list_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("password", response.data["errors"][0]["details"]["fields"]) + def test_admin_can_update_user_and_role(self): url = reverse("api_v1:user:admin-user-detail", args=[self.user.id]) @@ -382,6 +422,44 @@ class AdminUserManagementViewTest(APITestCase): {"first_name", "middle_name", "last_name", "full_name"}, ) + def test_admin_patch_user_returns_duplicate_email_error(self): + another = UserFactory.create_user() + url = reverse("api_v1:user:admin-user-detail", args=[another.id]) + + response = self.client.patch( + url, + {"email": self.user.email}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("email", response.data["errors"][0]["details"]["fields"]) + + def test_admin_patch_user_returns_duplicate_username_error(self): + another = UserFactory.create_user() + url = reverse("api_v1:user:admin-user-detail", args=[another.id]) + + response = self.client.patch( + url, + {"username": self.user.username}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("username", response.data["errors"][0]["details"]["fields"]) + + def test_admin_patch_user_returns_password_validation_error(self): + url = reverse("api_v1:user:admin-user-detail", args=[self.user.id]) + + response = self.client.patch( + url, + {"password": "123"}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("password", response.data["errors"][0]["details"]["fields"]) + def test_admin_cannot_patch_self_to_inactive(self): url = reverse("api_v1:user:admin-user-detail", args=[self.admin.id]) diff --git a/tests/test_api_inventory_e2e.py b/tests/test_api_inventory_e2e.py new file mode 100644 index 0000000..0a525d5 --- /dev/null +++ b/tests/test_api_inventory_e2e.py @@ -0,0 +1,719 @@ +"""End-to-end smoke coverage for the full HTTP API inventory.""" + +from __future__ import annotations + +import io +from datetime import date, timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import patch + +from apps.backups.models import BackupExportJob +from apps.core.models import BackgroundJob +from apps.exchange.models import ExchangeConnection +from apps.parsers.models import ( + FinancialReport, + FinancialReportLine, + ParserLoadLog, + ProcurementRecord, +) +from apps.user.services import UserService +from django.core.files.uploadedfile import SimpleUploadedFile +from django.urls import reverse +from django.utils import timezone +from django_celery_beat.models import PeriodicTask +from openpyxl import Workbook +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.parsers.factories import ( + IndustrialCertificateRecordFactory, + IndustrialProductRecordFactory, + InspectionRecordFactory, + ManufacturerRecordFactory, + ParserLoadLogFactory, +) +from tests.apps.registers.factories import RegisterFactory +from tests.apps.user.factories import UserFactory +from tests.utils.fixtures import fake + + +def _digits(length: int) -> str: + return "".join(str(fake.random_int(0, 9)) for _ in range(length)) + + +def _build_fns_excel_bytes() -> bytes: + workbook = Workbook() + worksheet = workbook.active + year = fake.random_int(min=2020, max=2025) + worksheet.append(["Form", None, year, None]) + worksheet.append([None, "Code", "Start", "End"]) + worksheet.append( + [fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)] + ) + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + return buffer.getvalue() + + +def _build_register_excel_bytes(rows: list[dict[str, str]]) -> bytes: + workbook = Workbook() + worksheet = workbook.active + worksheet.append(["pn_name", "mn_ogrn", "mn_inn", "in_kpp", "mn_okpo"]) + for row in rows: + worksheet.append( + [ + row["pn_name"], + row["mn_ogrn"], + row["mn_inn"], + row["in_kpp"], + row["mn_okpo"], + ] + ) + buffer = io.BytesIO() + workbook.save(buffer) + workbook.close() + return buffer.getvalue() + + +def _extract_results(response_data): + if hasattr(response_data, "get"): + data = response_data.get("data") + if isinstance(data, list): + return data + results = response_data.get("results") + if results is not None: + return results + return response_data + + +def _create_procurement_record() -> ProcurementRecord: + return ProcurementRecord.objects.create( + load_batch=fake.random_int(min=1, max=1000), + purchase_number=_digits(19), + purchase_name=fake.sentence(nb_words=6), + customer_inn=_digits(10), + customer_kpp=_digits(9), + customer_ogrn=_digits(13), + customer_name=fake.company(), + max_price=str(fake.pydecimal(left_digits=7, right_digits=2, positive=True)), + status=fake.word(), + law_type="44-FZ", + href=fake.url(), + region_code=f"{fake.random_int(min=1, max=99):02d}", + ) + + +class AuthenticatedApiMixin: + def authenticate(self, user): + tokens = UserService.get_tokens_for_user(user) + self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {tokens['access']}") + return tokens + + +class PublicApiInventoryE2ETest(APITestCase): + def test_health_and_swagger_endpoints(self): + swagger_response = self.client.get(reverse("schema-swagger-ui")) + health_response = self.client.get(reverse("core:health")) + live_response = self.client.get(reverse("core:liveness")) + ready_response = self.client.get(reverse("core:readiness")) + + self.assertEqual(swagger_response.status_code, status.HTTP_200_OK) + self.assertEqual(health_response.status_code, status.HTTP_200_OK) + self.assertEqual(live_response.status_code, status.HTTP_200_OK) + self.assertEqual(ready_response.status_code, status.HTTP_200_OK) + + +class UserApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): + def setUp(self): + self.admin = UserFactory.create_superuser() + + def test_auth_and_profile_endpoints(self): + initial_password = fake.password(length=16, special_chars=False) + new_password = fake.password(length=18, special_chars=False) + register_payload = { + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "password": initial_password, + "password_confirm": initial_password, + "phone": f"+7{fake.numerify('##########')}", + "first_name": "Ivan", + "middle_name": "Ivanovich", + "last_name": "Ivanov", + } + + register_response = self.client.post( + reverse("api_v1:user:register"), + register_payload, + format="json", + ) + self.assertEqual(register_response.status_code, status.HTTP_201_CREATED) + self.assertIn("tokens", register_response.data) + + login_response = self.client.post( + reverse("api_v1:user:login"), + { + "username": register_payload["username"], + "password": initial_password, + }, + format="json", + ) + self.assertEqual(login_response.status_code, status.HTTP_200_OK) + + verify_response = self.client.post( + reverse("api_v1:user:token_verify"), + {"token": login_response.data["access"]}, + format="json", + ) + self.assertEqual(verify_response.status_code, status.HTTP_200_OK) + + refresh_response = self.client.post( + reverse("api_v1:user:token_refresh"), + {"refresh": login_response.data["refresh"]}, + format="json", + ) + self.assertEqual(refresh_response.status_code, status.HTTP_200_OK) + self.client.credentials( + HTTP_AUTHORIZATION=f"Bearer {refresh_response.data['access']}" + ) + + me_response = self.client.get(reverse("api_v1:user:current_user")) + self.assertEqual(me_response.status_code, status.HTTP_200_OK) + self.assertEqual(me_response.data["username"], register_payload["username"]) + + me_update_response = self.client.patch( + reverse("api_v1:user:user_update"), + {"phone": f"+7{fake.numerify('##########')}"}, + format="json", + ) + self.assertEqual(me_update_response.status_code, status.HTTP_200_OK) + + profile_response = self.client.get(reverse("api_v1:user:profile_detail")) + self.assertEqual(profile_response.status_code, status.HTTP_200_OK) + + profile_patch_response = self.client.patch( + reverse("api_v1:user:profile_detail"), + { + "first_name": "Petr", + "middle_name": "Petrovich", + "last_name": "Petrov", + }, + format="json", + ) + self.assertEqual(profile_patch_response.status_code, status.HTTP_200_OK) + + profile_full_response = self.client.get(reverse("api_v1:user:profile_full")) + self.assertEqual(profile_full_response.status_code, status.HTTP_200_OK) + self.assertEqual( + profile_full_response.data["username"], register_payload["username"] + ) + + password_change_response = self.client.post( + reverse("api_v1:user:password_change"), + { + "old_password": initial_password, + "new_password": new_password, + "new_password_confirm": new_password, + }, + format="json", + ) + self.assertEqual(password_change_response.status_code, status.HTTP_200_OK) + + relogin_response = self.client.post( + reverse("api_v1:user:login"), + { + "username": register_payload["username"], + "password": new_password, + }, + format="json", + ) + self.assertEqual(relogin_response.status_code, status.HTTP_200_OK) + + self.client.credentials( + HTTP_AUTHORIZATION=f"Bearer {relogin_response.data['access']}" + ) + logout_response = self.client.post( + reverse("api_v1:user:logout"), + {}, + format="json", + ) + self.assertEqual(logout_response.status_code, status.HTTP_200_OK) + + def test_admin_user_management_endpoints(self): + self.authenticate(self.admin) + list_response = self.client.get(reverse("api_v1:user:admin-users")) + self.assertEqual(list_response.status_code, status.HTTP_200_OK) + + create_payload = { + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "phone": f"+7{fake.numerify('##########')}", + "password": "AdminManagedPass123", + "role": "user", + "first_name": "Alex", + "middle_name": "Alexeevich", + "last_name": "Alexeev", + } + create_response = self.client.post( + reverse("api_v1:user:admin-users"), + create_payload, + format="json", + ) + self.assertEqual(create_response.status_code, status.HTTP_201_CREATED) + managed_user_id = create_response.data["id"] + + detail_url = reverse("api_v1:user:admin-user-detail", args=[managed_user_id]) + detail_response = self.client.get(detail_url) + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + + patch_response = self.client.patch( + detail_url, + { + "role": "admin", + "first_name": "Sergey", + "middle_name": "Sergeevich", + "last_name": "Sergeev", + }, + format="json", + ) + self.assertEqual(patch_response.status_code, status.HTTP_200_OK) + self.assertEqual(patch_response.data["role"], "admin") + + deactivate_response = self.client.post( + reverse("api_v1:user:admin-user-deactivate", args=[managed_user_id]), + {}, + format="json", + ) + self.assertEqual(deactivate_response.status_code, status.HTTP_200_OK) + + activate_response = self.client.post( + reverse("api_v1:user:admin-user-activate", args=[managed_user_id]), + {}, + format="json", + ) + self.assertEqual(activate_response.status_code, status.HTTP_200_OK) + self.assertTrue(activate_response.data["is_active"]) + + +class JobsApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + + def _create_job(self, *, task_id: str, status_value: str) -> BackgroundJob: + started_at = timezone.now() + completed_at = started_at + timedelta(seconds=3) + return BackgroundJob.objects.create( + task_id=task_id, + task_name="apps.parsers.tasks.fake_task", + status=status_value, + user_id=self.user.id, + started_at=started_at, + completed_at=completed_at, + progress=100, + ) + + def test_jobs_endpoints(self): + job = self._create_job(task_id="inventory-job-1", status_value="success") + self.authenticate(self.user) + + list_response = self.client.get(reverse("api_v1:jobs:job-list")) + self.assertEqual(list_response.status_code, status.HTTP_200_OK) + self.assertEqual(list_response.data["results"][0]["task_id"], job.task_id) + + status_response = self.client.get( + reverse("api_v1:jobs:job-status", kwargs={"task_id": job.task_id}) + ) + self.assertEqual(status_response.status_code, status.HTTP_200_OK) + self.assertEqual(status_response.data["status"], "success") + + stream_response = self.client.get( + reverse("api_v1:jobs:job-stream", kwargs={"task_id": job.task_id}) + ) + self.assertEqual(stream_response.status_code, status.HTTP_200_OK) + stream_chunks = b"".join(stream_response.streaming_content).decode("utf-8") + self.assertIn("event: completed", stream_chunks) + self.assertIn(job.task_id, stream_chunks) + + +class ParsersApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + self.admin = UserFactory.create_superuser() + + def _create_financial_report(self) -> FinancialReport: + report = FinancialReport.objects.create( + external_id=_digits(5), + ogrn=_digits(13), + file_name=f"fin_{_digits(5)}_{_digits(13)}.xlsx", + file_hash=fake.sha256(raw_output=False), + load_batch=fake.random_int(min=1, max=1000), + status=FinancialReport.Status.SUCCESS, + source=FinancialReport.SourceType.API, + ) + FinancialReportLine.objects.create( + report=report, + form_code="1", + line_code="1100", + line_name="Assets", + year=2025, + period_start=100, + period_end=200, + ) + return report + + def test_registry_read_endpoints_for_parsers_data(self): + certificate = IndustrialCertificateRecordFactory() + manufacturer = ManufacturerRecordFactory() + product = IndustrialProductRecordFactory() + inspection = InspectionRecordFactory() + procurement = _create_procurement_record() + report = self._create_financial_report() + + self.authenticate(self.user) + + cert_list = self.client.get(reverse("api_v1:minpromtorg:certificates-list")) + cert_detail = self.client.get( + reverse("api_v1:minpromtorg:certificates-detail", args=[certificate.id]) + ) + manufacturers_list = self.client.get( + reverse("api_v1:minpromtorg:manufacturers-list") + ) + manufacturer_detail = self.client.get( + reverse("api_v1:minpromtorg:manufacturers-detail", args=[manufacturer.id]) + ) + products_list = self.client.get( + reverse("api_v1:minpromtorg:industrial-products-list") + ) + product_detail = self.client.get( + reverse("api_v1:minpromtorg:industrial-products-detail", args=[product.id]) + ) + inspections_list = self.client.get(reverse("api_v1:proverki:inspections-list")) + inspection_detail = self.client.get( + reverse("api_v1:proverki:inspections-detail", args=[inspection.id]) + ) + procurements_list = self.client.get(reverse("api_v1:zakupki:procurements-list")) + procurement_detail = self.client.get( + reverse("api_v1:zakupki:procurements-detail", args=[procurement.id]) + ) + fns_reports_list = self.client.get(reverse("api_v1:fns:fns-reports-list")) + fns_report_detail = self.client.get( + reverse("api_v1:fns:fns-reports-detail", args=[report.id]) + ) + + for response in ( + cert_list, + cert_detail, + manufacturers_list, + manufacturer_detail, + products_list, + product_detail, + inspections_list, + inspection_detail, + procurements_list, + procurement_detail, + fns_reports_list, + fns_report_detail, + ): + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_sources_parsing_and_system_endpoints(self): + shared_inn = _digits(10) + IndustrialCertificateRecordFactory(inn=shared_inn) + IndustrialProductRecordFactory(inn=shared_inn) + ManufacturerRecordFactory(load_batch=777, inn=shared_inn) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL, + status="success", + records_count=1, + ) + ParserLoadLogFactory( + source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, + status="success", + records_count=1, + ) + log = ParserLoadLogFactory( + source=ParserLoadLog.Source.MANUFACTURES, + batch_id=777, + status="success", + records_count=1, + ) + + self.authenticate(self.user) + sources_list = self.client.get(reverse("api_v1:sources:source-cards-list")) + sources_statuses = self.client.get( + reverse("api_v1:sources:source-cards-statuses") + ) + source_detail = self.client.get( + reverse( + "api_v1:sources:source-cards-detail", + kwargs={"slug": "manufacturers-and-products"}, + ) + ) + + self.assertEqual(sources_list.status_code, status.HTTP_200_OK) + self.assertEqual(sources_statuses.status_code, status.HTTP_200_OK) + self.assertEqual(source_detail.status_code, status.HTTP_200_OK) + + self.authenticate(self.admin) + parsing_url = reverse("api_v1:parsing:parsing-settings") + parsing_get = self.client.get(parsing_url) + parsing_patch = self.client.patch( + parsing_url, + {"planned_inspections": "weekly"}, + format="json", + ) + + logs_list = self.client.get(reverse("api_v1:system:parser-logs-list")) + logs_detail = self.client.get( + reverse("api_v1:system:parser-logs-detail", args=[log.id]) + ) + logs_export = self.client.get(reverse("api_v1:system:parser-logs-export")) + + with TemporaryDirectory() as tmp_dir: + base = Path(tmp_dir) + refresh_response = self.client.post( + reverse( + "api_v1:sources:source-cards-refresh", + kwargs={"slug": "financial-indicators"}, + ), + {}, + format="json", + ) + upload = SimpleUploadedFile( + f"fin_{_digits(5)}_{_digits(13)}.xlsx", + _build_fns_excel_bytes(), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + with self.settings( + FNS_WATCH_DIRECTORY=str(base / "watch"), + FNS_PROCESSED_DIRECTORY=str(base / "processed"), + FNS_FAILED_DIRECTORY=str(base / "failed"), + ): + fns_upload = self.client.post( + reverse("api_v1:fns:fns-upload"), + {"file": upload}, + format="multipart", + ) + + for response in ( + parsing_get, + parsing_patch, + logs_list, + logs_detail, + logs_export, + refresh_response, + fns_upload, + ): + self.assertIn( + response.status_code, + { + status.HTTP_200_OK, + status.HTTP_202_ACCEPTED, + }, + ) + + +class RegistersApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): + def setUp(self): + self.user = UserFactory.create_user() + self.admin = UserFactory.create_superuser() + + def test_registers_endpoints(self): + registry = RegisterFactory(name="Inventory Registry") + rows = [ + { + "pn_name": "Inventory Org", + "mn_ogrn": "1027600980990", + "mn_inn": "7601000086", + "in_kpp": "760401001", + "mn_okpo": "07506197", + } + ] + + self.authenticate(self.admin) + upload = SimpleUploadedFile( + "inventory-registry.xlsx", + _build_register_excel_bytes(rows), + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + ) + upload_response = self.client.post( + reverse("api_v1:registers:register-upload"), + { + "registry": str(registry.id), + "actual_date": date(2026, 1, 1).isoformat(), + "file": upload, + }, + format="multipart", + ) + self.assertEqual(upload_response.status_code, status.HTTP_201_CREATED) + + self.authenticate(self.user) + registries_list = self.client.get(reverse("api_v1:registers:registries-list")) + registries_detail = self.client.get( + reverse("api_v1:registers:registries-detail", args=[registry.id]) + ) + organizations_list = self.client.get( + reverse("api_v1:registers:organizations-list") + ) + organization_id = _extract_results(organizations_list.data)[0]["id"] + organization_detail = self.client.get( + reverse("api_v1:registers:organizations-detail", args=[organization_id]) + ) + registry_organizations = self.client.get( + reverse( + "api_v1:registers:registry-organizations-list", + args=[registry.id], + ) + ) + + for response in ( + registries_list, + registries_detail, + organizations_list, + organization_detail, + registry_organizations, + ): + self.assertEqual(response.status_code, status.HTTP_200_OK) + + +class ExchangeApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): + def setUp(self): + self.admin = UserFactory.create_superuser() + + @patch("apps.exchange.services.ExchangeConnectionService.validate_target_structure") + @patch("apps.exchange.services.ExchangeConnectionService.test_connection") + @patch("apps.exchange.services.ExchangeConnectionService.test_connection_payload") + @patch("apps.exchange.views.copy_parsers_data_async.delay") + @patch("apps.exchange.services.ExchangeConnectionService.get_active_connection") + def test_exchange_endpoints( + self, + get_active_connection_mock, + delay_mock, + test_connection_payload_mock, + _test_connection_mock, + _validate_mock, + ): + self.authenticate(self.admin) + connections_url = reverse("api_v1:exchange:connections") + test_connection_url = reverse("api_v1:exchange:connections-test") + copy_url = reverse("api_v1:exchange:copy") + periodic_tasks_url = reverse("api_v1:exchange:periodic-tasks") + + connection_payload = { + "server": "127.0.0.1", + "port": 5432, + "username": "postgres", + "password": "secret", + "database_name": "target_db", + "schema_name": "public", + } + test_connection_payload_mock.return_value = { + "status": "success", + "message": "ok", + } + + list_connections = self.client.get(connections_url) + create_connection = self.client.post( + connections_url, + connection_payload, + format="json", + ) + connection_test = self.client.post( + test_connection_url, + connection_payload, + format="json", + ) + + active_connection = ExchangeConnection.objects.get( + id=create_connection.data["id"] + ) + get_active_connection_mock.return_value = active_connection + delay_mock.return_value = SimpleNamespace(id="exchange-task-1") + copy_response = self.client.post(copy_url, {"mode": "all"}, format="json") + + list_periodic = self.client.get(periodic_tasks_url) + create_periodic = self.client.post( + periodic_tasks_url, + { + "name": "inventory-periodic-task", + "description": "inventory", + "enabled": True, + "schedule_type": "interval", + "interval_every": 1, + "interval_period": "hours", + "mode": "all", + "notify_on_error": True, + }, + format="json", + ) + periodic_id = create_periodic.data["id"] + periodic_detail_url = reverse( + "api_v1:exchange:periodic-task-detail", + args=[periodic_id], + ) + detail_periodic = self.client.get(periodic_detail_url) + patch_periodic = self.client.patch( + periodic_detail_url, + { + "name": "inventory-periodic-task-updated", + "enabled": False, + "schedule_type": "interval", + "interval_every": 2, + "interval_period": "hours", + "mode": "all", + "notify_on_error": False, + }, + format="json", + ) + + for response in ( + list_connections, + create_connection, + connection_test, + copy_response, + list_periodic, + create_periodic, + detail_periodic, + patch_periodic, + ): + self.assertIn( + response.status_code, + { + status.HTTP_200_OK, + status.HTTP_201_CREATED, + status.HTTP_202_ACCEPTED, + }, + ) + + self.assertTrue(PeriodicTask.objects.filter(id=periodic_id).exists()) + self.assertTrue( + ExchangeConnection.objects.filter(id=active_connection.id).exists() + ) + + +class BackupsApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): + def setUp(self): + self.admin = UserFactory.create_superuser() + + @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") + def test_backup_export_endpoint(self, enqueue_mock): + self.authenticate(self.admin) + today = timezone.localdate() + + with self.captureOnCommitCallbacks(execute=True): + response = self.client.post( + reverse("api_v1:backups:export"), + {"actual_date": today.isoformat()}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["status"], "started") + self.assertTrue(BackupExportJob.objects.filter(actual_date=today).exists()) + enqueue_mock.assert_called_once()