feat(registry): add new endpoints for registers, exchange, and backups; update routing and configurations
Some checks failed
CI/CD Pipeline / Code Quality Checks (push) Failing after 3m10s
CI/CD Pipeline / Run Tests (push) Successful in 3m35s
CI/CD Pipeline / Telegram Notify Success (push) Has been skipped
CI/CD Pipeline / Code Quality Checks (pull_request) Failing after 2m26s
CI/CD Pipeline / Run Tests (pull_request) Successful in 2m46s
CI/CD Pipeline / Telegram Notify Success (pull_request) Has been skipped
Some checks failed
CI/CD Pipeline / Code Quality Checks (push) Failing after 3m10s
CI/CD Pipeline / Run Tests (push) Successful in 3m35s
CI/CD Pipeline / Telegram Notify Success (push) Has been skipped
CI/CD Pipeline / Code Quality Checks (pull_request) Failing after 2m26s
CI/CD Pipeline / Run Tests (pull_request) Successful in 2m46s
CI/CD Pipeline / Telegram Notify Success (pull_request) Has been skipped
This commit is contained in:
1
src/apps/exchange/__init__.py
Normal file
1
src/apps/exchange/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Приложение для обмена данными с внешней БД."""
|
||||
25
src/apps/exchange/admin.py
Normal file
25
src/apps/exchange/admin.py
Normal file
@@ -0,0 +1,25 @@
|
||||
"""Admin configuration for exchange app."""
|
||||
|
||||
from apps.exchange.models import ExchangeConnection
|
||||
from django.contrib import admin
|
||||
|
||||
|
||||
@admin.register(ExchangeConnection)
|
||||
class ExchangeConnectionAdmin(admin.ModelAdmin):
|
||||
"""Admin для подключений обмена."""
|
||||
|
||||
list_display = [
|
||||
"id",
|
||||
"server",
|
||||
"port",
|
||||
"username",
|
||||
"database_name",
|
||||
"schema_name",
|
||||
"is_active",
|
||||
"last_checked_at",
|
||||
"created_at",
|
||||
]
|
||||
list_filter = ["is_active", "created_at", "last_checked_at"]
|
||||
search_fields = ["server", "username", "database_name", "schema_name"]
|
||||
readonly_fields = ["created_at", "updated_at", "last_checked_at", "last_error"]
|
||||
ordering = ["-is_active", "-created_at"]
|
||||
9
src/apps/exchange/apps.py
Normal file
9
src/apps/exchange/apps.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ExchangeConfig(AppConfig):
|
||||
"""Конфигурация приложения обмена данными."""
|
||||
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.exchange"
|
||||
verbose_name = "Обмен данными"
|
||||
41
src/apps/exchange/migrations/0001_initial.py
Normal file
41
src/apps/exchange/migrations/0001_initial.py
Normal file
@@ -0,0 +1,41 @@
|
||||
# Generated by Django 3.2.25 on 2026-03-04 11:15
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='ExchangeConnection',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='Дата и время создания записи', verbose_name='создано')),
|
||||
('updated_at', models.DateTimeField(auto_now=True, help_text='Дата и время последнего обновления', verbose_name='обновлено')),
|
||||
('server', models.CharField(max_length=255, verbose_name='сервер')),
|
||||
('port', models.PositiveIntegerField(default=5432, verbose_name='порт')),
|
||||
('username', models.CharField(max_length=255, verbose_name='пользователь')),
|
||||
('password', models.TextField(help_text='Хранится в открытом виде', verbose_name='пароль')),
|
||||
('database_name', models.CharField(max_length=255, verbose_name='имя БД')),
|
||||
('schema_name', models.CharField(default='public', max_length=255, verbose_name='имя схемы')),
|
||||
('is_active', models.BooleanField(db_index=True, default=False, verbose_name='активное')),
|
||||
('last_checked_at', models.DateTimeField(blank=True, null=True, verbose_name='последняя проверка')),
|
||||
('last_error', models.TextField(blank=True, verbose_name='последняя ошибка')),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'подключение обмена',
|
||||
'verbose_name_plural': 'подключения обмена',
|
||||
'db_table': 'exchange_connection',
|
||||
'ordering': ['-is_active', '-created_at'],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='exchangeconnection',
|
||||
constraint=models.UniqueConstraint(condition=models.Q(('is_active', True)), fields=('is_active',), name='unique_active_exchange_connection'),
|
||||
),
|
||||
]
|
||||
0
src/apps/exchange/migrations/__init__.py
Normal file
0
src/apps/exchange/migrations/__init__.py
Normal file
43
src/apps/exchange/models.py
Normal file
43
src/apps/exchange/models.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""Модели приложения обмена данными."""
|
||||
|
||||
from apps.core.mixins import TimestampMixin
|
||||
from django.db import models
|
||||
from django.db.models import Q
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
||||
class ExchangeConnection(TimestampMixin, models.Model):
|
||||
"""Подключение к целевой БД для обмена данными."""
|
||||
|
||||
server = models.CharField(_("сервер"), max_length=255)
|
||||
port = models.PositiveIntegerField(_("порт"), default=5432)
|
||||
username = models.CharField(_("пользователь"), max_length=255)
|
||||
password = models.TextField(_("пароль"), help_text=_("Хранится в открытом виде"))
|
||||
database_name = models.CharField(_("имя БД"), max_length=255)
|
||||
schema_name = models.CharField(_("имя схемы"), max_length=255, default="public")
|
||||
is_active = models.BooleanField(_("активное"), default=False, db_index=True)
|
||||
last_checked_at = models.DateTimeField(
|
||||
_("последняя проверка"),
|
||||
null=True,
|
||||
blank=True,
|
||||
)
|
||||
last_error = models.TextField(_("последняя ошибка"), blank=True)
|
||||
|
||||
class Meta:
|
||||
db_table = "exchange_connection"
|
||||
verbose_name = _("подключение обмена")
|
||||
verbose_name_plural = _("подключения обмена")
|
||||
ordering = ["-is_active", "-created_at"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["is_active"],
|
||||
condition=Q(is_active=True),
|
||||
name="unique_active_exchange_connection",
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"{self.username}@{self.server}:{self.port}/{self.database_name}"
|
||||
f"[{self.schema_name}]"
|
||||
)
|
||||
88
src/apps/exchange/serializers.py
Normal file
88
src/apps/exchange/serializers.py
Normal file
@@ -0,0 +1,88 @@
|
||||
"""Сериализаторы приложения обмена данными."""
|
||||
|
||||
from apps.exchange.models import ExchangeConnection
|
||||
from rest_framework import serializers
|
||||
|
||||
|
||||
class ExchangeConnectionSerializer(serializers.ModelSerializer):
|
||||
"""Сериализатор подключения без выдачи пароля в ответах."""
|
||||
|
||||
class Meta:
|
||||
model = ExchangeConnection
|
||||
fields = [
|
||||
"id",
|
||||
"server",
|
||||
"port",
|
||||
"username",
|
||||
"database_name",
|
||||
"schema_name",
|
||||
"is_active",
|
||||
"last_checked_at",
|
||||
"last_error",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
read_only_fields = fields
|
||||
|
||||
|
||||
class ExchangeConnectionCreateSerializer(serializers.Serializer):
|
||||
"""Входные данные для создания активного подключения."""
|
||||
|
||||
server = serializers.CharField(max_length=255)
|
||||
port = serializers.IntegerField(min_value=1, max_value=65535)
|
||||
username = serializers.CharField(max_length=255)
|
||||
password = serializers.CharField()
|
||||
database_name = serializers.CharField(max_length=255)
|
||||
schema_name = serializers.RegexField(
|
||||
regex=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
||||
max_length=255,
|
||||
error_messages={
|
||||
"invalid": (
|
||||
"Имя схемы должно начинаться с буквы/_, "
|
||||
"содержать только буквы, цифры и _"
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class ExchangeCopyRequestSerializer(serializers.Serializer):
|
||||
"""Параметры запуска копирования данных."""
|
||||
|
||||
mode = serializers.ChoiceField(
|
||||
choices=["all", "single", "selected"],
|
||||
default="all",
|
||||
)
|
||||
table = serializers.CharField(required=False)
|
||||
tables = serializers.ListField(
|
||||
child=serializers.CharField(),
|
||||
required=False,
|
||||
allow_empty=False,
|
||||
)
|
||||
truncate_before_copy = serializers.BooleanField(default=True)
|
||||
|
||||
def validate(self, attrs):
|
||||
mode = attrs["mode"]
|
||||
table = attrs.get("table")
|
||||
tables = attrs.get("tables")
|
||||
|
||||
if mode == "single" and not table:
|
||||
raise serializers.ValidationError(
|
||||
{"table": "Для mode=single нужно указать table"}
|
||||
)
|
||||
|
||||
if mode == "selected" and not tables:
|
||||
raise serializers.ValidationError(
|
||||
{"tables": "Для mode=selected нужно указать tables"}
|
||||
)
|
||||
|
||||
if mode != "single" and table:
|
||||
raise serializers.ValidationError(
|
||||
{"table": "Поле table допустимо только для mode=single"}
|
||||
)
|
||||
|
||||
if mode != "selected" and tables:
|
||||
raise serializers.ValidationError(
|
||||
{"tables": "Поле tables допустимо только для mode=selected"}
|
||||
)
|
||||
|
||||
return attrs
|
||||
447
src/apps/exchange/services.py
Normal file
447
src/apps/exchange/services.py
Normal file
@@ -0,0 +1,447 @@
|
||||
"""Сервисы приложения обмена данными."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
from apps.exchange.models import ExchangeConnection
|
||||
from django.apps import apps as django_apps
|
||||
from django.db import connections, transaction
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
class ExchangeServiceError(ValueError):
|
||||
"""Ошибка операций приложения обмена данными."""
|
||||
|
||||
|
||||
class ExchangeConnectionService:
|
||||
"""Сервис управления подключениями и синхронизацией данных."""
|
||||
|
||||
PARSER_MODEL_LABELS = [
|
||||
"parsers.ParserLoadLog",
|
||||
"parsers.IndustrialCertificateRecord",
|
||||
"parsers.ManufacturerRecord",
|
||||
"parsers.Proxy",
|
||||
"parsers.InspectionRecord",
|
||||
"parsers.ProcurementRecord",
|
||||
"parsers.FinancialReport",
|
||||
"parsers.FinancialReportLine",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
@transaction.atomic
|
||||
def create_active_connection_and_prepare(cls, **payload) -> ExchangeConnection:
|
||||
"""
|
||||
Создать активное подключение.
|
||||
|
||||
В рамках одной операции:
|
||||
1. Деактивировать текущее активное подключение.
|
||||
2. Сохранить новое как активное.
|
||||
3. Проверить соединение и структуру target DB.
|
||||
|
||||
Важно: сервис НЕ изменяет структуру target DB.
|
||||
"""
|
||||
ExchangeConnection.objects.filter(is_active=True).update(is_active=False)
|
||||
connection = ExchangeConnection.objects.create(is_active=True, **payload)
|
||||
|
||||
try:
|
||||
alias = cls.test_connection(connection)
|
||||
cls.validate_target_structure(
|
||||
connection=connection,
|
||||
alias=alias,
|
||||
schema_name=connection.schema_name,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise ExchangeServiceError(str(exc)) from exc
|
||||
|
||||
connection.last_checked_at = timezone.now()
|
||||
connection.last_error = ""
|
||||
connection.save(update_fields=["last_checked_at", "last_error", "updated_at"])
|
||||
|
||||
return connection
|
||||
|
||||
@classmethod
|
||||
def get_active_connection(cls) -> ExchangeConnection:
|
||||
connection = ExchangeConnection.objects.filter(is_active=True).first()
|
||||
if not connection:
|
||||
raise ExchangeServiceError("Активное подключение не найдено")
|
||||
return connection
|
||||
|
||||
@classmethod
|
||||
def test_connection(cls, connection: ExchangeConnection) -> str:
|
||||
alias = cls._configure_alias(connection)
|
||||
|
||||
try:
|
||||
db_connection = connections[alias]
|
||||
db_connection.ensure_connection()
|
||||
with db_connection.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
cls._mark_connection_error(connection, str(exc))
|
||||
raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc
|
||||
|
||||
return alias
|
||||
|
||||
@classmethod
|
||||
def validate_target_structure(
|
||||
cls,
|
||||
*,
|
||||
connection: ExchangeConnection,
|
||||
alias: str,
|
||||
schema_name: str,
|
||||
models_to_copy: list | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Проверить структуру target DB без изменений.
|
||||
|
||||
Проверяет:
|
||||
- существование схемы
|
||||
- наличие всех обязательных таблиц
|
||||
- наличие всех обязательных колонок в таблицах
|
||||
"""
|
||||
try:
|
||||
db_connection = connections[alias]
|
||||
db_connection.ensure_connection()
|
||||
required_models = models_to_copy or cls._extend_models_with_dependencies(
|
||||
cls._get_parser_models()
|
||||
)
|
||||
cls._validate_schema_exists(alias=alias, schema_name=schema_name)
|
||||
cls._validate_tables_exist(
|
||||
alias=alias,
|
||||
schema_name=schema_name,
|
||||
models_to_copy=required_models,
|
||||
)
|
||||
cls._validate_columns_exist(
|
||||
alias=alias,
|
||||
schema_name=schema_name,
|
||||
models_to_copy=required_models,
|
||||
)
|
||||
except ExchangeServiceError as exc:
|
||||
cls._mark_connection_error(connection, str(exc))
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
cls._mark_connection_error(connection, str(exc))
|
||||
raise ExchangeServiceError(
|
||||
f"Ошибка проверки структуры целевой БД: {exc}"
|
||||
) from exc
|
||||
|
||||
@classmethod
|
||||
def copy_parsers_data(
|
||||
cls,
|
||||
*,
|
||||
connection: ExchangeConnection,
|
||||
mode: str,
|
||||
table: str | None = None,
|
||||
tables: list[str] | None = None,
|
||||
truncate_before_copy: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Скопировать данные из локальной БД в целевую БД."""
|
||||
alias = cls._configure_alias(connection)
|
||||
selected_models = cls._resolve_models(mode=mode, table=table, tables=tables)
|
||||
models_to_copy = cls._extend_models_with_dependencies(selected_models)
|
||||
|
||||
try:
|
||||
connections[alias].ensure_connection()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
cls._mark_connection_error(connection, str(exc))
|
||||
raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc
|
||||
|
||||
cls.validate_target_structure(
|
||||
connection=connection,
|
||||
alias=alias,
|
||||
schema_name=connection.schema_name,
|
||||
models_to_copy=models_to_copy,
|
||||
)
|
||||
|
||||
if truncate_before_copy:
|
||||
cls._truncate_tables(alias=alias, models_to_copy=models_to_copy)
|
||||
|
||||
copied_by_table: dict[str, int] = {}
|
||||
for model in models_to_copy:
|
||||
copied_by_table[model._meta.db_table] = cls._copy_model_data(
|
||||
model=model,
|
||||
alias=alias,
|
||||
truncate_before_copy=truncate_before_copy,
|
||||
)
|
||||
|
||||
total_rows = sum(copied_by_table.values())
|
||||
|
||||
connection.last_checked_at = timezone.now()
|
||||
connection.last_error = ""
|
||||
connection.save(update_fields=["last_checked_at", "last_error", "updated_at"])
|
||||
|
||||
return {
|
||||
"mode": mode,
|
||||
"tables": list(copied_by_table.keys()),
|
||||
"rows_by_table": copied_by_table,
|
||||
"total_rows": total_rows,
|
||||
"truncate_before_copy": truncate_before_copy,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _configure_alias(cls, connection: ExchangeConnection) -> str:
|
||||
alias = f"exchange_target_{connection.id}"
|
||||
|
||||
config = {
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"NAME": connection.database_name,
|
||||
"USER": connection.username,
|
||||
"PASSWORD": connection.password,
|
||||
"HOST": connection.server,
|
||||
"PORT": connection.port,
|
||||
"OPTIONS": {
|
||||
"options": f"-c search_path={connection.schema_name},public",
|
||||
},
|
||||
"CONN_MAX_AGE": 0,
|
||||
"ATOMIC_REQUESTS": False,
|
||||
"AUTOCOMMIT": True,
|
||||
"TIME_ZONE": None,
|
||||
"TEST": {},
|
||||
}
|
||||
|
||||
if alias in connections.databases:
|
||||
with suppress(Exception):
|
||||
connections[alias].close()
|
||||
|
||||
connections.databases[alias] = config
|
||||
|
||||
storage = getattr(connections, "_connections", None)
|
||||
if storage is not None and hasattr(storage, "__dict__"):
|
||||
storage.__dict__.pop(alias, None)
|
||||
|
||||
return alias
|
||||
|
||||
@classmethod
|
||||
def _validate_schema_exists(cls, *, alias: str, schema_name: str) -> None:
|
||||
with connections[alias].cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM information_schema.schemata
|
||||
WHERE schema_name = %s
|
||||
""",
|
||||
[schema_name],
|
||||
)
|
||||
if cursor.fetchone() is None:
|
||||
raise ExchangeServiceError(
|
||||
f"Схема '{schema_name}' отсутствует в целевой БД"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_tables_exist(
|
||||
cls,
|
||||
*,
|
||||
alias: str,
|
||||
schema_name: str,
|
||||
models_to_copy: list,
|
||||
) -> None:
|
||||
expected_tables = {model._meta.db_table for model in models_to_copy}
|
||||
with connections[alias].cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = %s
|
||||
""",
|
||||
[schema_name],
|
||||
)
|
||||
existing_tables = {row[0] for row in cursor.fetchall()}
|
||||
|
||||
missing_tables = sorted(expected_tables - existing_tables)
|
||||
if missing_tables:
|
||||
raise ExchangeServiceError(
|
||||
"В целевой БД отсутствуют таблицы: " + ", ".join(missing_tables)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_columns_exist(
|
||||
cls,
|
||||
*,
|
||||
alias: str,
|
||||
schema_name: str,
|
||||
models_to_copy: list,
|
||||
) -> None:
|
||||
for model in models_to_copy:
|
||||
table_name = model._meta.db_table
|
||||
expected_columns = {field.column for field in model._meta.local_fields}
|
||||
|
||||
with connections[alias].cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
""",
|
||||
[schema_name, table_name],
|
||||
)
|
||||
existing_columns = {row[0] for row in cursor.fetchall()}
|
||||
|
||||
missing_columns = sorted(expected_columns - existing_columns)
|
||||
if missing_columns:
|
||||
raise ExchangeServiceError(
|
||||
f"В таблице '{table_name}' отсутствуют колонки: "
|
||||
+ ", ".join(missing_columns)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_parser_models(cls) -> list:
|
||||
return [django_apps.get_model(label) for label in cls.PARSER_MODEL_LABELS]
|
||||
|
||||
@classmethod
|
||||
def _resolve_models(
|
||||
cls,
|
||||
*,
|
||||
mode: str,
|
||||
table: str | None,
|
||||
tables: list[str] | None,
|
||||
) -> list:
|
||||
parser_models = cls._get_parser_models()
|
||||
|
||||
if mode == "all":
|
||||
return parser_models
|
||||
|
||||
mapping: dict[str, Any] = {}
|
||||
for model in parser_models:
|
||||
mapping[model._meta.db_table] = model
|
||||
mapping[model._meta.model_name] = model
|
||||
mapping[model.__name__.lower()] = model
|
||||
|
||||
requested_names: list[str]
|
||||
if mode == "single":
|
||||
requested_names = [table] if table else []
|
||||
else:
|
||||
requested_names = tables or []
|
||||
|
||||
resolved_models = []
|
||||
for requested_name in requested_names:
|
||||
model = mapping.get(requested_name)
|
||||
if not model:
|
||||
available = ", ".join(sorted(m._meta.db_table for m in parser_models))
|
||||
raise ExchangeServiceError(
|
||||
f"Неизвестная таблица '{requested_name}'. Доступные: {available}"
|
||||
)
|
||||
resolved_models.append(model)
|
||||
|
||||
return resolved_models
|
||||
|
||||
@classmethod
|
||||
def _extend_models_with_dependencies(cls, models_to_copy: list) -> list:
|
||||
"""Добавить обязательные зависимые модели для корректного copy."""
|
||||
if not cls._requires_registry_organizations(models_to_copy):
|
||||
return models_to_copy
|
||||
|
||||
organization_model = django_apps.get_model("registers.Organization")
|
||||
ordered_models = [organization_model, *models_to_copy]
|
||||
|
||||
unique_models = []
|
||||
seen = set()
|
||||
for model in ordered_models:
|
||||
model_key = (model._meta.app_label, model._meta.model_name)
|
||||
if model_key in seen:
|
||||
continue
|
||||
seen.add(model_key)
|
||||
unique_models.append(model)
|
||||
|
||||
return unique_models
|
||||
|
||||
@classmethod
|
||||
def _requires_registry_organizations(cls, models_to_copy: list) -> bool:
|
||||
return any(
|
||||
any(field.name == "registry_organization" for field in model._meta.local_fields)
|
||||
for model in models_to_copy
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _truncate_tables(cls, *, alias: str, models_to_copy: list) -> None:
|
||||
with connections[alias].cursor() as cursor:
|
||||
for model in reversed(models_to_copy):
|
||||
cursor.execute(
|
||||
f'TRUNCATE TABLE "{model._meta.db_table}" RESTART IDENTITY CASCADE'
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _copy_model_data(
|
||||
cls,
|
||||
*,
|
||||
model,
|
||||
alias: str,
|
||||
truncate_before_copy: bool,
|
||||
chunk_size: int = 1000,
|
||||
) -> int:
|
||||
field_names = [field.attname for field in model._meta.local_fields]
|
||||
queryset = model.objects.using("default").all().order_by("pk")
|
||||
|
||||
total_processed = 0
|
||||
batch = []
|
||||
pk_name = model._meta.pk.attname
|
||||
|
||||
for source_obj in queryset.iterator(chunk_size=chunk_size):
|
||||
row_data = {field_name: getattr(source_obj, field_name) for field_name in field_names}
|
||||
batch.append(model(**row_data))
|
||||
|
||||
if len(batch) >= chunk_size:
|
||||
total_processed += cls._insert_batch(
|
||||
model=model,
|
||||
alias=alias,
|
||||
batch=batch,
|
||||
pk_name=pk_name,
|
||||
chunk_size=chunk_size,
|
||||
truncate_before_copy=truncate_before_copy,
|
||||
)
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
total_processed += cls._insert_batch(
|
||||
model=model,
|
||||
alias=alias,
|
||||
batch=batch,
|
||||
pk_name=pk_name,
|
||||
chunk_size=chunk_size,
|
||||
truncate_before_copy=truncate_before_copy,
|
||||
)
|
||||
|
||||
return total_processed
|
||||
|
||||
@classmethod
|
||||
def _insert_batch(
|
||||
cls,
|
||||
*,
|
||||
model,
|
||||
alias: str,
|
||||
batch: list,
|
||||
pk_name: str,
|
||||
chunk_size: int,
|
||||
truncate_before_copy: bool,
|
||||
) -> int:
|
||||
if truncate_before_copy:
|
||||
model.objects.using(alias).bulk_create(
|
||||
batch,
|
||||
batch_size=chunk_size,
|
||||
ignore_conflicts=False,
|
||||
)
|
||||
return len(batch)
|
||||
|
||||
pk_values = [getattr(item, pk_name) for item in batch]
|
||||
existing_before = set(
|
||||
model.objects.using(alias)
|
||||
.filter(**{f"{pk_name}__in": pk_values})
|
||||
.values_list(pk_name, flat=True)
|
||||
)
|
||||
model.objects.using(alias).bulk_create(
|
||||
batch,
|
||||
batch_size=chunk_size,
|
||||
ignore_conflicts=True,
|
||||
)
|
||||
existing_after = set(
|
||||
model.objects.using(alias)
|
||||
.filter(**{f"{pk_name}__in": pk_values})
|
||||
.values_list(pk_name, flat=True)
|
||||
)
|
||||
return len(existing_after - existing_before)
|
||||
|
||||
@classmethod
|
||||
def _mark_connection_error(cls, connection: ExchangeConnection, error_message: str) -> None:
|
||||
connection.last_checked_at = timezone.now()
|
||||
connection.last_error = error_message
|
||||
connection.save(update_fields=["last_checked_at", "last_error", "updated_at"])
|
||||
67
src/apps/exchange/tasks.py
Normal file
67
src/apps/exchange/tasks.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""Celery-задачи приложения exchange."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from apps.core.services import BackgroundJobService
|
||||
from apps.exchange.models import ExchangeConnection
|
||||
from apps.exchange.services import ExchangeConnectionService
|
||||
from celery import shared_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task(bind=True)
|
||||
def copy_parsers_data_async(
|
||||
self,
|
||||
*,
|
||||
connection_id: int,
|
||||
payload: dict[str, Any],
|
||||
requested_by_id: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Асинхронное копирование parser-данных в target DB."""
|
||||
task_id = self.request.id or str(uuid.uuid4())
|
||||
background_job = BackgroundJobService.get_by_task_id_or_none(task_id)
|
||||
if not background_job:
|
||||
background_job = BackgroundJobService.create_job(
|
||||
task_id=task_id,
|
||||
task_name="apps.exchange.tasks.copy_parsers_data_async",
|
||||
user_id=requested_by_id,
|
||||
meta={
|
||||
"connection_id": connection_id,
|
||||
**payload,
|
||||
},
|
||||
)
|
||||
|
||||
connection = ExchangeConnection.objects.filter(id=connection_id, is_active=True).first()
|
||||
if connection is None:
|
||||
background_job.fail(error="Активное подключение не найдено")
|
||||
raise ValueError(f"Active exchange connection not found: {connection_id}")
|
||||
|
||||
background_job.mark_started()
|
||||
background_job.update_progress(10, "Проверка структуры целевой БД")
|
||||
|
||||
try:
|
||||
result = ExchangeConnectionService.copy_parsers_data(
|
||||
connection=connection,
|
||||
**payload,
|
||||
)
|
||||
background_job.update_progress(90, "Фиксация результата")
|
||||
output = {
|
||||
"status": "success",
|
||||
"connection_id": connection_id,
|
||||
**result,
|
||||
}
|
||||
background_job.complete(result=output)
|
||||
return output
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception(
|
||||
"Exchange copy failed (connection_id=%s, task_id=%s)",
|
||||
connection_id,
|
||||
task_id,
|
||||
)
|
||||
background_job.fail(error=str(exc))
|
||||
raise
|
||||
13
src/apps/exchange/urls.py
Normal file
13
src/apps/exchange/urls.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""URL конфигурация приложения exchange."""
|
||||
|
||||
from apps.exchange.views import ExchangeConnectionListCreateView, ExchangeCopyDataView
|
||||
from django.urls import path
|
||||
|
||||
app_name = "exchange"
|
||||
|
||||
exchange_urlpatterns = [
|
||||
path("connections/", ExchangeConnectionListCreateView.as_view(), name="connections"),
|
||||
path("copy/", ExchangeCopyDataView.as_view(), name="copy"),
|
||||
]
|
||||
|
||||
urlpatterns = []
|
||||
157
src/apps/exchange/views.py
Normal file
157
src/apps/exchange/views.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""API views для обмена данными с внешней БД."""
|
||||
|
||||
from contextlib import suppress
|
||||
|
||||
from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag
|
||||
from apps.core.response import api_created_response, api_response
|
||||
from apps.core.services import BackgroundJobService
|
||||
from apps.exchange.models import ExchangeConnection
|
||||
from apps.exchange.serializers import (
|
||||
ExchangeConnectionCreateSerializer,
|
||||
ExchangeConnectionSerializer,
|
||||
ExchangeCopyRequestSerializer,
|
||||
)
|
||||
from apps.exchange.services import ExchangeConnectionService, ExchangeServiceError
|
||||
from apps.exchange.tasks import copy_parsers_data_async
|
||||
from django.db import IntegrityError
|
||||
from drf_yasg import openapi
|
||||
from drf_yasg.utils import swagger_auto_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import ValidationError
|
||||
from rest_framework.permissions import IsAdminUser
|
||||
from rest_framework.views import APIView
|
||||
|
||||
EXCHANGE_TAG = swagger_tag("Обмен данными", "exchange")
|
||||
|
||||
|
||||
class ExchangeConnectionListCreateView(APIView):
|
||||
"""API списка и создания подключений обмена."""
|
||||
|
||||
permission_classes = [IsAdminUser]
|
||||
|
||||
@swagger_auto_schema(
|
||||
tags=[EXCHANGE_TAG],
|
||||
operation_summary="Список подключений",
|
||||
operation_description=(
|
||||
"Возвращает список всех сохранённых подключений для обмена.\n"
|
||||
"Пароль в ответ не возвращается."
|
||||
),
|
||||
responses={
|
||||
200: ExchangeConnectionSerializer(many=True),
|
||||
**ErrorResponses.ADMIN,
|
||||
},
|
||||
)
|
||||
def get(self, request):
|
||||
queryset = ExchangeConnection.objects.all().order_by("-is_active", "-created_at")
|
||||
serializer = ExchangeConnectionSerializer(queryset, many=True)
|
||||
return api_response(serializer.data, status_code=status.HTTP_200_OK)
|
||||
|
||||
@swagger_auto_schema(
|
||||
tags=[EXCHANGE_TAG],
|
||||
operation_summary="Создать активное подключение",
|
||||
operation_description=(
|
||||
"Создаёт новое подключение к целевой БД как активное.\n"
|
||||
"Перед созданием деактивирует текущее активное подключение.\n"
|
||||
"После сохранения проверяет соединение и валидирует структуру целевой БД."
|
||||
),
|
||||
request_body=ExchangeConnectionCreateSerializer,
|
||||
responses={
|
||||
201: ExchangeConnectionSerializer,
|
||||
400: CommonResponses.BAD_REQUEST,
|
||||
**ErrorResponses.ADMIN,
|
||||
},
|
||||
)
|
||||
def post(self, request):
|
||||
serializer = ExchangeConnectionCreateSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
try:
|
||||
connection = ExchangeConnectionService.create_active_connection_and_prepare(
|
||||
**serializer.validated_data
|
||||
)
|
||||
except ExchangeServiceError as exc:
|
||||
raise ValidationError({"connection": str(exc)}) from exc
|
||||
|
||||
output = ExchangeConnectionSerializer(connection)
|
||||
return api_created_response(output.data)
|
||||
|
||||
|
||||
class ExchangeCopyDataView(APIView):
|
||||
"""API запуска копирования данных в целевую БД."""
|
||||
|
||||
permission_classes = [IsAdminUser]
|
||||
|
||||
@swagger_auto_schema(
|
||||
tags=[EXCHANGE_TAG],
|
||||
operation_summary="Копировать данные parsers в target DB",
|
||||
operation_description=(
|
||||
"Асинхронно запускает копирование данных из локальной БД "
|
||||
"в активную целевую БД.\n"
|
||||
"Перед копированием выполняется только проверка структуры "
|
||||
"(без изменения схемы/миграций).\n"
|
||||
"Поддерживает режимы: all / single / selected."
|
||||
),
|
||||
request_body=ExchangeCopyRequestSerializer,
|
||||
responses={
|
||||
202: openapi.Response(
|
||||
description="Копирование поставлено в очередь",
|
||||
schema=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
"status": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
"message": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
"task_id": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
"connection_id": openapi.Schema(type=openapi.TYPE_INTEGER),
|
||||
"mode": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
"truncate_before_copy": openapi.Schema(
|
||||
type=openapi.TYPE_BOOLEAN
|
||||
),
|
||||
},
|
||||
),
|
||||
),
|
||||
400: CommonResponses.BAD_REQUEST,
|
||||
**ErrorResponses.ADMIN,
|
||||
},
|
||||
)
|
||||
def post(self, request):
|
||||
serializer = ExchangeCopyRequestSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
try:
|
||||
active_connection = ExchangeConnectionService.get_active_connection()
|
||||
task = copy_parsers_data_async.delay(
|
||||
connection_id=active_connection.id,
|
||||
payload=serializer.validated_data,
|
||||
requested_by_id=request.user.id if request.user.is_authenticated else None,
|
||||
)
|
||||
|
||||
# Предсоздаём запись для мгновенного отслеживания в /api/v1/jobs/{task_id}/
|
||||
with suppress(IntegrityError):
|
||||
BackgroundJobService.create_job(
|
||||
task_id=task.id,
|
||||
task_name="apps.exchange.tasks.copy_parsers_data_async",
|
||||
user_id=request.user.id if request.user.is_authenticated else None,
|
||||
meta={
|
||||
"connection_id": active_connection.id,
|
||||
"mode": serializer.validated_data["mode"],
|
||||
"table": serializer.validated_data.get("table"),
|
||||
"tables": serializer.validated_data.get("tables"),
|
||||
"truncate_before_copy": serializer.validated_data.get(
|
||||
"truncate_before_copy"
|
||||
),
|
||||
},
|
||||
)
|
||||
except ExchangeServiceError as exc:
|
||||
raise ValidationError({"copy": str(exc)}) from exc
|
||||
|
||||
return api_response(
|
||||
{
|
||||
"status": "started",
|
||||
"message": "Копирование запущено в фоне.",
|
||||
"task_id": task.id,
|
||||
"connection_id": active_connection.id,
|
||||
"mode": serializer.validated_data["mode"],
|
||||
"truncate_before_copy": serializer.validated_data["truncate_before_copy"],
|
||||
},
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
Reference in New Issue
Block a user