feat: expand platform APIs, sources, and test coverage
Some checks failed
CI/CD Pipeline / Run Tests (pull_request) Successful in 1m53s
CI/CD Pipeline / Telegram Notify Success (push) Has been cancelled
CI/CD Pipeline / Run Tests (push) Has been cancelled
CI/CD Pipeline / Code Quality Checks (push) Has been cancelled
CI/CD Pipeline / Code Quality Checks (pull_request) Failing after 2m54s
CI/CD Pipeline / Telegram Notify Success (pull_request) Has been skipped
Some checks failed
CI/CD Pipeline / Run Tests (pull_request) Successful in 1m53s
CI/CD Pipeline / Telegram Notify Success (push) Has been cancelled
CI/CD Pipeline / Run Tests (push) Has been cancelled
CI/CD Pipeline / Code Quality Checks (push) Has been cancelled
CI/CD Pipeline / Code Quality Checks (pull_request) Failing after 2m54s
CI/CD Pipeline / Telegram Notify Success (pull_request) Has been skipped
This commit is contained in:
@@ -1,6 +1,11 @@
|
||||
"""Модели приложения обмена данными."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
from apps.core.mixins import TimestampMixin
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
from django.db.models import Q
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
@@ -9,10 +14,15 @@ from django.utils.translation import gettext_lazy as _
|
||||
class ExchangeConnection(TimestampMixin, models.Model):
|
||||
"""Подключение к целевой БД для обмена данными."""
|
||||
|
||||
PASSWORD_PREFIX = "enc:v1:" # noqa: S105
|
||||
|
||||
server = models.CharField(_("сервер"), max_length=255)
|
||||
port = models.PositiveIntegerField(_("порт"), default=5432)
|
||||
username = models.CharField(_("пользователь"), max_length=255)
|
||||
password = models.TextField(_("пароль"), help_text=_("Хранится в открытом виде"))
|
||||
password = models.TextField(
|
||||
_("пароль"),
|
||||
help_text=_("Хранится в зашифрованном виде"),
|
||||
)
|
||||
database_name = models.CharField(_("имя БД"), max_length=255)
|
||||
schema_name = models.CharField(_("имя схемы"), max_length=255, default="public")
|
||||
is_active = models.BooleanField(_("активное"), default=False, db_index=True)
|
||||
@@ -41,3 +51,51 @@ class ExchangeConnection(TimestampMixin, models.Model):
|
||||
f"{self.username}@{self.server}:{self.port}/{self.database_name}"
|
||||
f"[{self.schema_name}]"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_cipher(cls) -> Fernet:
|
||||
secret_material = (
|
||||
getattr(settings, "EXCHANGE_CREDENTIALS_ENCRYPTION_KEY", "")
|
||||
or settings.SECRET_KEY
|
||||
)
|
||||
digest = hashlib.sha256(secret_material.encode("utf-8")).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(digest))
|
||||
|
||||
@classmethod
|
||||
def is_password_encrypted(cls, value: str) -> bool:
|
||||
return bool(value) and value.startswith(cls.PASSWORD_PREFIX)
|
||||
|
||||
@classmethod
|
||||
def encrypt_password(cls, raw_password: str) -> str:
|
||||
encrypted = (
|
||||
cls._get_cipher().encrypt(raw_password.encode("utf-8")).decode("ascii")
|
||||
)
|
||||
return f"{cls.PASSWORD_PREFIX}{encrypted}"
|
||||
|
||||
@classmethod
|
||||
def decrypt_password(cls, stored_password: str) -> str:
|
||||
if not cls.is_password_encrypted(stored_password):
|
||||
return stored_password
|
||||
|
||||
token = stored_password[len(cls.PASSWORD_PREFIX) :].encode("ascii")
|
||||
try:
|
||||
return cls._get_cipher().decrypt(token).decode("utf-8")
|
||||
except InvalidToken as exc:
|
||||
raise ValueError(
|
||||
"Не удалось расшифровать пароль exchange connection"
|
||||
) from exc
|
||||
|
||||
def get_decrypted_password(self) -> str:
|
||||
return self.decrypt_password(self.password)
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
password_was_encrypted = False
|
||||
if self.password and not self.is_password_encrypted(self.password):
|
||||
self.password = self.encrypt_password(self.password)
|
||||
password_was_encrypted = True
|
||||
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if password_was_encrypted and update_fields is not None:
|
||||
kwargs["update_fields"] = list(set(update_fields) | {"password"})
|
||||
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
@@ -79,7 +79,9 @@ class ExchangeConnectionService:
|
||||
cursor.execute("SELECT 1")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
cls._mark_connection_error(connection, str(exc))
|
||||
raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc
|
||||
raise ExchangeServiceError(
|
||||
f"Ошибка подключения к целевой БД: {exc}"
|
||||
) from exc
|
||||
|
||||
return alias
|
||||
|
||||
@@ -145,7 +147,9 @@ class ExchangeConnectionService:
|
||||
connections[alias].ensure_connection()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
cls._mark_connection_error(connection, str(exc))
|
||||
raise ExchangeServiceError(f"Ошибка подключения к целевой БД: {exc}") from exc
|
||||
raise ExchangeServiceError(
|
||||
f"Ошибка подключения к целевой БД: {exc}"
|
||||
) from exc
|
||||
|
||||
cls.validate_target_structure(
|
||||
connection=connection,
|
||||
@@ -187,7 +191,7 @@ class ExchangeConnectionService:
|
||||
"ENGINE": "django.db.backends.postgresql",
|
||||
"NAME": connection.database_name,
|
||||
"USER": connection.username,
|
||||
"PASSWORD": connection.password,
|
||||
"PASSWORD": connection.get_decrypted_password(),
|
||||
"HOST": connection.server,
|
||||
"PORT": connection.port,
|
||||
"OPTIONS": {
|
||||
@@ -348,7 +352,10 @@ class ExchangeConnectionService:
|
||||
@classmethod
|
||||
def _requires_registry_organizations(cls, models_to_copy: list) -> bool:
|
||||
return any(
|
||||
any(field.name == "registry_organization" for field in model._meta.local_fields)
|
||||
any(
|
||||
field.name == "registry_organization"
|
||||
for field in model._meta.local_fields
|
||||
)
|
||||
for model in models_to_copy
|
||||
)
|
||||
|
||||
@@ -377,7 +384,10 @@ class ExchangeConnectionService:
|
||||
pk_name = model._meta.pk.attname
|
||||
|
||||
for source_obj in queryset.iterator(chunk_size=chunk_size):
|
||||
row_data = {field_name: getattr(source_obj, field_name) for field_name in field_names}
|
||||
row_data = {
|
||||
field_name: getattr(source_obj, field_name)
|
||||
for field_name in field_names
|
||||
}
|
||||
batch.append(model(**row_data))
|
||||
|
||||
if len(batch) >= chunk_size:
|
||||
@@ -441,7 +451,9 @@ class ExchangeConnectionService:
|
||||
return len(existing_after - existing_before)
|
||||
|
||||
@classmethod
|
||||
def _mark_connection_error(cls, connection: ExchangeConnection, error_message: str) -> None:
|
||||
def _mark_connection_error(
|
||||
cls, connection: ExchangeConnection, error_message: str
|
||||
) -> None:
|
||||
connection.last_checked_at = timezone.now()
|
||||
connection.last_error = error_message
|
||||
connection.save(update_fields=["last_checked_at", "last_error", "updated_at"])
|
||||
|
||||
@@ -36,7 +36,9 @@ def copy_parsers_data_async(
|
||||
},
|
||||
)
|
||||
|
||||
connection = ExchangeConnection.objects.filter(id=connection_id, is_active=True).first()
|
||||
connection = ExchangeConnection.objects.filter(
|
||||
id=connection_id, is_active=True
|
||||
).first()
|
||||
if connection is None:
|
||||
background_job.fail(error="Активное подключение не найдено")
|
||||
raise ValueError(f"Active exchange connection not found: {connection_id}")
|
||||
|
||||
@@ -6,7 +6,9 @@ from django.urls import path
|
||||
app_name = "exchange"
|
||||
|
||||
exchange_urlpatterns = [
|
||||
path("connections/", ExchangeConnectionListCreateView.as_view(), name="connections"),
|
||||
path(
|
||||
"connections/", ExchangeConnectionListCreateView.as_view(), name="connections"
|
||||
),
|
||||
path("copy/", ExchangeCopyDataView.as_view(), name="copy"),
|
||||
]
|
||||
|
||||
|
||||
@@ -42,7 +42,9 @@ class ExchangeConnectionListCreateView(APIView):
|
||||
},
|
||||
)
|
||||
def get(self, request):
|
||||
queryset = ExchangeConnection.objects.all().order_by("-is_active", "-created_at")
|
||||
queryset = ExchangeConnection.objects.all().order_by(
|
||||
"-is_active", "-created_at"
|
||||
)
|
||||
serializer = ExchangeConnectionSerializer(queryset, many=True)
|
||||
return api_response(serializer.data, status_code=status.HTTP_200_OK)
|
||||
|
||||
@@ -122,7 +124,9 @@ class ExchangeCopyDataView(APIView):
|
||||
task = copy_parsers_data_async.delay(
|
||||
connection_id=active_connection.id,
|
||||
payload=serializer.validated_data,
|
||||
requested_by_id=request.user.id if request.user.is_authenticated else None,
|
||||
requested_by_id=request.user.id
|
||||
if request.user.is_authenticated
|
||||
else None,
|
||||
)
|
||||
|
||||
# Предсоздаём запись для мгновенного отслеживания в /api/v1/jobs/{task_id}/
|
||||
@@ -151,7 +155,9 @@ class ExchangeCopyDataView(APIView):
|
||||
"task_id": task.id,
|
||||
"connection_id": active_connection.id,
|
||||
"mode": serializer.validated_data["mode"],
|
||||
"truncate_before_copy": serializer.validated_data["truncate_before_copy"],
|
||||
"truncate_before_copy": serializer.validated_data[
|
||||
"truncate_before_copy"
|
||||
],
|
||||
},
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user