Add organizations v2 API and registry enrichment

src/apps/core/services.py

@@ -300,6 +300,7 @@ class BulkOperationsMixin:
         unique_fields: list[str],
         update_fields: list[str],
         create_defaults: dict | None = None,
+        chunk_size: int = 500,
     ) -> tuple[int, int]:
         """
         Upsert: обновить существующие или создать новые.
@@ -309,31 +310,96 @@ class BulkOperationsMixin:
             unique_fields: Поля для поиска существующих
             update_fields: Поля для обновления
             create_defaults: Значения по умолчанию для создания
+            chunk_size: Размер чанка для bulk_create/bulk_update
 
         Returns:
             (created_count, updated_count)
         """
-        created_count = 0
-        updated_count = 0
+        if not items:
+            return 0, 0
+
         defaults = create_defaults or {}
 
-        for item in items:
-            lookup = {field: item[field] for field in unique_fields}
-            update_data = {
-                field: item[field] for field in update_fields if field in item
-            }
+        items_by_lookup = {
+            tuple(item[field] for field in unique_fields): item for item in items
+        }
+        existing_by_lookup = cls._bulk_existing_by_lookup(
+            lookup_keys=list(items_by_lookup),
+            unique_fields=unique_fields,
+            chunk_size=chunk_size,
+        )
 
-            obj, created = cls.model.objects.update_or_create(
-                **lookup,
-                defaults={**update_data, **defaults},
+        create_instances = []
+        update_instances = []
+        effective_update_fields = set(update_fields) | set(defaults)
+        if hasattr(cls.model, "updated_at"):
+            effective_update_fields.add("updated_at")
+        now = timezone.now()
+
+        for lookup_key, item in items_by_lookup.items():
+            existing = existing_by_lookup.get(lookup_key)
+            if existing is None:
+                create_instances.append(cls.model(**{**item, **defaults}))
+                continue
+
+            for field in update_fields:
+                if field in item:
+                    setattr(existing, field, item[field])
+            for field, value in defaults.items():
+                setattr(existing, field, value)
+            if hasattr(existing, "updated_at"):
+                existing.updated_at = now
+            update_instances.append(existing)
+
+        created_count = cls.bulk_create_chunked(
+            create_instances,
+            chunk_size=chunk_size,
+        )
+
+        update_fields_list = list(effective_update_fields)
+        for i in range(0, len(update_instances), chunk_size):
+            chunk = update_instances[i : i + chunk_size]
+            cls.model.objects.bulk_update(
+                chunk,
+                fields=update_fields_list,
+                batch_size=chunk_size,
             )
 
-            if created:
-                created_count += 1
-            else:
-                updated_count += 1
+        return created_count, len(update_instances)
 
-        return created_count, updated_count
+    @classmethod
+    def _bulk_existing_by_lookup(
+        cls,
+        *,
+        lookup_keys: list[tuple],
+        unique_fields: list[str],
+        chunk_size: int,
+    ) -> dict[tuple, models.Model]:
+        """Получить существующие записи по unique_fields без per-row запросов."""
+        if not lookup_keys:
+            return {}
+
+        existing_by_lookup = {}
+        if len(unique_fields) == 1:
+            field = unique_fields[0]
+            values = [key[0] for key in lookup_keys]
+            for i in range(0, len(values), chunk_size):
+                chunk = values[i : i + chunk_size]
+                for obj in cls.model.objects.filter(**{f"{field}__in": chunk}):
+                    existing_by_lookup[(getattr(obj, field),)] = obj
+            return existing_by_lookup
+
+        for i in range(0, len(lookup_keys), chunk_size):
+            chunk = lookup_keys[i : i + chunk_size]
+            query = Q()
+            for lookup_key in chunk:
+                query |= Q(**dict(zip(unique_fields, lookup_key, strict=True)))
+            for obj in cls.model.objects.filter(query):
+                existing_by_lookup[
+                    tuple(getattr(obj, field) for field in unique_fields)
+                ] = obj
+
+        return existing_by_lookup
 
     @classmethod
     @transaction.atomic