perf: batch acceptance backfill for deployed source volume
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-10-12 09:52:06 +02:00
parent 9b3da5b505
commit 1ffdcd0aaf
3 changed files with 96 additions and 20 deletions

View File

@@ -5,6 +5,7 @@ import json
from apps.parsers.models import ParserSourceArtifact
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Q
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_cache import invalidate_source_data_cache
@@ -31,10 +32,19 @@ class Command(BaseCommand):
}
sources = set()
simulated_branches = {}
organizations = {}
with transaction.atomic():
queryset = OrganizationSourceRecord.objects.filter(
extension__source_group__in=BRANCH_GROUPS
).order_by("uid")
queryset = (
OrganizationSourceRecord.objects.filter(
extension__source_group__in=BRANCH_GROUPS
)
.filter(
Q(source__in=("industrial", "fstec"))
| Q(payload__has_any_keys=("is_branch", "filial"))
| Q(extension__organization__is_branch__isnull=True)
)
.order_by("uid")
)
if apply:
queryset = queryset.select_for_update()
# Keyset batches keep memory bounded and avoid server-side cursor writes.
@@ -43,14 +53,33 @@ class Command(BaseCommand):
page = (
queryset if last_uid is None else queryset.filter(uid__gt=last_uid)
)
records = list(
page.select_related("extension", "extension__organization")[:500]
)
records = list(page.select_related("extension")[:500])
if not records:
break
missing = {
record.extension.organization_id for record in records
} - organizations.keys()
organization_query = Organization.objects.filter(pk__in=missing)
if apply:
organization_query = organization_query.select_for_update()
organizations.update(
(organization.pk, organization)
for organization in organization_query.order_by("pk")
)
changed = []
for record in records:
self._normalize_record(
record, apply, result, sources, simulated_branches
if self._normalize_record(
record,
apply,
result,
sources,
simulated_branches,
organizations[record.extension.organization_id],
):
changed.append(record)
if apply and changed:
OrganizationSourceRecord.objects.bulk_update(
changed, ["payload"], batch_size=500
)
last_uid = records[-1].uid
if apply and sources:
@@ -69,14 +98,11 @@ class Command(BaseCommand):
return json.dumps(result, sort_keys=True)
@staticmethod
def _normalize_record(record, apply, result, sources, simulated_branches):
def _normalize_record(
record, apply, result, sources, simulated_branches, organization
):
result["scanned"] += 1
organization = record.extension.organization
if apply:
organization = Organization.objects.select_for_update().get(
pk=organization.pk
)
elif organization.pk in simulated_branches:
if not apply and organization.pk in simulated_branches:
organization.is_branch = simulated_branches[organization.pk]
previous = organization.is_branch
payload = normalize_record_payload(
@@ -104,7 +130,6 @@ class Command(BaseCommand):
if payload != record.payload:
result["changed_records"] += 1
sources.add(record.source)
if apply:
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
payload=payload
)
record.payload = payload
return True
return False