perf: batch acceptance backfill for deployed source volume
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
@@ -5,6 +5,7 @@ import json
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from organizations.source_cache import invalidate_source_data_cache
|
||||
@@ -31,10 +32,19 @@ class Command(BaseCommand):
|
||||
}
|
||||
sources = set()
|
||||
simulated_branches = {}
|
||||
organizations = {}
|
||||
with transaction.atomic():
|
||||
queryset = OrganizationSourceRecord.objects.filter(
|
||||
extension__source_group__in=BRANCH_GROUPS
|
||||
).order_by("uid")
|
||||
queryset = (
|
||||
OrganizationSourceRecord.objects.filter(
|
||||
extension__source_group__in=BRANCH_GROUPS
|
||||
)
|
||||
.filter(
|
||||
Q(source__in=("industrial", "fstec"))
|
||||
| Q(payload__has_any_keys=("is_branch", "filial"))
|
||||
| Q(extension__organization__is_branch__isnull=True)
|
||||
)
|
||||
.order_by("uid")
|
||||
)
|
||||
if apply:
|
||||
queryset = queryset.select_for_update()
|
||||
# Keyset batches keep memory bounded and avoid server-side cursor writes.
|
||||
@@ -43,14 +53,33 @@ class Command(BaseCommand):
|
||||
page = (
|
||||
queryset if last_uid is None else queryset.filter(uid__gt=last_uid)
|
||||
)
|
||||
records = list(
|
||||
page.select_related("extension", "extension__organization")[:500]
|
||||
)
|
||||
records = list(page.select_related("extension")[:500])
|
||||
if not records:
|
||||
break
|
||||
missing = {
|
||||
record.extension.organization_id for record in records
|
||||
} - organizations.keys()
|
||||
organization_query = Organization.objects.filter(pk__in=missing)
|
||||
if apply:
|
||||
organization_query = organization_query.select_for_update()
|
||||
organizations.update(
|
||||
(organization.pk, organization)
|
||||
for organization in organization_query.order_by("pk")
|
||||
)
|
||||
changed = []
|
||||
for record in records:
|
||||
self._normalize_record(
|
||||
record, apply, result, sources, simulated_branches
|
||||
if self._normalize_record(
|
||||
record,
|
||||
apply,
|
||||
result,
|
||||
sources,
|
||||
simulated_branches,
|
||||
organizations[record.extension.organization_id],
|
||||
):
|
||||
changed.append(record)
|
||||
if apply and changed:
|
||||
OrganizationSourceRecord.objects.bulk_update(
|
||||
changed, ["payload"], batch_size=500
|
||||
)
|
||||
last_uid = records[-1].uid
|
||||
if apply and sources:
|
||||
@@ -69,14 +98,11 @@ class Command(BaseCommand):
|
||||
return json.dumps(result, sort_keys=True)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_record(record, apply, result, sources, simulated_branches):
|
||||
def _normalize_record(
|
||||
record, apply, result, sources, simulated_branches, organization
|
||||
):
|
||||
result["scanned"] += 1
|
||||
organization = record.extension.organization
|
||||
if apply:
|
||||
organization = Organization.objects.select_for_update().get(
|
||||
pk=organization.pk
|
||||
)
|
||||
elif organization.pk in simulated_branches:
|
||||
if not apply and organization.pk in simulated_branches:
|
||||
organization.is_branch = simulated_branches[organization.pk]
|
||||
previous = organization.is_branch
|
||||
payload = normalize_record_payload(
|
||||
@@ -104,7 +130,6 @@ class Command(BaseCommand):
|
||||
if payload != record.payload:
|
||||
result["changed_records"] += 1
|
||||
sources.add(record.source)
|
||||
if apply:
|
||||
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
|
||||
payload=payload
|
||||
)
|
||||
record.payload = payload
|
||||
return True
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user