perf: batch acceptance backfill for deployed source volume
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-10-12 09:52:06 +02:00
parent 9b3da5b505
commit 1ffdcd0aaf
3 changed files with 96 additions and 20 deletions

View File

@@ -88,7 +88,9 @@ python src/manage.py build_source_record_exports
```
Первый вызов нормализации — dry run с количествами предполагаемых изменений.
Перед `--apply` проверить warnings и правило конфликтов. Apply не меняет uid,
Перед `--apply` проверить warnings и правило конфликтов. Нормализация пропускает записи без затронутых полей при уже известном филиале,
читает и блокирует каждую организацию один раз и обновляет payload пакетами.
Apply не меняет uid,
created_at, updated_at записей и raw; увеличивает revision затронутых published
артефактов и инвалидирует cache. Для legacy sources без artifact revision нет:
выдача tickets должна оставаться остановленной до обязательной пересборки exports.
@@ -119,3 +121,7 @@ created_at, updated_at записей и raw; увеличивает revision з
Это локальные backend/contract проверки на тестовых данных. Выкладка, применение
backfill к рабочей БД и приёмка через рабочий frontend не выполнялись.
При подготовке выкладки на dev обнаружено 3,86 млн записей. Добавлена проверка
пакетной нормализации 1001 записи одной организации с бюджетом менее 30 SQL
запросов; весь `test_testing_amendments.py` после оптимизации: **32 passed**.

View File

@@ -5,6 +5,7 @@ import json
from apps.parsers.models import ParserSourceArtifact
from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Q
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_cache import invalidate_source_data_cache
@@ -31,10 +32,19 @@ class Command(BaseCommand):
}
sources = set()
simulated_branches = {}
organizations = {}
with transaction.atomic():
queryset = OrganizationSourceRecord.objects.filter(
extension__source_group__in=BRANCH_GROUPS
).order_by("uid")
queryset = (
OrganizationSourceRecord.objects.filter(
extension__source_group__in=BRANCH_GROUPS
)
.filter(
Q(source__in=("industrial", "fstec"))
| Q(payload__has_any_keys=("is_branch", "filial"))
| Q(extension__organization__is_branch__isnull=True)
)
.order_by("uid")
)
if apply:
queryset = queryset.select_for_update()
# Keyset batches keep memory bounded and avoid server-side cursor writes.
@@ -43,14 +53,33 @@ class Command(BaseCommand):
page = (
queryset if last_uid is None else queryset.filter(uid__gt=last_uid)
)
records = list(
page.select_related("extension", "extension__organization")[:500]
)
records = list(page.select_related("extension")[:500])
if not records:
break
missing = {
record.extension.organization_id for record in records
} - organizations.keys()
organization_query = Organization.objects.filter(pk__in=missing)
if apply:
organization_query = organization_query.select_for_update()
organizations.update(
(organization.pk, organization)
for organization in organization_query.order_by("pk")
)
changed = []
for record in records:
self._normalize_record(
record, apply, result, sources, simulated_branches
if self._normalize_record(
record,
apply,
result,
sources,
simulated_branches,
organizations[record.extension.organization_id],
):
changed.append(record)
if apply and changed:
OrganizationSourceRecord.objects.bulk_update(
changed, ["payload"], batch_size=500
)
last_uid = records[-1].uid
if apply and sources:
@@ -69,14 +98,11 @@ class Command(BaseCommand):
return json.dumps(result, sort_keys=True)
@staticmethod
def _normalize_record(record, apply, result, sources, simulated_branches):
def _normalize_record(
record, apply, result, sources, simulated_branches, organization
):
result["scanned"] += 1
organization = record.extension.organization
if apply:
organization = Organization.objects.select_for_update().get(
pk=organization.pk
)
elif organization.pk in simulated_branches:
if not apply and organization.pk in simulated_branches:
organization.is_branch = simulated_branches[organization.pk]
previous = organization.is_branch
payload = normalize_record_payload(
@@ -104,7 +130,6 @@ class Command(BaseCommand):
if payload != record.payload:
result["changed_records"] += 1
sources.add(record.source)
if apply:
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
payload=payload
)
record.payload = payload
return True
return False

View File

@@ -385,3 +385,48 @@ def test_amount_ordering_remains_decimal_null_last(
response = client.get(LIST_URL, {"source_group": group, "ordering": ordering})
assert response.status_code == 200, response.data
assert [int(item["external_id"]) for item in response.data["data"]] == expected
def test_backfill_batches_repeated_organizations_without_per_record_queries(client):
from django.db import connection
from django.test.utils import CaptureQueriesContext
first = record(
"government_procurements",
"procurements_44fz",
1,
payload={"is_branch": False},
is_branch=False,
)
OrganizationSourceRecord.objects.bulk_create(
[
OrganizationSourceRecord(
extension=first.extension,
source=first.source,
record_type=first.record_type,
external_id=str(index),
title="Existing",
payload={"is_branch": True},
)
for index in range(2, 1002)
]
)
before = list(
OrganizationSourceRecord.objects.order_by("uid").values_list(
"uid", "updated_at"
)
)
with CaptureQueriesContext(connection) as queries:
output = StringIO()
call_command("normalize_testing_amendments", apply=True, stdout=output)
assert len(queries) < 30
assert json.loads(output.getvalue())["scanned"] == 1001
assert not OrganizationSourceRecord.objects.filter(payload__is_branch=True).exists()
assert (
list(
OrganizationSourceRecord.objects.order_by("uid").values_list(
"uid", "updated_at"
)
)
== before
)