perf: batch acceptance backfill for deployed source volume
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
@@ -88,7 +88,9 @@ python src/manage.py build_source_record_exports
|
||||
```
|
||||
|
||||
Первый вызов нормализации — dry run с количествами предполагаемых изменений.
|
||||
Перед `--apply` проверить warnings и правило конфликтов. Apply не меняет uid,
|
||||
Перед `--apply` проверить warnings и правило конфликтов. Нормализация пропускает записи без затронутых полей при уже известном филиале,
|
||||
читает и блокирует каждую организацию один раз и обновляет payload пакетами.
|
||||
Apply не меняет uid,
|
||||
created_at, updated_at записей и raw; увеличивает revision затронутых published
|
||||
артефактов и инвалидирует cache. Для legacy sources без artifact revision нет:
|
||||
выдача tickets должна оставаться остановленной до обязательной пересборки exports.
|
||||
@@ -119,3 +121,7 @@ created_at, updated_at записей и raw; увеличивает revision з
|
||||
|
||||
Это локальные backend/contract проверки на тестовых данных. Выкладка, применение
|
||||
backfill к рабочей БД и приёмка через рабочий frontend не выполнялись.
|
||||
|
||||
При подготовке выкладки на dev обнаружено 3,86 млн записей. Добавлена проверка
|
||||
пакетной нормализации 1001 записи одной организации с бюджетом менее 30 SQL
|
||||
запросов; весь `test_testing_amendments.py` после оптимизации: **32 passed**.
|
||||
|
||||
@@ -5,6 +5,7 @@ import json
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
from django.db.models import Q
|
||||
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from organizations.source_cache import invalidate_source_data_cache
|
||||
@@ -31,10 +32,19 @@ class Command(BaseCommand):
|
||||
}
|
||||
sources = set()
|
||||
simulated_branches = {}
|
||||
organizations = {}
|
||||
with transaction.atomic():
|
||||
queryset = OrganizationSourceRecord.objects.filter(
|
||||
extension__source_group__in=BRANCH_GROUPS
|
||||
).order_by("uid")
|
||||
queryset = (
|
||||
OrganizationSourceRecord.objects.filter(
|
||||
extension__source_group__in=BRANCH_GROUPS
|
||||
)
|
||||
.filter(
|
||||
Q(source__in=("industrial", "fstec"))
|
||||
| Q(payload__has_any_keys=("is_branch", "filial"))
|
||||
| Q(extension__organization__is_branch__isnull=True)
|
||||
)
|
||||
.order_by("uid")
|
||||
)
|
||||
if apply:
|
||||
queryset = queryset.select_for_update()
|
||||
# Keyset batches keep memory bounded and avoid server-side cursor writes.
|
||||
@@ -43,14 +53,33 @@ class Command(BaseCommand):
|
||||
page = (
|
||||
queryset if last_uid is None else queryset.filter(uid__gt=last_uid)
|
||||
)
|
||||
records = list(
|
||||
page.select_related("extension", "extension__organization")[:500]
|
||||
)
|
||||
records = list(page.select_related("extension")[:500])
|
||||
if not records:
|
||||
break
|
||||
missing = {
|
||||
record.extension.organization_id for record in records
|
||||
} - organizations.keys()
|
||||
organization_query = Organization.objects.filter(pk__in=missing)
|
||||
if apply:
|
||||
organization_query = organization_query.select_for_update()
|
||||
organizations.update(
|
||||
(organization.pk, organization)
|
||||
for organization in organization_query.order_by("pk")
|
||||
)
|
||||
changed = []
|
||||
for record in records:
|
||||
self._normalize_record(
|
||||
record, apply, result, sources, simulated_branches
|
||||
if self._normalize_record(
|
||||
record,
|
||||
apply,
|
||||
result,
|
||||
sources,
|
||||
simulated_branches,
|
||||
organizations[record.extension.organization_id],
|
||||
):
|
||||
changed.append(record)
|
||||
if apply and changed:
|
||||
OrganizationSourceRecord.objects.bulk_update(
|
||||
changed, ["payload"], batch_size=500
|
||||
)
|
||||
last_uid = records[-1].uid
|
||||
if apply and sources:
|
||||
@@ -69,14 +98,11 @@ class Command(BaseCommand):
|
||||
return json.dumps(result, sort_keys=True)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_record(record, apply, result, sources, simulated_branches):
|
||||
def _normalize_record(
|
||||
record, apply, result, sources, simulated_branches, organization
|
||||
):
|
||||
result["scanned"] += 1
|
||||
organization = record.extension.organization
|
||||
if apply:
|
||||
organization = Organization.objects.select_for_update().get(
|
||||
pk=organization.pk
|
||||
)
|
||||
elif organization.pk in simulated_branches:
|
||||
if not apply and organization.pk in simulated_branches:
|
||||
organization.is_branch = simulated_branches[organization.pk]
|
||||
previous = organization.is_branch
|
||||
payload = normalize_record_payload(
|
||||
@@ -104,7 +130,6 @@ class Command(BaseCommand):
|
||||
if payload != record.payload:
|
||||
result["changed_records"] += 1
|
||||
sources.add(record.source)
|
||||
if apply:
|
||||
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
|
||||
payload=payload
|
||||
)
|
||||
record.payload = payload
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -385,3 +385,48 @@ def test_amount_ordering_remains_decimal_null_last(
|
||||
response = client.get(LIST_URL, {"source_group": group, "ordering": ordering})
|
||||
assert response.status_code == 200, response.data
|
||||
assert [int(item["external_id"]) for item in response.data["data"]] == expected
|
||||
|
||||
|
||||
def test_backfill_batches_repeated_organizations_without_per_record_queries(client):
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
first = record(
|
||||
"government_procurements",
|
||||
"procurements_44fz",
|
||||
1,
|
||||
payload={"is_branch": False},
|
||||
is_branch=False,
|
||||
)
|
||||
OrganizationSourceRecord.objects.bulk_create(
|
||||
[
|
||||
OrganizationSourceRecord(
|
||||
extension=first.extension,
|
||||
source=first.source,
|
||||
record_type=first.record_type,
|
||||
external_id=str(index),
|
||||
title="Existing",
|
||||
payload={"is_branch": True},
|
||||
)
|
||||
for index in range(2, 1002)
|
||||
]
|
||||
)
|
||||
before = list(
|
||||
OrganizationSourceRecord.objects.order_by("uid").values_list(
|
||||
"uid", "updated_at"
|
||||
)
|
||||
)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
output = StringIO()
|
||||
call_command("normalize_testing_amendments", apply=True, stdout=output)
|
||||
assert len(queries) < 30
|
||||
assert json.loads(output.getvalue())["scanned"] == 1001
|
||||
assert not OrganizationSourceRecord.objects.filter(payload__is_branch=True).exists()
|
||||
assert (
|
||||
list(
|
||||
OrganizationSourceRecord.objects.order_by("uid").values_list(
|
||||
"uid", "updated_at"
|
||||
)
|
||||
)
|
||||
== before
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user