perf: batch acceptance backfill for deployed source volume
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
@@ -88,7 +88,9 @@ python src/manage.py build_source_record_exports
|
|||||||
```
|
```
|
||||||
|
|
||||||
Первый вызов нормализации — dry run с количествами предполагаемых изменений.
|
Первый вызов нормализации — dry run с количествами предполагаемых изменений.
|
||||||
Перед `--apply` проверить warnings и правило конфликтов. Apply не меняет uid,
|
Перед `--apply` проверить warnings и правило конфликтов. Нормализация пропускает записи без затронутых полей при уже известном филиале,
|
||||||
|
читает и блокирует каждую организацию один раз и обновляет payload пакетами.
|
||||||
|
Apply не меняет uid,
|
||||||
created_at, updated_at записей и raw; увеличивает revision затронутых published
|
created_at, updated_at записей и raw; увеличивает revision затронутых published
|
||||||
артефактов и инвалидирует cache. Для legacy sources без artifact revision нет:
|
артефактов и инвалидирует cache. Для legacy sources без artifact revision нет:
|
||||||
выдача tickets должна оставаться остановленной до обязательной пересборки exports.
|
выдача tickets должна оставаться остановленной до обязательной пересборки exports.
|
||||||
@@ -119,3 +121,7 @@ created_at, updated_at записей и raw; увеличивает revision з
|
|||||||
|
|
||||||
Это локальные backend/contract проверки на тестовых данных. Выкладка, применение
|
Это локальные backend/contract проверки на тестовых данных. Выкладка, применение
|
||||||
backfill к рабочей БД и приёмка через рабочий frontend не выполнялись.
|
backfill к рабочей БД и приёмка через рабочий frontend не выполнялись.
|
||||||
|
|
||||||
|
При подготовке выкладки на dev обнаружено 3,86 млн записей. Добавлена проверка
|
||||||
|
пакетной нормализации 1001 записи одной организации с бюджетом менее 30 SQL
|
||||||
|
запросов; весь `test_testing_amendments.py` после оптимизации: **32 passed**.
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import json
|
|||||||
from apps.parsers.models import ParserSourceArtifact
|
from apps.parsers.models import ParserSourceArtifact
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
|
from django.db.models import Q
|
||||||
|
|
||||||
from organizations.models import Organization, OrganizationSourceRecord
|
from organizations.models import Organization, OrganizationSourceRecord
|
||||||
from organizations.source_cache import invalidate_source_data_cache
|
from organizations.source_cache import invalidate_source_data_cache
|
||||||
@@ -31,10 +32,19 @@ class Command(BaseCommand):
|
|||||||
}
|
}
|
||||||
sources = set()
|
sources = set()
|
||||||
simulated_branches = {}
|
simulated_branches = {}
|
||||||
|
organizations = {}
|
||||||
with transaction.atomic():
|
with transaction.atomic():
|
||||||
queryset = OrganizationSourceRecord.objects.filter(
|
queryset = (
|
||||||
extension__source_group__in=BRANCH_GROUPS
|
OrganizationSourceRecord.objects.filter(
|
||||||
).order_by("uid")
|
extension__source_group__in=BRANCH_GROUPS
|
||||||
|
)
|
||||||
|
.filter(
|
||||||
|
Q(source__in=("industrial", "fstec"))
|
||||||
|
| Q(payload__has_any_keys=("is_branch", "filial"))
|
||||||
|
| Q(extension__organization__is_branch__isnull=True)
|
||||||
|
)
|
||||||
|
.order_by("uid")
|
||||||
|
)
|
||||||
if apply:
|
if apply:
|
||||||
queryset = queryset.select_for_update()
|
queryset = queryset.select_for_update()
|
||||||
# Keyset batches keep memory bounded and avoid server-side cursor writes.
|
# Keyset batches keep memory bounded and avoid server-side cursor writes.
|
||||||
@@ -43,14 +53,33 @@ class Command(BaseCommand):
|
|||||||
page = (
|
page = (
|
||||||
queryset if last_uid is None else queryset.filter(uid__gt=last_uid)
|
queryset if last_uid is None else queryset.filter(uid__gt=last_uid)
|
||||||
)
|
)
|
||||||
records = list(
|
records = list(page.select_related("extension")[:500])
|
||||||
page.select_related("extension", "extension__organization")[:500]
|
|
||||||
)
|
|
||||||
if not records:
|
if not records:
|
||||||
break
|
break
|
||||||
|
missing = {
|
||||||
|
record.extension.organization_id for record in records
|
||||||
|
} - organizations.keys()
|
||||||
|
organization_query = Organization.objects.filter(pk__in=missing)
|
||||||
|
if apply:
|
||||||
|
organization_query = organization_query.select_for_update()
|
||||||
|
organizations.update(
|
||||||
|
(organization.pk, organization)
|
||||||
|
for organization in organization_query.order_by("pk")
|
||||||
|
)
|
||||||
|
changed = []
|
||||||
for record in records:
|
for record in records:
|
||||||
self._normalize_record(
|
if self._normalize_record(
|
||||||
record, apply, result, sources, simulated_branches
|
record,
|
||||||
|
apply,
|
||||||
|
result,
|
||||||
|
sources,
|
||||||
|
simulated_branches,
|
||||||
|
organizations[record.extension.organization_id],
|
||||||
|
):
|
||||||
|
changed.append(record)
|
||||||
|
if apply and changed:
|
||||||
|
OrganizationSourceRecord.objects.bulk_update(
|
||||||
|
changed, ["payload"], batch_size=500
|
||||||
)
|
)
|
||||||
last_uid = records[-1].uid
|
last_uid = records[-1].uid
|
||||||
if apply and sources:
|
if apply and sources:
|
||||||
@@ -69,14 +98,11 @@ class Command(BaseCommand):
|
|||||||
return json.dumps(result, sort_keys=True)
|
return json.dumps(result, sort_keys=True)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _normalize_record(record, apply, result, sources, simulated_branches):
|
def _normalize_record(
|
||||||
|
record, apply, result, sources, simulated_branches, organization
|
||||||
|
):
|
||||||
result["scanned"] += 1
|
result["scanned"] += 1
|
||||||
organization = record.extension.organization
|
if not apply and organization.pk in simulated_branches:
|
||||||
if apply:
|
|
||||||
organization = Organization.objects.select_for_update().get(
|
|
||||||
pk=organization.pk
|
|
||||||
)
|
|
||||||
elif organization.pk in simulated_branches:
|
|
||||||
organization.is_branch = simulated_branches[organization.pk]
|
organization.is_branch = simulated_branches[organization.pk]
|
||||||
previous = organization.is_branch
|
previous = organization.is_branch
|
||||||
payload = normalize_record_payload(
|
payload = normalize_record_payload(
|
||||||
@@ -104,7 +130,6 @@ class Command(BaseCommand):
|
|||||||
if payload != record.payload:
|
if payload != record.payload:
|
||||||
result["changed_records"] += 1
|
result["changed_records"] += 1
|
||||||
sources.add(record.source)
|
sources.add(record.source)
|
||||||
if apply:
|
record.payload = payload
|
||||||
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
|
return True
|
||||||
payload=payload
|
return False
|
||||||
)
|
|
||||||
|
|||||||
@@ -385,3 +385,48 @@ def test_amount_ordering_remains_decimal_null_last(
|
|||||||
response = client.get(LIST_URL, {"source_group": group, "ordering": ordering})
|
response = client.get(LIST_URL, {"source_group": group, "ordering": ordering})
|
||||||
assert response.status_code == 200, response.data
|
assert response.status_code == 200, response.data
|
||||||
assert [int(item["external_id"]) for item in response.data["data"]] == expected
|
assert [int(item["external_id"]) for item in response.data["data"]] == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_batches_repeated_organizations_without_per_record_queries(client):
|
||||||
|
from django.db import connection
|
||||||
|
from django.test.utils import CaptureQueriesContext
|
||||||
|
|
||||||
|
first = record(
|
||||||
|
"government_procurements",
|
||||||
|
"procurements_44fz",
|
||||||
|
1,
|
||||||
|
payload={"is_branch": False},
|
||||||
|
is_branch=False,
|
||||||
|
)
|
||||||
|
OrganizationSourceRecord.objects.bulk_create(
|
||||||
|
[
|
||||||
|
OrganizationSourceRecord(
|
||||||
|
extension=first.extension,
|
||||||
|
source=first.source,
|
||||||
|
record_type=first.record_type,
|
||||||
|
external_id=str(index),
|
||||||
|
title="Existing",
|
||||||
|
payload={"is_branch": True},
|
||||||
|
)
|
||||||
|
for index in range(2, 1002)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
before = list(
|
||||||
|
OrganizationSourceRecord.objects.order_by("uid").values_list(
|
||||||
|
"uid", "updated_at"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
with CaptureQueriesContext(connection) as queries:
|
||||||
|
output = StringIO()
|
||||||
|
call_command("normalize_testing_amendments", apply=True, stdout=output)
|
||||||
|
assert len(queries) < 30
|
||||||
|
assert json.loads(output.getvalue())["scanned"] == 1001
|
||||||
|
assert not OrganizationSourceRecord.objects.filter(payload__is_branch=True).exists()
|
||||||
|
assert (
|
||||||
|
list(
|
||||||
|
OrganizationSourceRecord.objects.order_by("uid").values_list(
|
||||||
|
"uid", "updated_at"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
== before
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user