diff --git a/docs/backend-testing-amendments-2026-10.md b/docs/backend-testing-amendments-2026-10.md index 3c672d0..22b7c7e 100644 --- a/docs/backend-testing-amendments-2026-10.md +++ b/docs/backend-testing-amendments-2026-10.md @@ -88,7 +88,9 @@ python src/manage.py build_source_record_exports ``` Первый вызов нормализации — dry run с количествами предполагаемых изменений. -Перед `--apply` проверить warnings и правило конфликтов. Apply не меняет uid, +Перед `--apply` проверить warnings и правило конфликтов. Нормализация пропускает записи без затронутых полей при уже известном филиале, +читает и блокирует каждую организацию один раз и обновляет payload пакетами. +Apply не меняет uid, created_at, updated_at записей и raw; увеличивает revision затронутых published артефактов и инвалидирует cache. Для legacy sources без artifact revision нет: выдача tickets должна оставаться остановленной до обязательной пересборки exports. @@ -119,3 +121,7 @@ created_at, updated_at записей и raw; увеличивает revision з Это локальные backend/contract проверки на тестовых данных. Выкладка, применение backfill к рабочей БД и приёмка через рабочий frontend не выполнялись. + +При подготовке выкладки на dev обнаружено 3,86 млн записей. Добавлена проверка +пакетной нормализации 1001 записи одной организации с бюджетом менее 30 SQL +запросов; весь `test_testing_amendments.py` после оптимизации: **32 passed**. diff --git a/src/organizations/management/commands/normalize_testing_amendments.py b/src/organizations/management/commands/normalize_testing_amendments.py index aaa4f8b..a02eafc 100644 --- a/src/organizations/management/commands/normalize_testing_amendments.py +++ b/src/organizations/management/commands/normalize_testing_amendments.py @@ -5,6 +5,7 @@ import json from apps.parsers.models import ParserSourceArtifact from django.core.management.base import BaseCommand from django.db import transaction +from django.db.models import Q from organizations.models import Organization, OrganizationSourceRecord from organizations.source_cache import invalidate_source_data_cache @@ -31,10 +32,19 @@ class Command(BaseCommand): } sources = set() simulated_branches = {} + organizations = {} with transaction.atomic(): - queryset = OrganizationSourceRecord.objects.filter( - extension__source_group__in=BRANCH_GROUPS - ).order_by("uid") + queryset = ( + OrganizationSourceRecord.objects.filter( + extension__source_group__in=BRANCH_GROUPS + ) + .filter( + Q(source__in=("industrial", "fstec")) + | Q(payload__has_any_keys=("is_branch", "filial")) + | Q(extension__organization__is_branch__isnull=True) + ) + .order_by("uid") + ) if apply: queryset = queryset.select_for_update() # Keyset batches keep memory bounded and avoid server-side cursor writes. @@ -43,14 +53,33 @@ class Command(BaseCommand): page = ( queryset if last_uid is None else queryset.filter(uid__gt=last_uid) ) - records = list( - page.select_related("extension", "extension__organization")[:500] - ) + records = list(page.select_related("extension")[:500]) if not records: break + missing = { + record.extension.organization_id for record in records + } - organizations.keys() + organization_query = Organization.objects.filter(pk__in=missing) + if apply: + organization_query = organization_query.select_for_update() + organizations.update( + (organization.pk, organization) + for organization in organization_query.order_by("pk") + ) + changed = [] for record in records: - self._normalize_record( - record, apply, result, sources, simulated_branches + if self._normalize_record( + record, + apply, + result, + sources, + simulated_branches, + organizations[record.extension.organization_id], + ): + changed.append(record) + if apply and changed: + OrganizationSourceRecord.objects.bulk_update( + changed, ["payload"], batch_size=500 ) last_uid = records[-1].uid if apply and sources: @@ -69,14 +98,11 @@ class Command(BaseCommand): return json.dumps(result, sort_keys=True) @staticmethod - def _normalize_record(record, apply, result, sources, simulated_branches): + def _normalize_record( + record, apply, result, sources, simulated_branches, organization + ): result["scanned"] += 1 - organization = record.extension.organization - if apply: - organization = Organization.objects.select_for_update().get( - pk=organization.pk - ) - elif organization.pk in simulated_branches: + if not apply and organization.pk in simulated_branches: organization.is_branch = simulated_branches[organization.pk] previous = organization.is_branch payload = normalize_record_payload( @@ -104,7 +130,6 @@ class Command(BaseCommand): if payload != record.payload: result["changed_records"] += 1 sources.add(record.source) - if apply: - OrganizationSourceRecord.objects.filter(pk=record.pk).update( - payload=payload - ) + record.payload = payload + return True + return False diff --git a/tests/apps/organizations/test_testing_amendments.py b/tests/apps/organizations/test_testing_amendments.py index 36fc7a0..d0074ff 100644 --- a/tests/apps/organizations/test_testing_amendments.py +++ b/tests/apps/organizations/test_testing_amendments.py @@ -385,3 +385,48 @@ def test_amount_ordering_remains_decimal_null_last( response = client.get(LIST_URL, {"source_group": group, "ordering": ordering}) assert response.status_code == 200, response.data assert [int(item["external_id"]) for item in response.data["data"]] == expected + + +def test_backfill_batches_repeated_organizations_without_per_record_queries(client): + from django.db import connection + from django.test.utils import CaptureQueriesContext + + first = record( + "government_procurements", + "procurements_44fz", + 1, + payload={"is_branch": False}, + is_branch=False, + ) + OrganizationSourceRecord.objects.bulk_create( + [ + OrganizationSourceRecord( + extension=first.extension, + source=first.source, + record_type=first.record_type, + external_id=str(index), + title="Existing", + payload={"is_branch": True}, + ) + for index in range(2, 1002) + ] + ) + before = list( + OrganizationSourceRecord.objects.order_by("uid").values_list( + "uid", "updated_at" + ) + ) + with CaptureQueriesContext(connection) as queries: + output = StringIO() + call_command("normalize_testing_amendments", apply=True, stdout=output) + assert len(queries) < 30 + assert json.loads(output.getvalue())["scanned"] == 1001 + assert not OrganizationSourceRecord.objects.filter(payload__is_branch=True).exists() + assert ( + list( + OrganizationSourceRecord.objects.order_by("uid").values_list( + "uid", "updated_at" + ) + ) + == before + )