feat: complete published registry contracts and gated SRO ingestion

tests/apps/parsers/test_registry_payload_contract.py

@@ -0,0 +1,727 @@
+"""Source-first payload contracts; all upstream input is local synthetic evidence."""
+
+import json
+from copy import deepcopy
+from io import StringIO
+from tempfile import TemporaryDirectory
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+from apps.parsers.budget_registry import refresh_budget_registry
+from apps.parsers.models import ParserSourceArtifact
+from apps.parsers.sme_support import import_sme_support, support_payload
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.test import TestCase, override_settings
+from django.utils import timezone
+from lxml import etree
+from organizations.budget_payload import budget_list_payload, normalize_budget_payload
+from organizations.models import Organization, OrganizationSourceRecord
+from organizations.registry_payload_serializers import (
+    BudgetRegistryRecordDetailPayloadSerializer,
+    BudgetRegistryRecordListPayloadSerializer,
+    RegistryCodeNameSerializer,
+    SmeSupportRecordDetailPayloadSerializer,
+    SmeSupportRecordListPayloadSerializer,
+)
+from organizations.registry_payload_values import registry_boolean, registry_date
+from organizations.registry_payloads import (
+    build_registry_payload_context,
+    serialize_registry_payload,
+)
+from organizations.sme_payload import (
+    VIOLATION_LABELS,
+    normalize_sme_payload,
+    sme_list_payload,
+)
+
+from tests.apps.parsers.test_registry_snapshots import _archive, _budget_record, _xml
+
+
+def full_budget_fixture():
+    value = _budget_record()
+    value["info"].update(
+        {
+            "orgTypeCode": "03",
+            "orgTypeName": "Учреждение",
+            "isObosob": "false",
+            "isOGV": "0",
+            "isUch": "1",
+            "isReorg": "unobserved",
+            "inclusionDate": "2025-11-13 18:44:39.0",
+            "regDate": "30.12.2017",
+            "regionCode": "01",
+            "regionName": "Регион",
+            "regNum": "00017",
+            "newInfoField": "raw-kept",
+        }
+    )
+    value.update(
+        {
+            "heads": [
+                {
+                    "fio": "Руководитель",
+                    "post": "Директор",
+                    "headMain": "1",
+                    "docDate": "2024-01-02",
+                }
+            ],
+            "contacts": [
+                {
+                    "phone": "123",
+                    "mail": "fixture@example.test",
+                    "site": "https://example.test",
+                }
+            ],
+            "activities": [
+                {
+                    "activityCode": "01.01",
+                    "activityName": "Деятельность",
+                    "activityKind": "основной",
+                }
+            ],
+            "authorities": [
+                {
+                    "authorityCode": "001",
+                    "authorityName": "Орган",
+                    "permissions": [
+                        {"permissionCode": "002", "permissionName": "Полномочие"}
+                    ],
+                }
+            ],
+            "participantPermissions": [
+                {"code": "001", "name": "Участник", "startDate": "2024-01-02"}
+            ],
+            "nonParticipantPermissions": [
+                {
+                    "code": "002",
+                    "registryNum": "00077",
+                    "authBudgCode": "01",
+                    "authBudgName": "Бюджет",
+                }
+            ],
+            "procurementPermissions": [{"code": "201", "name": "заказчик"}],
+            "acceptAuths": [
+                {"authRegNum": "009", "authGiverCode": "007", "authGiverName": "Орган"}
+            ],
+            "transfauth": [
+                {
+                    "authregnum": "004",
+                    "authfomunicipalcode": "001",
+                    "authfomunicipalname": "Район",
+                }
+            ],
+            "ubptransfauthbp": [
+                {"budgetnsicode": "001", "budgetnsiname": "Бюджет", "codebk": "009"}
+            ],
+            "facialAccounts": [
+                {
+                    "num": "000123",
+                    "kindCode": "001",
+                    "kindName": "Лицевой",
+                    "createDate": "2020-01-01",
+                    "accountorgcode": "002",
+                    "accountorgfullname": "Организация",
+                }
+            ],
+            "foAccounts": [{"num": "000456", "foCode": "009", "foName": "Орган"}],
+            "ksaccounts": [
+                {
+                    "num": "000789",
+                    "opendate": "2020-02-03",
+                    "opentofkcode": "006",
+                    "opentofkname": "Казначейство",
+                }
+            ],
+            "successions": [
+                {
+                    "numberdoc": "00002",
+                    "documentdate": "2020-01-02",
+                    "parentCode": "001",
+                }
+            ],
+            "contracts": [{"contractnumber": "00003", "signdate": "2024-01-02"}],
+            "attachment": [{"unobserved": {"keep": ["0001", None]}}],
+            "ubpfin": [],
+        }
+    )
+    return value
+
+
+def test_budget_partial_nullable_empty_collections_and_strict_booleans():
+    payload = normalize_budget_payload({"info": {}})
+    assert payload["registry"]["registration_number"] is None
+    assert payload["classification"]["organization_type"] is None
+    assert payload["is_separate_division"] is None
+    assert all(value == [] for value in payload["accounts"].values())
+    assert all(value == [] for value in payload["permissions"].values())
+    assert payload["summary"]["accounts_count"] == 0
+    assert payload["summary"]["has_procurement_permission"] is False
+    assert BudgetRegistryRecordDetailPayloadSerializer(payload).data == payload
+    assert BudgetRegistryRecordListPayloadSerializer(
+        budget_list_payload(payload)
+    ).data == budget_list_payload(payload)
+
+
+def test_full_budget_blocks_strings_lineage_and_list_detail_agreement():
+    raw = full_budget_fixture()
+    before = deepcopy(raw)
+    payload = normalize_budget_payload(raw)
+    assert raw == before
+    assert payload["upstream"] == before
+    assert payload["registry"]["inclusion_date"] == "2025-11-13T18:44:39Z"
+    assert payload["registry"]["registration_date"] == "2017-12-30"
+    assert payload["address"]["full"] == "Регион"
+    assert payload["classification"]["flags"]["is_reorganized"] is None
+    assert payload["accounts"]["personal"][0]["number"] == "000123"
+    assert payload["accounts"]["financial_authority"][0]["number"] == "000456"
+    assert payload["accounts"]["treasury"][0]["number"] == "000789"
+    assert payload["permissions"]["non_participant"][0]["registry_number"] == "00077"
+    assert payload["summary"] == {
+        "activities_count": 1,
+        "authorities_count": 1,
+        "permissions_count": 6,
+        "accounts_count": 3,
+        "successions_count": 1,
+        "has_procurement_permission": True,
+    }
+    assert payload["unclassified_blocks"]["attachment"] == raw["attachment"]
+    assert payload["unclassified_blocks"]["info"] == [{"newInfoField": "raw-kept"}]
+    assert payload["schema_drift"]["unclassified_blocks_count"] == 3
+    assert {item["json_path"] for item in payload["schema_drift"]["lineage"]} == {
+        "$.attachment",
+        "$.info",
+        "$.unknown_new_block",
+    }
+    assert BudgetRegistryRecordDetailPayloadSerializer(payload).data == payload
+    listing = budget_list_payload(payload)
+    assert "upstream" not in listing and "accounts" not in listing
+    assert listing["registry"] == {
+        key: payload["registry"][key] for key in listing["registry"]
+    }
+    for name, field in BudgetRegistryRecordDetailPayloadSerializer().fields.items():
+        assert field.required, name
+
+
+def test_large_budget_detail_keeps_every_collection_item():
+    raw = full_budget_fixture()
+    raw["successions"] = [{"numberdoc": f"{i:08d}"} for i in range(2000)]
+    payload = normalize_budget_payload(raw)
+    assert (
+        len(payload["successions"]) == payload["summary"]["successions_count"] == 2000
+    )
+    assert payload["successions"][-1]["document_number"] == "00001999"
+    assert "successions" not in budget_list_payload(payload)
+
+
+def test_incomplete_budget_dictionary_is_null_with_raw_lineage_and_metric():
+    raw = {"info": {"orgTypeCode": "003", "budgetLvlName": "Бюджет"}}
+    payload = normalize_budget_payload(raw)
+    assert payload["classification"]["organization_type"] is None
+    assert payload["budget"]["level"] is None
+    assert payload["upstream"] == raw
+    assert set(payload["schema_drift"]["incomplete_dictionaries"]) == {
+        "$.info.orgTypeCode",
+        "$.info.budgetLvlCode",
+    }
+    assert not RegistryCodeNameSerializer(data={"code": "003", "name": None}).is_valid()
+    assert RegistryCodeNameSerializer(
+        data={"code": "003", "name": "Учреждение"}
+    ).is_valid()
+    record = SimpleNamespace(
+        source="budget_ubpandnubp",
+        payload={"classification": {"organization_type": {"code": "003", "name": ""}}},
+    )
+    assert (
+        serialize_registry_payload(record, detail=False)["classification"][
+            "organization_type"
+        ]
+        is None
+    )
+
+
+def test_partial_authority_permission_retains_raw_without_invalid_dictionary():
+    raw = {
+        "info": {"code": "001"},
+        "authorities": [
+            {
+                "authorityCode": "001",
+                "authorityName": "Орган",
+                "permissions": [
+                    {"permissionCode": "0002"},
+                    {"permissionCode": "0003", "permissionName": "Полномочие"},
+                ],
+            }
+        ],
+    }
+    payload = normalize_budget_payload(raw)
+    assert payload["authorities"][0]["permissions"] == [
+        {"code": "0003", "name": "Полномочие"}
+    ]
+    assert payload["unclassified_blocks"]["authorities"] == raw["authorities"]
+    assert payload["upstream"] == raw
+    assert payload["schema_drift"]["incomplete_dictionaries"] == [
+        "$.authorities[0].permissions[0]"
+    ]
+    serializer = BudgetRegistryRecordDetailPayloadSerializer(data=payload)
+    assert serializer.is_valid(), serializer.errors
+
+
+def test_partial_budget_without_raw_preserves_existing_list_and_detail_values():
+    partial = {
+        "registry": {"code": "0001", "registration_number": "0002"},
+        "is_separate_division": True,
+        "summary": {"activities_count": 7, "has_procurement_permission": True},
+        "contacts": [{"phone": "123", "email": None, "website": None}],
+        "classification": {"organization_type": {"code": "03", "name": "Учреждение"}},
+        "unclassified_blocks": {"retained": [{"code": "001"}]},
+    }
+    record = SimpleNamespace(source="budget_ubpandnubp", payload=partial)
+    before = deepcopy(partial)
+    detail = serialize_registry_payload(record, detail=True)
+    assert detail["is_separate_division"] is True
+    assert detail["summary"]["activities_count"] == 7
+    assert detail["summary"]["has_procurement_permission"] is True
+    assert detail["summary"]["accounts_count"] == 0
+    assert detail["contacts"] == partial["contacts"]
+    assert detail["unclassified_blocks"] == partial["unclassified_blocks"]
+    assert detail["registry"]["registration_number"] == "0002"
+    assert partial == before
+    assert serialize_registry_payload(record, detail=False) == budget_list_payload(
+        detail
+    )
+
+
+def test_partial_budget_raw_falls_back_only_for_missing_upstream_fields():
+    raw = {"id": "source-id", "info": {"statusCode": "2"}}
+    payload = {
+        "registry": {"code": "0001", "registration_number": "0002"},
+        "budget": {"name": "Известный бюджет"},
+        "address": {"region": {"code": "77", "name": "Москва"}},
+        "classification": {"organization_type": {"code": "03", "name": "Учреждение"}},
+        "upstream": raw,
+    }
+    record = SimpleNamespace(source="budget_ubpandnubp", payload=payload)
+    original = deepcopy(payload)
+    detail = serialize_registry_payload(record, detail=True)
+    assert detail["registry"]["code"] == "0001"
+    assert detail["registry"]["registration_number"] == "0002"
+    assert detail["budget"]["name"] == "Известный бюджет"
+    assert detail["address"]["region"] == {"code": "77", "name": "Москва"}
+    assert detail["upstream"] == raw and record.payload == original
+    raw["info"].update(regNum=None, budgetName="", regionCode="78", orgTypeName=None)
+    detail = serialize_registry_payload(record, detail=True)
+    assert detail["registry"]["registration_number"] is None
+    assert detail["budget"]["name"] is None
+    assert detail["address"]["region"] is None
+    assert detail["classification"]["organization_type"] is None
+    assert serialize_registry_payload(record, detail=False) == budget_list_payload(
+        detail
+    )
+
+
+def test_budget_registry_code_is_required_and_never_substituted_from_id():
+    from apps.parsers.budget_registry import normalize_budget_record
+    from apps.parsers.registry_snapshots import SnapshotValidationError
+
+    raw = full_budget_fixture()
+    for missing in (None, "", "—"):
+        raw["info"]["code"] = missing
+        with pytest.raises(
+            SnapshotValidationError, match="missing_budget_registry_code"
+        ):
+            normalize_budget_record(raw)
+        serializer = BudgetRegistryRecordListPayloadSerializer(
+            data=budget_list_payload(normalize_budget_payload(raw))
+        )
+        assert not serializer.is_valid()
+        assert "code" in serializer.errors["registry"]
+
+
+def test_named_payload_openapi_schemas_have_required_typed_components():
+    from drf_yasg import openapi
+    from drf_yasg.inspectors import SwaggerAutoSchema
+    from rest_framework.views import APIView
+
+    components = openapi.ReferenceResolver(openapi.SCHEMA_DEFINITIONS, force_init=True)
+    inspector = SwaggerAutoSchema(APIView(), "/fixture/", "GET", components, None, {})
+    for schema in (
+        BudgetRegistryRecordDetailPayloadSerializer,
+        SmeSupportRecordDetailPayloadSerializer,
+    ):
+        inspector.serializer_to_schema(schema())
+    budget = components.get(
+        "BudgetRegistryRecordDetailPayload", scope=openapi.SCHEMA_DEFINITIONS
+    )
+    sme = components.get(
+        "SmeSupportRecordDetailPayload", scope=openapi.SCHEMA_DEFINITIONS
+    )
+    assert {
+        "registry",
+        "accounts",
+        "summary",
+        "permissions",
+        "unclassified_blocks",
+    } <= set(budget["required"])
+    assert {"support_sizes", "provider", "violations", "provenance"} <= set(
+        sme["required"]
+    )
+    assert budget["properties"]["accounts"]["$ref"].endswith("/BudgetAccounts")
+    assert sme["properties"]["violations"]["items"]["$ref"].endswith(
+        "/SmeSupportViolation"
+    )
+    assert budget["properties"]["upstream"]["additionalProperties"] is True
+    assert (
+        budget["properties"]["unclassified_blocks"]["additionalProperties"]["items"][
+            "additionalProperties"
+        ]
+        is True
+    )
+    assert sme["properties"]["upstream"]["additionalProperties"] is True
+    unclassified = components.get(
+        "BudgetUnclassifiedItem", scope=openapi.SCHEMA_DEFINITIONS
+    )
+    assert unclassified["additionalProperties"] is True
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("0", False),
+        ("1", True),
+        ("FALSE", False),
+        ("true", True),
+        (None, None),
+        ("unknown", None),
+    ],
+)
+def test_registry_boolean_is_strict(raw, expected):
+    assert registry_boolean(raw) is expected
+
+
+def test_registry_timestamps_use_explicit_utc_application_rule_and_preserve_dates():
+    assert (
+        registry_date("2025-01-02 03:04:05.0", timestamp=True) == "2025-01-02T03:04:05Z"
+    )
+    assert (
+        registry_date("2025-01-02T03:04:05+03:00", timestamp=True)
+        == "2025-01-02T00:04:05Z"
+    )
+    assert registry_date("2025-01-02", timestamp=True) == "2025-01-02T00:00:00Z"
+    assert registry_date("2025-01-02T03:04:05+03:00") == "2025-01-02"
+    with pytest.raises(ValueError):
+        registry_date("2025-02-30")
+
+
+def test_sme_mixed_sizes_documents_violations_and_read_compatibility():
+    root = etree.fromstring(_xml(units=("1", "2", "3", "4", "5")))  # noqa: S320 -- local fixture only
+    measure = root.find("Документ/СвПредПод")
+    measure.find("Нарушения").set("СрокНаруш", "01.04.2025")
+    payload, amount = support_payload("doc", measure, "2026-08-15")
+    assert amount == "2.50"
+    assert [size["unit"] for size in payload["support_sizes"]] == [
+        "RUB",
+        "square_meter",
+        "hour",
+        "percent",
+        "unit",
+    ]
+    assert payload["recipient_type"] == {"code": "1", "name": "Юридическое лицо"}
+    assert payload["sme_category"] == {"code": "1", "name": "Микропредприятие"}
+    assert payload["violations"][0] == {
+        "type": {"code": "1", "name": VIOLATION_LABELS["1"]},
+        "recognized_date": "2025-02-01",
+        "remedy_deadline": "2025-04-01",
+        "remedied_date": None,
+    }
+    assert [doc["number"] for doc in payload["regulatory_documents"]] == ["A", "B"]
+    assert payload["regulatory_documents"][0]["date"] is None
+    assert (
+        payload["violation_count"] == 1 and payload["regulatory_documents_count"] == 2
+    )
+    assert payload["upstream"]["violations"][0]["ВидНаруш"] == "1"
+    assert SmeSupportRecordDetailPayloadSerializer(payload).data == payload
+    assert SmeSupportRecordListPayloadSerializer(
+        sme_list_payload(payload)
+    ).data == sme_list_payload(payload)
+    assert normalize_sme_payload(payload) == payload
+
+
+@pytest.mark.parametrize(
+    "recipient,category,violation",
+    [("1", "1", "1"), ("2", "2", "2"), ("3", "3", "1"), ("4", "4", "2")],
+)
+def test_official_fns_dictionary_codes_have_names(recipient, category, violation):
+    root = etree.fromstring(_xml())  # noqa: S320 -- local fixture only
+    measure = root.find("Документ/СвПредПод")
+    measure.set("ВидПП", recipient)
+    measure.set("КатСуб", category)
+    measure.find("Нарушения").set("ВидНаруш", violation)
+    payload, _ = support_payload("doc", measure, "2026-08-15")
+    for name in ("recipient_type", "sme_category"):
+        assert payload[name]["name"] and payload[name]["name"] != payload[name]["code"]
+    if recipient == "3":
+        assert "Налог на профессиональный доход" in payload["recipient_type"]["name"]
+    if recipient == "4":
+        assert "Глава крестьянского" in payload["recipient_type"]["name"]
+        assert payload["sme_category"]["name"] == "Отсутствует"
+    assert payload["violations"][0]["type"]["name"] == VIOLATION_LABELS[violation]
+
+
+def test_unknown_sme_dictionary_is_not_published_with_invented_label():
+    root = etree.fromstring(_xml())  # noqa: S320 -- local fixture only
+    measure = root.find("Документ/СвПредПод")
+    measure.set("КатСуб", "9")
+    with pytest.raises(ValueError, match="unknown_sme_dictionary_code"):
+        support_payload("doc", measure, "2026-08-15")
+
+
+def test_sme_compatibility_normalizes_optional_regions_and_official_labels():
+    root = etree.fromstring(_xml())  # noqa: S320 -- local fixture only
+    payload, _ = support_payload("doc", root.find("Документ/СвПредПод"), "2026-08-15")
+    payload["region"] = {"code": "01", "name": None}
+    payload["provider"]["region"] = {"code": None, "name": "Регион"}
+    payload["recipient_type"]["name"] = "Устаревшее имя"
+    normalized = normalize_sme_payload(payload)
+    assert normalized["region"] is None
+    assert normalized["provider"]["region"] is None
+    assert normalized["recipient_type"]["name"] == "Юридическое лицо"
+    serializer = SmeSupportRecordDetailPayloadSerializer(data=normalized)
+    assert serializer.is_valid(), serializer.errors
+    payload["recipient_type"] = {"code": "9", "name": "Неофициальное значение"}
+    with pytest.raises(ValueError, match="unknown_sme_dictionary_code"):
+        normalize_sme_payload(payload)
+
+
+class RegistryPayloadContextTest(TestCase):
+    def setUp(self):
+        self.storage = TemporaryDirectory(prefix="registry-payload-test-")
+        self.addCleanup(self.storage.cleanup)
+        self.settings_override = override_settings(MEDIA_ROOT=self.storage.name)
+        self.settings_override.enable()
+        self.addCleanup(self.settings_override.disable)
+        Organization.objects.create(
+            name="Получатель",
+            inn="1234567890",
+            ogrn="1027700132195",
+            okpo="00123456",
+            directory_imported_at=timezone.now(),
+        )
+        self.provider = Organization.objects.create(
+            name="Каноническое имя",
+            inn="9876543210",
+            ogrn="1027700132195",
+            okpo="00012345",
+            directory_imported_at=timezone.now(),
+        )
+        self.artifact, _ = import_sme_support(
+            handle=_archive(_xml()),
+            original_name="fixture.zip",
+            snapshot_date="2026-08-15",
+            load_batch=1,
+        )
+
+    def test_page_context_is_bounded_and_preserves_upstream_provider(self):
+        records = list(
+            OrganizationSourceRecord.objects.select_related("extension__organization")
+        )
+        with self.assertNumQueries(2):
+            context = build_registry_payload_context(records * 25)
+        with self.assertNumQueries(0):
+            for record in records:
+                data = serialize_registry_payload(record, detail=True, context=context)
+                self.assertEqual(data["provider"]["name"], "Поставщик")
+                self.assertEqual(
+                    data["provider"]["organization_uid"], str(self.provider.uid)
+                )
+                self.assertEqual(data["provider"]["okpo"], "00012345")
+                self.assertIsNone(data["region"])
+                self.assertEqual(
+                    data["provenance"]["archive_sha256"], self.artifact.sha256
+                )
+                self.assertEqual(data, record.payload)
+
+    def test_old_sme_payload_and_budget_raw_get_current_shape_without_write(self):
+        record = OrganizationSourceRecord.objects.first()
+        old = deepcopy(record.payload)
+        old["recipient_type"] = "1"
+        old["sme_category"] = "1"
+        old["violations"] = deepcopy(old["upstream"]["violations"])
+        old["regulatory_documents"] = deepcopy(old["upstream"]["regulatory_documents"])
+        record.payload = old
+        context = build_registry_payload_context([record])
+        with self.assertNumQueries(0):
+            normalized = serialize_registry_payload(
+                record, detail=True, context=context
+            )
+        self.assertIsInstance(normalized["recipient_type"], dict)
+        self.assertEqual(record.payload, old)
+        raw = full_budget_fixture()
+        old_budget = SimpleNamespace(
+            source="budget_ubpandnubp", payload={"upstream": raw}
+        )
+        with self.assertNumQueries(0):
+            self.assertEqual(
+                serialize_registry_payload(old_budget, detail=True),
+                normalize_budget_payload(raw),
+            )
+
+    def legacy_records(self):
+        records = list(OrganizationSourceRecord.objects.order_by("uid"))
+        for record in records:
+            record.payload["recipient_type"] = "1"
+            record.payload["sme_category"] = "1"
+            record.status = ""
+            record.save(update_fields=["payload", "status"])
+        return records
+
+    def test_backfill_dry_run_default_then_apply_is_atomic_and_idempotent(self):
+        records = self.legacy_records()
+        before = {
+            record.uid: (deepcopy(record.payload), record.status, record.updated_at)
+            for record in records
+        }
+        command_module = (
+            "organizations.management.commands.normalize_published_registry_payloads"
+        )
+        output = StringIO()
+        with patch(f"{command_module}.invalidate_source_data_cache") as invalidate:
+            dry = json.loads(
+                call_command(
+                    "normalize_published_registry_payloads", stdout=output, silent=True
+                )
+            )
+            assert dry["dry_run"] and dry["changed"] == 2
+            invalidate.assert_not_called()
+            for record in records:
+                record.refresh_from_db()
+                assert (record.payload, record.status, record.updated_at) == before[
+                    record.uid
+                ]
+            with self.captureOnCommitCallbacks(execute=True):
+                applied = json.loads(
+                    call_command(
+                        "normalize_published_registry_payloads",
+                        apply=True,
+                        stdout=output,
+                        silent=True,
+                    )
+                )
+            assert not applied["dry_run"] and applied["changed"] == 2
+            assert applied["exports_rebuild_required"]
+            assert invalidate.call_count == 2
+        for record in records:
+            record.refresh_from_db()
+            assert record.status == "published"
+            assert record.payload["sme_category"]["code"] == "1"
+            assert record.updated_at == before[record.uid][2]
+        repeated = json.loads(
+            call_command(
+                "normalize_published_registry_payloads",
+                apply=True,
+                stdout=output,
+                silent=True,
+            )
+        )
+        assert repeated["changed"] == 0
+
+    def test_backfill_late_invalid_record_rolls_back_earlier_batch(self):
+        records = self.legacy_records()
+        records[-1].payload["support_sizes"] = []
+        records[-1].save(update_fields=["payload"])
+        with pytest.raises(CommandError):
+            call_command(
+                "normalize_published_registry_payloads",
+                apply=True,
+                batch_size=1,
+                stdout=StringIO(),
+                stderr=StringIO(),
+                silent=True,
+            )
+        records[0].refresh_from_db()
+        assert records[0].status == "" and records[0].payload["sme_category"] == "1"
+
+    def test_backfill_cache_failure_rolls_back_payload_and_status(self):
+        records = self.legacy_records()
+        with patch(
+            "organizations.management.commands.normalize_published_registry_payloads.invalidate_source_data_cache",
+            side_effect=RuntimeError("cache unavailable"),
+        ), pytest.raises(CommandError):
+            call_command(
+                "normalize_published_registry_payloads",
+                apply=True,
+                stdout=StringIO(),
+                stderr=StringIO(),
+                silent=True,
+            )
+        for record in records:
+            record.refresh_from_db()
+            assert record.status == "" and record.payload["sme_category"] == "1"
+
+    def test_post_commit_cache_failure_reports_committed_state(self):
+        records = self.legacy_records()
+        module = (
+            "organizations.management.commands.normalize_published_registry_payloads"
+        )
+        with patch(
+            f"{module}.invalidate_source_data_cache",
+            side_effect=[None, RuntimeError("cache unavailable")],
+        ), patch(
+            f"{module}.transaction.on_commit", side_effect=lambda callback: callback()
+        ), pytest.raises(CommandError, match="updates committed"):
+            call_command(
+                "normalize_published_registry_payloads",
+                apply=True,
+                stdout=StringIO(),
+                stderr=StringIO(),
+                silent=True,
+            )
+        for record in records:
+            record.refresh_from_db()
+            assert record.status == "published"
+
+    def test_budget_import_persists_unknown_block_metric_and_page_version(self):
+        raw = full_budget_fixture()
+        raw["info"]["okpoCode"] = "00123456"
+        page = {"data": [raw], "recordCount": 1, "pageNum": 1, "version": "10"}
+        with patch("apps.parsers.budget_registry.budget_page", return_value=page):
+            artifact, _ = refresh_budget_registry(load_batch=2, session=object())
+        assert artifact.metadata["schema_drift_records_count"] == 1
+        assert artifact.metadata["schema_drift_blocks"] == {
+            "attachment": 1,
+            "info": 1,
+            "unknown_new_block": 1,
+        }
+        record = OrganizationSourceRecord.objects.get(source="budget_ubpandnubp")
+        assert record.payload["upstream_audit"]["source_version"] == "10"
+        with self.assertNumQueries(1):
+            context = build_registry_payload_context([record])
+        with self.assertNumQueries(0):
+            assert (
+                serialize_registry_payload(record, detail=True, context=context)
+                == record.payload
+            )
+
+    def test_budget_missing_required_code_keeps_last_published_snapshot(self):
+        from apps.parsers.registry_snapshots import SnapshotValidationError
+
+        raw = full_budget_fixture()
+        raw["info"]["okpoCode"] = "00123456"
+        page = {"data": [raw], "recordCount": 1, "pageNum": 1, "version": "10"}
+        with patch("apps.parsers.budget_registry.budget_page", return_value=page):
+            refresh_budget_registry(load_batch=2, session=object())
+        before = OrganizationSourceRecord.objects.get(source="budget_ubpandnubp")
+        del raw["info"]["code"]
+        with patch(
+            "apps.parsers.budget_registry.budget_page", return_value=page
+        ), pytest.raises(SnapshotValidationError, match="missing_budget_registry_code"):
+            refresh_budget_registry(load_batch=3, session=object())
+        current = OrganizationSourceRecord.objects.get(source="budget_ubpandnubp")
+        assert (current.uid, current.payload) == (before.uid, before.payload)
+        assert (
+            ParserSourceArtifact.objects.get(
+                source="budget_ubpandnubp", load_batch=3
+            ).status
+            == "rejected"
+        )