18971d33ec
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s
728 lines
30 KiB
Python
728 lines
30 KiB
Python
"""Source-first payload contracts; all upstream input is local synthetic evidence."""
|
|
|
|
import json
|
|
from copy import deepcopy
|
|
from io import StringIO
|
|
from tempfile import TemporaryDirectory
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from apps.parsers.budget_registry import refresh_budget_registry
|
|
from apps.parsers.models import ParserSourceArtifact
|
|
from apps.parsers.sme_support import import_sme_support, support_payload
|
|
from django.core.management import call_command
|
|
from django.core.management.base import CommandError
|
|
from django.test import TestCase, override_settings
|
|
from django.utils import timezone
|
|
from lxml import etree
|
|
from organizations.budget_payload import budget_list_payload, normalize_budget_payload
|
|
from organizations.models import Organization, OrganizationSourceRecord
|
|
from organizations.registry_payload_serializers import (
|
|
BudgetRegistryRecordDetailPayloadSerializer,
|
|
BudgetRegistryRecordListPayloadSerializer,
|
|
RegistryCodeNameSerializer,
|
|
SmeSupportRecordDetailPayloadSerializer,
|
|
SmeSupportRecordListPayloadSerializer,
|
|
)
|
|
from organizations.registry_payload_values import registry_boolean, registry_date
|
|
from organizations.registry_payloads import (
|
|
build_registry_payload_context,
|
|
serialize_registry_payload,
|
|
)
|
|
from organizations.sme_payload import (
|
|
VIOLATION_LABELS,
|
|
normalize_sme_payload,
|
|
sme_list_payload,
|
|
)
|
|
|
|
from tests.apps.parsers.test_registry_snapshots import _archive, _budget_record, _xml
|
|
|
|
|
|
def full_budget_fixture():
|
|
value = _budget_record()
|
|
value["info"].update(
|
|
{
|
|
"orgTypeCode": "03",
|
|
"orgTypeName": "Учреждение",
|
|
"isObosob": "false",
|
|
"isOGV": "0",
|
|
"isUch": "1",
|
|
"isReorg": "unobserved",
|
|
"inclusionDate": "2025-11-13 18:44:39.0",
|
|
"regDate": "30.12.2017",
|
|
"regionCode": "01",
|
|
"regionName": "Регион",
|
|
"regNum": "00017",
|
|
"newInfoField": "raw-kept",
|
|
}
|
|
)
|
|
value.update(
|
|
{
|
|
"heads": [
|
|
{
|
|
"fio": "Руководитель",
|
|
"post": "Директор",
|
|
"headMain": "1",
|
|
"docDate": "2024-01-02",
|
|
}
|
|
],
|
|
"contacts": [
|
|
{
|
|
"phone": "123",
|
|
"mail": "fixture@example.test",
|
|
"site": "https://example.test",
|
|
}
|
|
],
|
|
"activities": [
|
|
{
|
|
"activityCode": "01.01",
|
|
"activityName": "Деятельность",
|
|
"activityKind": "основной",
|
|
}
|
|
],
|
|
"authorities": [
|
|
{
|
|
"authorityCode": "001",
|
|
"authorityName": "Орган",
|
|
"permissions": [
|
|
{"permissionCode": "002", "permissionName": "Полномочие"}
|
|
],
|
|
}
|
|
],
|
|
"participantPermissions": [
|
|
{"code": "001", "name": "Участник", "startDate": "2024-01-02"}
|
|
],
|
|
"nonParticipantPermissions": [
|
|
{
|
|
"code": "002",
|
|
"registryNum": "00077",
|
|
"authBudgCode": "01",
|
|
"authBudgName": "Бюджет",
|
|
}
|
|
],
|
|
"procurementPermissions": [{"code": "201", "name": "заказчик"}],
|
|
"acceptAuths": [
|
|
{"authRegNum": "009", "authGiverCode": "007", "authGiverName": "Орган"}
|
|
],
|
|
"transfauth": [
|
|
{
|
|
"authregnum": "004",
|
|
"authfomunicipalcode": "001",
|
|
"authfomunicipalname": "Район",
|
|
}
|
|
],
|
|
"ubptransfauthbp": [
|
|
{"budgetnsicode": "001", "budgetnsiname": "Бюджет", "codebk": "009"}
|
|
],
|
|
"facialAccounts": [
|
|
{
|
|
"num": "000123",
|
|
"kindCode": "001",
|
|
"kindName": "Лицевой",
|
|
"createDate": "2020-01-01",
|
|
"accountorgcode": "002",
|
|
"accountorgfullname": "Организация",
|
|
}
|
|
],
|
|
"foAccounts": [{"num": "000456", "foCode": "009", "foName": "Орган"}],
|
|
"ksaccounts": [
|
|
{
|
|
"num": "000789",
|
|
"opendate": "2020-02-03",
|
|
"opentofkcode": "006",
|
|
"opentofkname": "Казначейство",
|
|
}
|
|
],
|
|
"successions": [
|
|
{
|
|
"numberdoc": "00002",
|
|
"documentdate": "2020-01-02",
|
|
"parentCode": "001",
|
|
}
|
|
],
|
|
"contracts": [{"contractnumber": "00003", "signdate": "2024-01-02"}],
|
|
"attachment": [{"unobserved": {"keep": ["0001", None]}}],
|
|
"ubpfin": [],
|
|
}
|
|
)
|
|
return value
|
|
|
|
|
|
def test_budget_partial_nullable_empty_collections_and_strict_booleans():
|
|
payload = normalize_budget_payload({"info": {}})
|
|
assert payload["registry"]["registration_number"] is None
|
|
assert payload["classification"]["organization_type"] is None
|
|
assert payload["is_separate_division"] is None
|
|
assert all(value == [] for value in payload["accounts"].values())
|
|
assert all(value == [] for value in payload["permissions"].values())
|
|
assert payload["summary"]["accounts_count"] == 0
|
|
assert payload["summary"]["has_procurement_permission"] is False
|
|
assert BudgetRegistryRecordDetailPayloadSerializer(payload).data == payload
|
|
assert BudgetRegistryRecordListPayloadSerializer(
|
|
budget_list_payload(payload)
|
|
).data == budget_list_payload(payload)
|
|
|
|
|
|
def test_full_budget_blocks_strings_lineage_and_list_detail_agreement():
|
|
raw = full_budget_fixture()
|
|
before = deepcopy(raw)
|
|
payload = normalize_budget_payload(raw)
|
|
assert raw == before
|
|
assert payload["upstream"] == before
|
|
assert payload["registry"]["inclusion_date"] == "2025-11-13T18:44:39Z"
|
|
assert payload["registry"]["registration_date"] == "2017-12-30"
|
|
assert payload["address"]["full"] == "Регион"
|
|
assert payload["classification"]["flags"]["is_reorganized"] is None
|
|
assert payload["accounts"]["personal"][0]["number"] == "000123"
|
|
assert payload["accounts"]["financial_authority"][0]["number"] == "000456"
|
|
assert payload["accounts"]["treasury"][0]["number"] == "000789"
|
|
assert payload["permissions"]["non_participant"][0]["registry_number"] == "00077"
|
|
assert payload["summary"] == {
|
|
"activities_count": 1,
|
|
"authorities_count": 1,
|
|
"permissions_count": 6,
|
|
"accounts_count": 3,
|
|
"successions_count": 1,
|
|
"has_procurement_permission": True,
|
|
}
|
|
assert payload["unclassified_blocks"]["attachment"] == raw["attachment"]
|
|
assert payload["unclassified_blocks"]["info"] == [{"newInfoField": "raw-kept"}]
|
|
assert payload["schema_drift"]["unclassified_blocks_count"] == 3
|
|
assert {item["json_path"] for item in payload["schema_drift"]["lineage"]} == {
|
|
"$.attachment",
|
|
"$.info",
|
|
"$.unknown_new_block",
|
|
}
|
|
assert BudgetRegistryRecordDetailPayloadSerializer(payload).data == payload
|
|
listing = budget_list_payload(payload)
|
|
assert "upstream" not in listing and "accounts" not in listing
|
|
assert listing["registry"] == {
|
|
key: payload["registry"][key] for key in listing["registry"]
|
|
}
|
|
for name, field in BudgetRegistryRecordDetailPayloadSerializer().fields.items():
|
|
assert field.required, name
|
|
|
|
|
|
def test_large_budget_detail_keeps_every_collection_item():
|
|
raw = full_budget_fixture()
|
|
raw["successions"] = [{"numberdoc": f"{i:08d}"} for i in range(2000)]
|
|
payload = normalize_budget_payload(raw)
|
|
assert (
|
|
len(payload["successions"]) == payload["summary"]["successions_count"] == 2000
|
|
)
|
|
assert payload["successions"][-1]["document_number"] == "00001999"
|
|
assert "successions" not in budget_list_payload(payload)
|
|
|
|
|
|
def test_incomplete_budget_dictionary_is_null_with_raw_lineage_and_metric():
|
|
raw = {"info": {"orgTypeCode": "003", "budgetLvlName": "Бюджет"}}
|
|
payload = normalize_budget_payload(raw)
|
|
assert payload["classification"]["organization_type"] is None
|
|
assert payload["budget"]["level"] is None
|
|
assert payload["upstream"] == raw
|
|
assert set(payload["schema_drift"]["incomplete_dictionaries"]) == {
|
|
"$.info.orgTypeCode",
|
|
"$.info.budgetLvlCode",
|
|
}
|
|
assert not RegistryCodeNameSerializer(data={"code": "003", "name": None}).is_valid()
|
|
assert RegistryCodeNameSerializer(
|
|
data={"code": "003", "name": "Учреждение"}
|
|
).is_valid()
|
|
record = SimpleNamespace(
|
|
source="budget_ubpandnubp",
|
|
payload={"classification": {"organization_type": {"code": "003", "name": ""}}},
|
|
)
|
|
assert (
|
|
serialize_registry_payload(record, detail=False)["classification"][
|
|
"organization_type"
|
|
]
|
|
is None
|
|
)
|
|
|
|
|
|
def test_partial_authority_permission_retains_raw_without_invalid_dictionary():
|
|
raw = {
|
|
"info": {"code": "001"},
|
|
"authorities": [
|
|
{
|
|
"authorityCode": "001",
|
|
"authorityName": "Орган",
|
|
"permissions": [
|
|
{"permissionCode": "0002"},
|
|
{"permissionCode": "0003", "permissionName": "Полномочие"},
|
|
],
|
|
}
|
|
],
|
|
}
|
|
payload = normalize_budget_payload(raw)
|
|
assert payload["authorities"][0]["permissions"] == [
|
|
{"code": "0003", "name": "Полномочие"}
|
|
]
|
|
assert payload["unclassified_blocks"]["authorities"] == raw["authorities"]
|
|
assert payload["upstream"] == raw
|
|
assert payload["schema_drift"]["incomplete_dictionaries"] == [
|
|
"$.authorities[0].permissions[0]"
|
|
]
|
|
serializer = BudgetRegistryRecordDetailPayloadSerializer(data=payload)
|
|
assert serializer.is_valid(), serializer.errors
|
|
|
|
|
|
def test_partial_budget_without_raw_preserves_existing_list_and_detail_values():
|
|
partial = {
|
|
"registry": {"code": "0001", "registration_number": "0002"},
|
|
"is_separate_division": True,
|
|
"summary": {"activities_count": 7, "has_procurement_permission": True},
|
|
"contacts": [{"phone": "123", "email": None, "website": None}],
|
|
"classification": {"organization_type": {"code": "03", "name": "Учреждение"}},
|
|
"unclassified_blocks": {"retained": [{"code": "001"}]},
|
|
}
|
|
record = SimpleNamespace(source="budget_ubpandnubp", payload=partial)
|
|
before = deepcopy(partial)
|
|
detail = serialize_registry_payload(record, detail=True)
|
|
assert detail["is_separate_division"] is True
|
|
assert detail["summary"]["activities_count"] == 7
|
|
assert detail["summary"]["has_procurement_permission"] is True
|
|
assert detail["summary"]["accounts_count"] == 0
|
|
assert detail["contacts"] == partial["contacts"]
|
|
assert detail["unclassified_blocks"] == partial["unclassified_blocks"]
|
|
assert detail["registry"]["registration_number"] == "0002"
|
|
assert partial == before
|
|
assert serialize_registry_payload(record, detail=False) == budget_list_payload(
|
|
detail
|
|
)
|
|
|
|
|
|
def test_partial_budget_raw_falls_back_only_for_missing_upstream_fields():
|
|
raw = {"id": "source-id", "info": {"statusCode": "2"}}
|
|
payload = {
|
|
"registry": {"code": "0001", "registration_number": "0002"},
|
|
"budget": {"name": "Известный бюджет"},
|
|
"address": {"region": {"code": "77", "name": "Москва"}},
|
|
"classification": {"organization_type": {"code": "03", "name": "Учреждение"}},
|
|
"upstream": raw,
|
|
}
|
|
record = SimpleNamespace(source="budget_ubpandnubp", payload=payload)
|
|
original = deepcopy(payload)
|
|
detail = serialize_registry_payload(record, detail=True)
|
|
assert detail["registry"]["code"] == "0001"
|
|
assert detail["registry"]["registration_number"] == "0002"
|
|
assert detail["budget"]["name"] == "Известный бюджет"
|
|
assert detail["address"]["region"] == {"code": "77", "name": "Москва"}
|
|
assert detail["upstream"] == raw and record.payload == original
|
|
raw["info"].update(regNum=None, budgetName="", regionCode="78", orgTypeName=None)
|
|
detail = serialize_registry_payload(record, detail=True)
|
|
assert detail["registry"]["registration_number"] is None
|
|
assert detail["budget"]["name"] is None
|
|
assert detail["address"]["region"] is None
|
|
assert detail["classification"]["organization_type"] is None
|
|
assert serialize_registry_payload(record, detail=False) == budget_list_payload(
|
|
detail
|
|
)
|
|
|
|
|
|
def test_budget_registry_code_is_required_and_never_substituted_from_id():
|
|
from apps.parsers.budget_registry import normalize_budget_record
|
|
from apps.parsers.registry_snapshots import SnapshotValidationError
|
|
|
|
raw = full_budget_fixture()
|
|
for missing in (None, "", "—"):
|
|
raw["info"]["code"] = missing
|
|
with pytest.raises(
|
|
SnapshotValidationError, match="missing_budget_registry_code"
|
|
):
|
|
normalize_budget_record(raw)
|
|
serializer = BudgetRegistryRecordListPayloadSerializer(
|
|
data=budget_list_payload(normalize_budget_payload(raw))
|
|
)
|
|
assert not serializer.is_valid()
|
|
assert "code" in serializer.errors["registry"]
|
|
|
|
|
|
def test_named_payload_openapi_schemas_have_required_typed_components():
|
|
from drf_yasg import openapi
|
|
from drf_yasg.inspectors import SwaggerAutoSchema
|
|
from rest_framework.views import APIView
|
|
|
|
components = openapi.ReferenceResolver(openapi.SCHEMA_DEFINITIONS, force_init=True)
|
|
inspector = SwaggerAutoSchema(APIView(), "/fixture/", "GET", components, None, {})
|
|
for schema in (
|
|
BudgetRegistryRecordDetailPayloadSerializer,
|
|
SmeSupportRecordDetailPayloadSerializer,
|
|
):
|
|
inspector.serializer_to_schema(schema())
|
|
budget = components.get(
|
|
"BudgetRegistryRecordDetailPayload", scope=openapi.SCHEMA_DEFINITIONS
|
|
)
|
|
sme = components.get(
|
|
"SmeSupportRecordDetailPayload", scope=openapi.SCHEMA_DEFINITIONS
|
|
)
|
|
assert {
|
|
"registry",
|
|
"accounts",
|
|
"summary",
|
|
"permissions",
|
|
"unclassified_blocks",
|
|
} <= set(budget["required"])
|
|
assert {"support_sizes", "provider", "violations", "provenance"} <= set(
|
|
sme["required"]
|
|
)
|
|
assert budget["properties"]["accounts"]["$ref"].endswith("/BudgetAccounts")
|
|
assert sme["properties"]["violations"]["items"]["$ref"].endswith(
|
|
"/SmeSupportViolation"
|
|
)
|
|
assert budget["properties"]["upstream"]["additionalProperties"] is True
|
|
assert (
|
|
budget["properties"]["unclassified_blocks"]["additionalProperties"]["items"][
|
|
"additionalProperties"
|
|
]
|
|
is True
|
|
)
|
|
assert sme["properties"]["upstream"]["additionalProperties"] is True
|
|
unclassified = components.get(
|
|
"BudgetUnclassifiedItem", scope=openapi.SCHEMA_DEFINITIONS
|
|
)
|
|
assert unclassified["additionalProperties"] is True
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw,expected",
|
|
[
|
|
("0", False),
|
|
("1", True),
|
|
("FALSE", False),
|
|
("true", True),
|
|
(None, None),
|
|
("unknown", None),
|
|
],
|
|
)
|
|
def test_registry_boolean_is_strict(raw, expected):
|
|
assert registry_boolean(raw) is expected
|
|
|
|
|
|
def test_registry_timestamps_use_explicit_utc_application_rule_and_preserve_dates():
|
|
assert (
|
|
registry_date("2025-01-02 03:04:05.0", timestamp=True) == "2025-01-02T03:04:05Z"
|
|
)
|
|
assert (
|
|
registry_date("2025-01-02T03:04:05+03:00", timestamp=True)
|
|
== "2025-01-02T00:04:05Z"
|
|
)
|
|
assert registry_date("2025-01-02", timestamp=True) == "2025-01-02T00:00:00Z"
|
|
assert registry_date("2025-01-02T03:04:05+03:00") == "2025-01-02"
|
|
with pytest.raises(ValueError):
|
|
registry_date("2025-02-30")
|
|
|
|
|
|
def test_sme_mixed_sizes_documents_violations_and_read_compatibility():
|
|
root = etree.fromstring(_xml(units=("1", "2", "3", "4", "5"))) # noqa: S320 -- local fixture only
|
|
measure = root.find("Документ/СвПредПод")
|
|
measure.find("Нарушения").set("СрокНаруш", "01.04.2025")
|
|
payload, amount = support_payload("doc", measure, "2026-08-15")
|
|
assert amount == "2.50"
|
|
assert [size["unit"] for size in payload["support_sizes"]] == [
|
|
"RUB",
|
|
"square_meter",
|
|
"hour",
|
|
"percent",
|
|
"unit",
|
|
]
|
|
assert payload["recipient_type"] == {"code": "1", "name": "Юридическое лицо"}
|
|
assert payload["sme_category"] == {"code": "1", "name": "Микропредприятие"}
|
|
assert payload["violations"][0] == {
|
|
"type": {"code": "1", "name": VIOLATION_LABELS["1"]},
|
|
"recognized_date": "2025-02-01",
|
|
"remedy_deadline": "2025-04-01",
|
|
"remedied_date": None,
|
|
}
|
|
assert [doc["number"] for doc in payload["regulatory_documents"]] == ["A", "B"]
|
|
assert payload["regulatory_documents"][0]["date"] is None
|
|
assert (
|
|
payload["violation_count"] == 1 and payload["regulatory_documents_count"] == 2
|
|
)
|
|
assert payload["upstream"]["violations"][0]["ВидНаруш"] == "1"
|
|
assert SmeSupportRecordDetailPayloadSerializer(payload).data == payload
|
|
assert SmeSupportRecordListPayloadSerializer(
|
|
sme_list_payload(payload)
|
|
).data == sme_list_payload(payload)
|
|
assert normalize_sme_payload(payload) == payload
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"recipient,category,violation",
|
|
[("1", "1", "1"), ("2", "2", "2"), ("3", "3", "1"), ("4", "4", "2")],
|
|
)
|
|
def test_official_fns_dictionary_codes_have_names(recipient, category, violation):
|
|
root = etree.fromstring(_xml()) # noqa: S320 -- local fixture only
|
|
measure = root.find("Документ/СвПредПод")
|
|
measure.set("ВидПП", recipient)
|
|
measure.set("КатСуб", category)
|
|
measure.find("Нарушения").set("ВидНаруш", violation)
|
|
payload, _ = support_payload("doc", measure, "2026-08-15")
|
|
for name in ("recipient_type", "sme_category"):
|
|
assert payload[name]["name"] and payload[name]["name"] != payload[name]["code"]
|
|
if recipient == "3":
|
|
assert "Налог на профессиональный доход" in payload["recipient_type"]["name"]
|
|
if recipient == "4":
|
|
assert "Глава крестьянского" in payload["recipient_type"]["name"]
|
|
assert payload["sme_category"]["name"] == "Отсутствует"
|
|
assert payload["violations"][0]["type"]["name"] == VIOLATION_LABELS[violation]
|
|
|
|
|
|
def test_unknown_sme_dictionary_is_not_published_with_invented_label():
|
|
root = etree.fromstring(_xml()) # noqa: S320 -- local fixture only
|
|
measure = root.find("Документ/СвПредПод")
|
|
measure.set("КатСуб", "9")
|
|
with pytest.raises(ValueError, match="unknown_sme_dictionary_code"):
|
|
support_payload("doc", measure, "2026-08-15")
|
|
|
|
|
|
def test_sme_compatibility_normalizes_optional_regions_and_official_labels():
|
|
root = etree.fromstring(_xml()) # noqa: S320 -- local fixture only
|
|
payload, _ = support_payload("doc", root.find("Документ/СвПредПод"), "2026-08-15")
|
|
payload["region"] = {"code": "01", "name": None}
|
|
payload["provider"]["region"] = {"code": None, "name": "Регион"}
|
|
payload["recipient_type"]["name"] = "Устаревшее имя"
|
|
normalized = normalize_sme_payload(payload)
|
|
assert normalized["region"] is None
|
|
assert normalized["provider"]["region"] is None
|
|
assert normalized["recipient_type"]["name"] == "Юридическое лицо"
|
|
serializer = SmeSupportRecordDetailPayloadSerializer(data=normalized)
|
|
assert serializer.is_valid(), serializer.errors
|
|
payload["recipient_type"] = {"code": "9", "name": "Неофициальное значение"}
|
|
with pytest.raises(ValueError, match="unknown_sme_dictionary_code"):
|
|
normalize_sme_payload(payload)
|
|
|
|
|
|
class RegistryPayloadContextTest(TestCase):
|
|
def setUp(self):
|
|
self.storage = TemporaryDirectory(prefix="registry-payload-test-")
|
|
self.addCleanup(self.storage.cleanup)
|
|
self.settings_override = override_settings(MEDIA_ROOT=self.storage.name)
|
|
self.settings_override.enable()
|
|
self.addCleanup(self.settings_override.disable)
|
|
Organization.objects.create(
|
|
name="Получатель",
|
|
inn="1234567890",
|
|
ogrn="1027700132195",
|
|
okpo="00123456",
|
|
directory_imported_at=timezone.now(),
|
|
)
|
|
self.provider = Organization.objects.create(
|
|
name="Каноническое имя",
|
|
inn="9876543210",
|
|
ogrn="1027700132195",
|
|
okpo="00012345",
|
|
directory_imported_at=timezone.now(),
|
|
)
|
|
self.artifact, _ = import_sme_support(
|
|
handle=_archive(_xml()),
|
|
original_name="fixture.zip",
|
|
snapshot_date="2026-08-15",
|
|
load_batch=1,
|
|
)
|
|
|
|
def test_page_context_is_bounded_and_preserves_upstream_provider(self):
|
|
records = list(
|
|
OrganizationSourceRecord.objects.select_related("extension__organization")
|
|
)
|
|
with self.assertNumQueries(2):
|
|
context = build_registry_payload_context(records * 25)
|
|
with self.assertNumQueries(0):
|
|
for record in records:
|
|
data = serialize_registry_payload(record, detail=True, context=context)
|
|
self.assertEqual(data["provider"]["name"], "Поставщик")
|
|
self.assertEqual(
|
|
data["provider"]["organization_uid"], str(self.provider.uid)
|
|
)
|
|
self.assertEqual(data["provider"]["okpo"], "00012345")
|
|
self.assertIsNone(data["region"])
|
|
self.assertEqual(
|
|
data["provenance"]["archive_sha256"], self.artifact.sha256
|
|
)
|
|
self.assertEqual(data, record.payload)
|
|
|
|
def test_old_sme_payload_and_budget_raw_get_current_shape_without_write(self):
|
|
record = OrganizationSourceRecord.objects.first()
|
|
old = deepcopy(record.payload)
|
|
old["recipient_type"] = "1"
|
|
old["sme_category"] = "1"
|
|
old["violations"] = deepcopy(old["upstream"]["violations"])
|
|
old["regulatory_documents"] = deepcopy(old["upstream"]["regulatory_documents"])
|
|
record.payload = old
|
|
context = build_registry_payload_context([record])
|
|
with self.assertNumQueries(0):
|
|
normalized = serialize_registry_payload(
|
|
record, detail=True, context=context
|
|
)
|
|
self.assertIsInstance(normalized["recipient_type"], dict)
|
|
self.assertEqual(record.payload, old)
|
|
raw = full_budget_fixture()
|
|
old_budget = SimpleNamespace(
|
|
source="budget_ubpandnubp", payload={"upstream": raw}
|
|
)
|
|
with self.assertNumQueries(0):
|
|
self.assertEqual(
|
|
serialize_registry_payload(old_budget, detail=True),
|
|
normalize_budget_payload(raw),
|
|
)
|
|
|
|
def legacy_records(self):
|
|
records = list(OrganizationSourceRecord.objects.order_by("uid"))
|
|
for record in records:
|
|
record.payload["recipient_type"] = "1"
|
|
record.payload["sme_category"] = "1"
|
|
record.status = ""
|
|
record.save(update_fields=["payload", "status"])
|
|
return records
|
|
|
|
def test_backfill_dry_run_default_then_apply_is_atomic_and_idempotent(self):
|
|
records = self.legacy_records()
|
|
before = {
|
|
record.uid: (deepcopy(record.payload), record.status, record.updated_at)
|
|
for record in records
|
|
}
|
|
command_module = (
|
|
"organizations.management.commands.normalize_published_registry_payloads"
|
|
)
|
|
output = StringIO()
|
|
with patch(f"{command_module}.invalidate_source_data_cache") as invalidate:
|
|
dry = json.loads(
|
|
call_command(
|
|
"normalize_published_registry_payloads", stdout=output, silent=True
|
|
)
|
|
)
|
|
assert dry["dry_run"] and dry["changed"] == 2
|
|
invalidate.assert_not_called()
|
|
for record in records:
|
|
record.refresh_from_db()
|
|
assert (record.payload, record.status, record.updated_at) == before[
|
|
record.uid
|
|
]
|
|
with self.captureOnCommitCallbacks(execute=True):
|
|
applied = json.loads(
|
|
call_command(
|
|
"normalize_published_registry_payloads",
|
|
apply=True,
|
|
stdout=output,
|
|
silent=True,
|
|
)
|
|
)
|
|
assert not applied["dry_run"] and applied["changed"] == 2
|
|
assert applied["exports_rebuild_required"]
|
|
assert invalidate.call_count == 2
|
|
for record in records:
|
|
record.refresh_from_db()
|
|
assert record.status == "published"
|
|
assert record.payload["sme_category"]["code"] == "1"
|
|
assert record.updated_at == before[record.uid][2]
|
|
repeated = json.loads(
|
|
call_command(
|
|
"normalize_published_registry_payloads",
|
|
apply=True,
|
|
stdout=output,
|
|
silent=True,
|
|
)
|
|
)
|
|
assert repeated["changed"] == 0
|
|
|
|
def test_backfill_late_invalid_record_rolls_back_earlier_batch(self):
|
|
records = self.legacy_records()
|
|
records[-1].payload["support_sizes"] = []
|
|
records[-1].save(update_fields=["payload"])
|
|
with pytest.raises(CommandError):
|
|
call_command(
|
|
"normalize_published_registry_payloads",
|
|
apply=True,
|
|
batch_size=1,
|
|
stdout=StringIO(),
|
|
stderr=StringIO(),
|
|
silent=True,
|
|
)
|
|
records[0].refresh_from_db()
|
|
assert records[0].status == "" and records[0].payload["sme_category"] == "1"
|
|
|
|
def test_backfill_cache_failure_rolls_back_payload_and_status(self):
|
|
records = self.legacy_records()
|
|
with patch(
|
|
"organizations.management.commands.normalize_published_registry_payloads.invalidate_source_data_cache",
|
|
side_effect=RuntimeError("cache unavailable"),
|
|
), pytest.raises(CommandError):
|
|
call_command(
|
|
"normalize_published_registry_payloads",
|
|
apply=True,
|
|
stdout=StringIO(),
|
|
stderr=StringIO(),
|
|
silent=True,
|
|
)
|
|
for record in records:
|
|
record.refresh_from_db()
|
|
assert record.status == "" and record.payload["sme_category"] == "1"
|
|
|
|
def test_post_commit_cache_failure_reports_committed_state(self):
|
|
records = self.legacy_records()
|
|
module = (
|
|
"organizations.management.commands.normalize_published_registry_payloads"
|
|
)
|
|
with patch(
|
|
f"{module}.invalidate_source_data_cache",
|
|
side_effect=[None, RuntimeError("cache unavailable")],
|
|
), patch(
|
|
f"{module}.transaction.on_commit", side_effect=lambda callback: callback()
|
|
), pytest.raises(CommandError, match="updates committed"):
|
|
call_command(
|
|
"normalize_published_registry_payloads",
|
|
apply=True,
|
|
stdout=StringIO(),
|
|
stderr=StringIO(),
|
|
silent=True,
|
|
)
|
|
for record in records:
|
|
record.refresh_from_db()
|
|
assert record.status == "published"
|
|
|
|
def test_budget_import_persists_unknown_block_metric_and_page_version(self):
|
|
raw = full_budget_fixture()
|
|
raw["info"]["okpoCode"] = "00123456"
|
|
page = {"data": [raw], "recordCount": 1, "pageNum": 1, "version": "10"}
|
|
with patch("apps.parsers.budget_registry.budget_page", return_value=page):
|
|
artifact, _ = refresh_budget_registry(load_batch=2, session=object())
|
|
assert artifact.metadata["schema_drift_records_count"] == 1
|
|
assert artifact.metadata["schema_drift_blocks"] == {
|
|
"attachment": 1,
|
|
"info": 1,
|
|
"unknown_new_block": 1,
|
|
}
|
|
record = OrganizationSourceRecord.objects.get(source="budget_ubpandnubp")
|
|
assert record.payload["upstream_audit"]["source_version"] == "10"
|
|
with self.assertNumQueries(1):
|
|
context = build_registry_payload_context([record])
|
|
with self.assertNumQueries(0):
|
|
assert (
|
|
serialize_registry_payload(record, detail=True, context=context)
|
|
== record.payload
|
|
)
|
|
|
|
def test_budget_missing_required_code_keeps_last_published_snapshot(self):
|
|
from apps.parsers.registry_snapshots import SnapshotValidationError
|
|
|
|
raw = full_budget_fixture()
|
|
raw["info"]["okpoCode"] = "00123456"
|
|
page = {"data": [raw], "recordCount": 1, "pageNum": 1, "version": "10"}
|
|
with patch("apps.parsers.budget_registry.budget_page", return_value=page):
|
|
refresh_budget_registry(load_batch=2, session=object())
|
|
before = OrganizationSourceRecord.objects.get(source="budget_ubpandnubp")
|
|
del raw["info"]["code"]
|
|
with patch(
|
|
"apps.parsers.budget_registry.budget_page", return_value=page
|
|
), pytest.raises(SnapshotValidationError, match="missing_budget_registry_code"):
|
|
refresh_budget_registry(load_batch=3, session=object())
|
|
current = OrganizationSourceRecord.objects.get(source="budget_ubpandnubp")
|
|
assert (current.uid, current.payload) == (before.uid, before.payload)
|
|
assert (
|
|
ParserSourceArtifact.objects.get(
|
|
source="budget_ubpandnubp", load_batch=3
|
|
).status
|
|
== "rejected"
|
|
)
|