feat: configure export year window and publish ZIP archives under 99 MB
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 4m1s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 4m9s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 3m5s

This commit is contained in:
Aleksandr Meshchryakov
2026-11-03 10:02:04 +01:00
parent a7bf5412e9
commit f0a0266495
12 changed files with 1018 additions and 55 deletions

View File

@@ -93,9 +93,8 @@ class SourceExportCatalogTest(APITestCase):
side_effect=render_while_source_changes,
), patch.object(
refresh_source_export_catalog, "retry", side_effect=Retry()
) as retry:
with self.assertRaises(Retry):
refresh_source_export_catalog.run(source_group=GROUP)
) as retry, self.assertRaises(Retry):
refresh_source_export_catalog.run(source_group=GROUP)
retry.assert_called_once_with(countdown=30, max_retries=1000)
self.assertEqual(SourceExportGeneration.objects.count(), 1)
source = next(s for s in catalog()["sources"] if s["source_group"] == GROUP)
@@ -166,6 +165,16 @@ class SourceExportCatalogTest(APITestCase):
}
if kind == "file":
result["part_id"] = generation.manifest["parts"][0]["part_id"]
if (
kind == "source_archive"
and generation.manifest.get("version", 0) >= 5
and generation.manifest["archives"]
):
result["archive_id"] = next(
a["archive_id"]
for a in generation.manifest["archives"]
if a["format"] == fmt
)
return result
def bytes(self, response):
@@ -321,7 +330,7 @@ class SourceExportCatalogTest(APITestCase):
self.user, self.selector(generation, "source_archive", "csv")
)
response = consume_ticket(ticket["ticket"])
self.assertNotIn("Content-Length", response)
self.assertEqual(int(response["Content-Length"]), ticket["size_bytes"])
with zipfile.ZipFile(BytesIO(self.bytes(response))) as archive:
self.assertEqual(len(archive.namelist()), 4)
self.assertEqual(
@@ -329,7 +338,9 @@ class SourceExportCatalogTest(APITestCase):
)
manifest = json.loads(archive.read("manifest.json"))
self.assertNotIn("relative_path", json.dumps(manifest))
self.assertEqual(manifest["generation_id"], str(generation.pk))
self.assertEqual(
manifest["export_policy"], generation.manifest["export_policy"]
)
def test_ticket_survives_new_current(self):
self.records()
@@ -801,13 +812,12 @@ class SourceExportCatalogTest(APITestCase):
2021: {str(records[0].uid)},
2022: {str(records[1].uid)},
2024: {str(records[2].uid)},
None: {str(records[3].uid), str(records[4].uid)},
},
)
self.assertEqual(generation.manifest["exported_records_count"], 5)
self.assertEqual(generation.manifest["exported_records_count"], 3)
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]],
[2021, 2022, 2024, None],
[2021, 2022, 2024],
)
def test_year_parts_numbers_formats_and_zip_directories(self):
@@ -845,7 +855,7 @@ class SourceExportCatalogTest(APITestCase):
self.assertEqual(manifest["partitioning"], "year")
self.assertEqual(manifest["records_count"], len(records))
def test_financial_record_without_lines_and_invalid_line_year_are_retained(self):
def test_financial_records_outside_window_and_invalid_line_year_are_excluded(self):
extension = FinancialIndicatorsExtension.objects.create(
organization=self.organization
)
@@ -866,22 +876,16 @@ class SourceExportCatalogTest(APITestCase):
source_record=invalid, year=0, form_code="1", line_code="1"
)
generation = build_source("financial_indicators")
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2019, None]
self.assertEqual(generation.manifest["datasets"], [])
self.assertEqual(generation.manifest["records_count"], 0)
self.assertEqual(generation.manifest["exported_records_count"], 0)
self.assertEqual(generation.manifest["archives"], [])
self.assertTrue(
OrganizationSourceRecord.objects.filter(
pk__in=[empty.pk, invalid.pk]
).count(),
2,
)
self.assertEqual(generation.manifest["records_count"], 2)
self.assertEqual(generation.manifest["exported_records_count"], 2)
rows = []
for part in generation.manifest["parts"]:
variant = part["variants"][0]
rows.extend(
r
for org in json.loads(
(_directory(generation) / variant["file_name"]).read_bytes()
)
for r in org["financial_indicators"]
)
self.assertEqual({r["uid"] for r in rows}, {str(empty.uid), str(invalid.uid)})
def test_pre_partition_generation_metadata_and_ticket_stay_readable(self):
self.records(1)
@@ -921,7 +925,7 @@ class SourceExportCatalogTest(APITestCase):
).first() or descriptor.extension_model.objects.create(
organization=self.organization
)
for year in [2020, 2021]:
for year in [2021, 2022]:
record = OrganizationSourceRecord.objects.create(
extension=extension,
source=descriptor.source,
@@ -944,7 +948,7 @@ class SourceExportCatalogTest(APITestCase):
for generation in SourceExportGeneration.objects.all():
self.assertEqual(generation.manifest["partitioning"], "year")
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2020, 2021]
[d["year"] for d in generation.manifest["datasets"]], [2021, 2022]
)
self.assertEqual(generation.manifest["records_count"], 2)
@@ -997,3 +1001,341 @@ class SourceExportCatalogTest(APITestCase):
PeriodicTasks.objects.filter(ident=1).update(last_update=old)
migration.disable(apps, None)
self.assertGreater(PeriodicTasks.objects.get(ident=1).last_update, old)
def test_export_window_admin_setting_and_calendar_rollover(self):
from datetime import UTC, datetime
from organizations import source_export_catalog as exports
from organizations.models import SourceExportSettings
records = self.records(6)
for record, value in zip(
records,
["2020-12-31", "2021-01-01", "2026-12-31", "2027-01-01", "invalid", ""],
strict=True,
):
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
record_date=value, created_at=datetime(2025, 7, 1, tzinfo=UTC)
)
with patch.object(
exports.timezone, "now", return_value=datetime(2026, 11, 3, tzinfo=UTC)
):
generation = build_source(GROUP)
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2021, 2025, 2026]
)
self.assertEqual(generation.manifest["records_count"], 3)
self.assertFalse(
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
"is_stale"
]
)
policy, _ = SourceExportSettings.objects.update_or_create(
singleton_key=1, defaults={"years_back": 2}
)
self.assertTrue(
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
"is_stale"
]
)
narrowed = build_source(GROUP)
self.assertEqual(
[d["year"] for d in narrowed.manifest["datasets"]], [2025, 2026]
)
policy.years_back = 5
policy.save()
with patch.object(
exports.timezone, "now", return_value=datetime(2027, 1, 1, tzinfo=UTC)
):
self.assertTrue(
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
"is_stale"
]
)
self.assertEqual(OrganizationSourceRecord.objects.count(), 6)
def test_financial_window_counts_unique_records_and_filters_lines(self):
extension = FinancialIndicatorsExtension.objects.create(
organization=self.organization
)
record = OrganizationSourceRecord.objects.create(
extension=extension,
source="fns_reports",
record_type="financial_report",
external_id=str(uuid4()),
)
for year in [2020, 2021, 2026, 2027, 0]:
OrganizationSourceFinancialLine.objects.create(
source_record=record, year=year, form_code="1", line_code="1"
)
generation = build_source("financial_indicators")
self.assertEqual(generation.manifest["records_count"], 1)
self.assertEqual(generation.manifest["exported_records_count"], 2)
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2021, 2026]
)
for part in generation.manifest["parts"]:
rows = json.loads(
(_directory(generation) / part["variants"][0]["file_name"]).read_bytes()
)
lines = [
line
for org in rows
for row in org["financial_indicators"]
for line in row["financial_lines"]
]
self.assertEqual({line["year"] for line in lines}, {part["year"]})
def test_archive_selector_requires_id_and_checks_format(self):
self.records()
generation = build_source(GROUP)
selection = self.selector(generation, "source_archive")
for change, code in [
({"archive_id": str(uuid4())}, "source_export_archive_not_found"),
({"format": "csv"}, "source_export_format_unavailable"),
]:
with self.assertRaises(ExportError) as error:
issue_ticket(self.user, {**selection, **change})
self.assertEqual(error.exception.code, code)
selection.pop("archive_id")
response = self.client.post(
self.base + "export-ticket/", selection, format="json"
)
self.assertEqual(response.status_code, 400)
self.assertIn("archive_id", str(response.data))
response = self.client.post(
self.base + "export-ticket/",
self.selector(generation, "source_archive"),
format="json",
)
self.assertEqual(response.status_code, 201, response.data)
def test_split_archives_are_complete_verified_and_pinned(self):
from organizations import source_export_catalog as exports
records = self.records(20)
with patch.object(exports, "ARCHIVE_MAX_BYTES", 12500):
generation = build_source(GROUP)
archives = [a for a in generation.manifest["archives"] if a["format"] == "json"]
self.assertGreater(len(archives), 1)
observed = []
tickets = []
for index, metadata in enumerate(archives, 1):
self.assertLessEqual(metadata["size_bytes"], 12500)
self.assertEqual(metadata["archive_number"], index)
self.assertEqual(metadata["archives_count"], len(archives))
selection = self.selector(generation, "source_archive")
selection["archive_id"] = metadata["archive_id"]
tickets.append((issue_ticket(self.user, selection), metadata))
build_source(GROUP)
for ticket, metadata in tickets:
body = self.bytes(consume_ticket(ticket["ticket"]))
self.assertEqual(hashlib.sha256(body).hexdigest(), metadata["sha256"])
self.assertEqual(len(body), metadata["size_bytes"])
with zipfile.ZipFile(BytesIO(body)) as archive:
self.assertIsNone(archive.testzip())
archived = json.loads(archive.read("manifest.json"))
self.assertEqual(
{p["part_id"] for p in archived["parts"]}, set(metadata["part_ids"])
)
for member in archive.namelist():
if member == "manifest.json":
continue
year = int(member.split("/")[0])
for org in json.loads(archive.read(member)):
for row in org[GROUP]:
self.assertEqual(int(row["record_date"][:4]), year)
observed.append(row["uid"])
self.assertCountEqual(observed, [str(r.uid) for r in records])
self.assertTrue(exports.verify_generation(generation))
damaged = _directory(generation) / archives[0]["file_name"]
data = bytearray(damaged.read_bytes())
data[40] ^= 1
damaged.write_bytes(data)
self.assertFalse(exports.verify_generation(generation))
def test_single_record_too_large_preserves_current_generation(self):
from organizations import source_export_catalog as exports
self.records(1)
previous = build_source(GROUP)
with patch.object(exports, "ARCHIVE_MAX_BYTES", 100), self.assertRaises(
ExportError
) as error:
build_source(GROUP)
self.assertEqual(error.exception.code, "source_export_record_too_large")
self.assertEqual(
SourceExportState.objects.get(pk=GROUP).current_generation, previous.pk
)
self.assertEqual(SourceExportGeneration.objects.count(), 1)
self.assertFalse(list(_directory(previous).parent.glob(".building-*")))
def test_v4_archive_ticket_remains_streamable_after_new_generation(self):
self.records(3)
old = build_source(GROUP)
old.manifest["version"] = 4
old.manifest.pop("archives")
old.manifest.pop("export_policy")
old.save(update_fields=["manifest"])
ticket = issue_ticket(self.user, self.selector(old, "source_archive"))
build_source(GROUP)
with zipfile.ZipFile(
BytesIO(self.bytes(consume_ticket(ticket["ticket"])))
) as archive:
self.assertIsNone(archive.testzip())
self.assertEqual(
json.loads(archive.read("manifest.json"))["generation_id"], str(old.pk)
)
def test_exact_99mb_zip_limit_includes_headers_and_manifest(self):
from organizations import source_export_catalog as exports
self.records(1)
generation = build_source(GROUP)
part = generation.manifest["parts"][0]
variant = next(v for v in part["variants"] if v["format"] == "json")
directory = _directory(generation)
# Sparse fixture proves the real 99,000,000-byte boundary without a huge ORM payload.
for delta in [0, 1]:
variant["size_bytes"] = 98_000_000
while True:
measured = exports._archive_size(
[part], "json", generation.manifest["export_policy"]
)
correction = 99_000_000 + delta - measured
if not correction:
break
variant["size_bytes"] += correction
with (directory / variant["file_name"]).open("wb") as handle:
handle.truncate(variant["size_bytes"])
manifest = {
**generation.manifest,
"parts": [part],
"supported_formats": ["json"],
}
if delta:
with self.assertRaises(ExportError):
exports._build_archives(directory, manifest)
else:
built = exports._build_archives(directory, manifest)
self.assertEqual(built[0]["size_bytes"], 99_000_000)
with zipfile.ZipFile(directory / built[0]["file_name"]) as archive:
self.assertIsNone(archive.testzip())
def test_admin_singleton_policy_validation(self):
from django.contrib.admin.sites import AdminSite
from django.core.exceptions import ValidationError
from django.db import IntegrityError, transaction
from django.test import RequestFactory
from organizations.admin import SourceExportSettingsAdmin
from organizations.models import SourceExportSettings
model_admin = SourceExportSettingsAdmin(SourceExportSettings, AdminSite())
request = RequestFactory().get("/")
request.user = self.user
SourceExportSettings.objects.update_or_create(singleton_key=1)
self.assertFalse(model_admin.has_add_permission(request))
self.assertFalse(model_admin.has_delete_permission(request))
with transaction.atomic(), self.assertRaises(IntegrityError):
SourceExportSettings.objects.create(singleton_key=2)
with self.assertRaises(ValidationError):
SourceExportSettings(years_back=10000).full_clean()
def test_large_year_is_split_at_record_boundaries_for_all_formats(self):
from organizations import source_export_catalog as exports
records = self.records(8)
for record in records:
record.record_date = "2025-01-01"
record.payload = {"subject": "x" * 1500}
record.save(update_fields=["record_date", "payload"])
with patch.object(exports, "ARCHIVE_MAX_BYTES", 14000), override_settings(
SOURCE_EXPORT_ROWS_PER_PART=100
):
generation = build_source(GROUP)
self.assertGreater(len(generation.manifest["parts"]), 1)
for fmt in ("json", "csv", "xlsx"):
archives = [
a for a in generation.manifest["archives"] if a["format"] == fmt
]
selected_parts = []
for artifact in archives:
self.assertLessEqual(artifact["size_bytes"], 14000)
selected_parts.extend(artifact["part_ids"])
with zipfile.ZipFile(
_directory(generation) / artifact["file_name"]
) as archive:
self.assertIsNone(archive.testzip())
self.assertCountEqual(
selected_parts, [p["part_id"] for p in generation.manifest["parts"]]
)
self.assertEqual(
sum(p["records_count"] for p in generation.manifest["parts"]), 8
)
def test_recursive_render_splits_keep_all_rows(self):
from organizations import source_export_catalog as exports
from organizations.source_record_export import _build_record_row
records = self.records(64)
directory = Path(self.directory.name)
spool = directory / ".nested.jsonl"
spool.write_bytes(
b"".join(
exports._json_bytes(
_build_record_row(record, include_financial_lines=False)
)
+ b"\n"
for record in records
)
)
# Bypass the initial size heuristic: force several actual render/split levels.
with override_settings(SOURCE_EXPORT_MAX_FILE_BYTES=7000):
parts = exports._render_part(
GROUP, spool, directory, [], len(records), ("json",), 1, 2025
)
self.assertGreater(len(parts), 4)
observed = []
for part in parts:
variant = part["variants"][0]
self.assertLessEqual(variant["size_bytes"], 7000)
observed.extend(
r["uid"]
for org in json.loads((directory / variant["file_name"]).read_bytes())
for r in org[GROUP]
)
self.assertCountEqual(observed, [str(r.uid) for r in records])
self.assertFalse(list(directory.glob("*.left*")))
self.assertFalse(list(directory.glob("*.right*")))
def test_build_uses_one_policy_even_if_admin_changes_during_render(self):
from organizations import source_export_catalog as exports
from organizations.models import SourceExportSettings
self.records()
render = exports._render_parts
def change_policy(*args, **kwargs):
SourceExportSettings.objects.update_or_create(
singleton_key=1, defaults={"years_back": 0}
)
return render(*args, **kwargs)
with patch.object(exports, "_render_parts", side_effect=change_policy):
generation = build_source(GROUP)
self.assertEqual(generation.manifest["export_policy"]["years_back"], 5)
for metadata in generation.manifest["archives"]:
with zipfile.ZipFile(
_directory(generation) / metadata["file_name"]
) as archive:
self.assertEqual(
json.loads(archive.read("manifest.json"))["export_policy"][
"years_back"
],
5,
)
self.assertTrue(
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
"is_stale"
]
)

View File

@@ -17,12 +17,13 @@ from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
@pytest.mark.parametrize("selection_kind", ["file", "source_archive"])
@pytest.mark.django_db(transaction=True)
@pytest.mark.skipif(
connection.vendor != "postgresql",
reason="Requires PostgreSQL row locks and independent connections",
)
def test_two_simultaneous_native_posts_one_ticket(tmp_path):
def test_two_simultaneous_native_posts_one_ticket(tmp_path, selection_kind):
with override_settings(SOURCE_EXPORT_DIRECTORY=str(tmp_path)):
user = UserFactory.create_superuser()
organization = Organization.objects.create(
@@ -41,10 +42,20 @@ def test_two_simultaneous_native_posts_one_ticket(tmp_path):
ticket = issue_ticket(
user,
{
"selection_kind": "file",
"selection_kind": selection_kind,
"source_group": "government_procurements",
"generation_id": str(generation.pk),
"part_id": generation.manifest["parts"][0]["part_id"],
**(
{"part_id": generation.manifest["parts"][0]["part_id"]}
if selection_kind == "file"
else {
"archive_id": next(
a["archive_id"]
for a in generation.manifest["archives"]
if a["format"] == "json"
)
}
),
"format": "json",
},
)