fix: keep fitting export years together and name ZIP year ranges
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m48s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 2m51s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m21s

This commit is contained in:
Aleksandr Meshchryakov
2026-11-03 14:48:13 +01:00
parent f0a0266495
commit d6d1c8112a
4 changed files with 194 additions and 15 deletions

View File

@@ -1339,3 +1339,140 @@ class SourceExportCatalogTest(APITestCase):
"is_stale"
]
)
@staticmethod
def _packing_part(year, number, size):
return {
"part_id": str(uuid4()),
"dataset_key": f"records_{year}",
"year": year,
"part_number": number,
"records_count": 1,
"variants": [
{
"artifact_id": str(uuid4()),
"format": "json",
"file_name": f"records-{year}-{number:06d}.json",
"mime_type": "application/json",
"size_bytes": size,
"sha256": "0" * 64,
}
],
}
def test_twenty_mb_years_are_combined_and_named_by_range(self):
from organizations import source_export_catalog as exports
parts = [self._packing_part(year, 1, 20_000_000) for year in range(2021, 2026)]
directory = Path(self.directory.name)
for part in parts:
# Opaque sparse payloads exercise actual ZIP size, not serializer sizing.
with (directory / part["variants"][0]["file_name"]).open("wb") as handle:
handle.truncate(20_000_000)
manifest = {
"source_group": GROUP,
"parts": parts,
"supported_formats": ["json"],
"export_policy": exports._export_policy(),
}
archives = exports._build_archives(directory, manifest)
self.assertEqual(
[a["years"] for a in archives], [[2021, 2022, 2023, 2024], [2025]]
)
self.assertIn("-2021-2024-json-archive-000001.zip", archives[0]["file_name"])
self.assertIn("-2025-json-archive-000002.zip", archives[1]["file_name"])
for metadata in archives:
self.assertLessEqual(metadata["size_bytes"], 99_000_000)
with zipfile.ZipFile(directory / metadata["file_name"]) as archive:
self.assertIsNone(archive.testzip())
self.assertEqual(
{
int(n.split("/")[0])
for n in archive.namelist()
if n != "manifest.json"
},
set(metadata["years"]),
)
def test_year_that_fits_alone_is_not_split_across_archives(self):
from organizations import source_export_catalog as exports
parts = [
self._packing_part(2021, 1, 4000),
self._packing_part(2022, 1, 2000),
self._packing_part(2022, 2, 2000),
]
policy = exports._export_policy()
cap = exports._archive_size(parts[:2], "json", policy)
self.assertLessEqual(exports._archive_size(parts[1:], "json", policy), cap)
self.assertGreater(exports._archive_size(parts, "json", policy), cap)
with patch.object(exports, "ARCHIVE_MAX_BYTES", cap):
batches = exports._archive_batches(parts, "json", policy)
self.assertEqual(batches, [[parts[0]], parts[1:]])
def test_year_fit_is_calculated_separately_for_each_format(self):
from organizations import source_export_catalog as exports
parts = [
self._packing_part(2021, 1, 4000),
self._packing_part(2022, 1, 2000),
self._packing_part(2022, 2, 2000),
]
for part in parts:
part["variants"].append(
{
**part["variants"][0],
"format": "csv",
"file_name": part["variants"][0]["file_name"].replace(
".json", ".csv"
),
"mime_type": "text/csv",
"size_bytes": 1000,
}
)
policy = exports._export_policy()
cap = exports._archive_size(parts[:2], "json", policy)
with patch.object(exports, "ARCHIVE_MAX_BYTES", cap):
self.assertEqual(
exports._archive_batches(parts, "json", policy), [[parts[0]], parts[1:]]
)
self.assertEqual(exports._archive_batches(parts, "csv", policy), [parts])
def test_only_oversized_year_can_span_multiple_archives(self):
from organizations import source_export_catalog as exports
previous = self._packing_part(2021, 1, 2000)
annual = [self._packing_part(2022, i, 3500) for i in range(1, 4)]
policy = exports._export_policy()
cap = exports._archive_size(annual[:2], "json", policy)
self.assertGreater(exports._archive_size(annual, "json", policy), cap)
with patch.object(exports, "ARCHIVE_MAX_BYTES", cap):
batches = exports._archive_batches([previous, *annual], "json", policy)
self.assertEqual(batches, [[previous, annual[0]], annual[1:]])
self.assertTrue(
all(
exports._archive_size(batch, "json", policy) <= cap for batch in batches
)
)
def test_previous_packing_is_stale_but_its_archive_ticket_still_works(self):
self.records(3)
generation = build_source(GROUP)
self.assertEqual(generation.manifest.pop("archive_packing"), "whole_year")
generation.save(update_fields=["manifest"])
self.assertTrue(
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
"is_stale"
]
)
ticket = issue_ticket(self.user, self.selector(generation, "source_archive"))
with zipfile.ZipFile(
BytesIO(self.bytes(consume_ticket(ticket["ticket"])))
) as archive:
self.assertIsNone(archive.testzip())
build_source(GROUP)
self.assertFalse(
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
"is_stale"
]
)