fix: keep fitting export years together and name ZIP year ranges
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m48s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 2m51s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m21s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m48s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 2m51s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m21s
This commit is contained in:
@@ -161,3 +161,10 @@ SOURCE_EXPORT_TRANSPORT_LOAD=1 SOURCE_EXPORT_TRANSPORT_REPORT=/tmp/source-export
|
||||
а также staging и retained generations. Все файлы остаются в приватном generation directory.
|
||||
`--verify-only` проверяет размеры и SHA-256 как частей, так и ZIP. Cleanup удаляет их
|
||||
вместе с поколением, защищённым действующими leases. Ночная задача и расписание не меняются.
|
||||
|
||||
|
||||
Уточнение упаковки: год, который целиком помещается в ZIP выбранного формата,
|
||||
не разделяется между архивами даже при нескольких исходных частях. При нехватке
|
||||
остатка он целиком уходит в следующий ZIP; только большой год >99 МБ делится.
|
||||
Имена ZIP включают диапазон лет. После обновления builder прежний packing помечается
|
||||
stale; штатная пересборка публикует новое immutable поколение, сохраняя старые tickets.
|
||||
|
||||
@@ -22,6 +22,21 @@
|
||||
**99 000 000 байт** вместе со служебными данными ZIP. ZIP_STORED, самостоятельные архивы,
|
||||
не многотомный ZIP. Большой год делится по границам записей; один архив может включать
|
||||
несколько лет. Файлы частей согласованы по записям между форматами. Финансы — JSON-only.
|
||||
Годы добавляются в архив последовательно. Если **все части года выбранного формата**
|
||||
помещаются в отдельный ZIP до 99 МБ, этот год сохраняется целиком: при нехватке места
|
||||
в текущем ZIP он полностью переносится в следующий. Только год, который сам превышает
|
||||
лимит, разрешено распределять между архивами по готовым частям. Правило считается
|
||||
отдельно для CSV/XLSX/JSON; набор записей соответствующих частей не меняется.
|
||||
Например, пять годов по 20 МБ: первый ZIP содержит 2021–2024 (~80 МБ), следующий —
|
||||
2025; ZIP не дополняется фиктивными байтами до лимита.
|
||||
|
||||
Имя ZIP показывает реальный диапазон: `public-procurements-2021-2024-json-archive-000001.zip`;
|
||||
один год — `public-procurements-2025-json-archive-000002.zip`. Номер сохраняет уникальность
|
||||
при нескольких архивах одного большого года. Поле archives.years содержит точный список
|
||||
лет, включая пропуски внутри диапазона. Новые manifest v5 помечены внутренним
|
||||
`archive_packing: "whole_year"`; прежние поколения доступны, но stale до пересборки.
|
||||
Публичные selectors и структура API не меняются.
|
||||
|
||||
Размер архива проверяется перед атомарной публикацией. Слишком большая одиночная запись
|
||||
даёт `source_export_record_too_large`, прежнее current поколение сохраняется.
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ IDLE_BUILD = {
|
||||
"error_code": None,
|
||||
}
|
||||
ARCHIVE_MAX_BYTES = 99_000_000
|
||||
ARCHIVE_PACKING = "whole_year"
|
||||
CHUNK_SIZE = 1024 * 1024
|
||||
HEARTBEAT_RENEW_SECONDS = 30
|
||||
TICKET_PATTERN = re.compile(r"se1_[A-Za-z0-9_-]{43}")
|
||||
@@ -210,7 +211,8 @@ def summary(entry, state=None, generation=None, revision=None, policy=None):
|
||||
stale = bool(
|
||||
manifest
|
||||
and (
|
||||
manifest.get("export_policy") != (policy or _export_policy())
|
||||
manifest.get("archive_packing") != ARCHIVE_PACKING
|
||||
or manifest.get("export_policy") != (policy or _export_policy())
|
||||
or build["state"] == "failed"
|
||||
or generation.created_at < timezone.now() - timedelta(hours=36)
|
||||
or generation.data_revision
|
||||
@@ -599,6 +601,7 @@ def _archive_manifest(parts, fmt, policy=None):
|
||||
return {
|
||||
"version": 5,
|
||||
"partitioning": "year",
|
||||
"archive_packing": ARCHIVE_PACKING,
|
||||
"supported_formats": [fmt],
|
||||
"records_count": sum(p["records_count"] for p in parts),
|
||||
"export_policy": policy or _export_policy(),
|
||||
@@ -637,25 +640,41 @@ def _archive_size(parts, fmt, policy=None):
|
||||
return len(buffer.getvalue()) + payload_bytes
|
||||
|
||||
|
||||
def _build_archives(directory, manifest):
|
||||
result = []
|
||||
for fmt in manifest["supported_formats"]:
|
||||
batches, batch = [], []
|
||||
for part in manifest["parts"]:
|
||||
if (
|
||||
_archive_size([*batch, part], fmt, manifest["export_policy"])
|
||||
> ARCHIVE_MAX_BYTES
|
||||
):
|
||||
def _archive_batches(parts, fmt, policy):
|
||||
"""Keep a year together when it fits alone; split only oversized years."""
|
||||
from itertools import groupby
|
||||
|
||||
batches, batch = [], []
|
||||
for _, annual in groupby(parts, key=lambda part: part["year"]):
|
||||
annual = list(annual)
|
||||
units = (
|
||||
[annual]
|
||||
if _archive_size(annual, fmt, policy) <= ARCHIVE_MAX_BYTES
|
||||
else [[part] for part in annual]
|
||||
)
|
||||
for unit in units:
|
||||
if _archive_size([*batch, *unit], fmt, policy) > ARCHIVE_MAX_BYTES:
|
||||
if not batch:
|
||||
raise ExportError("record_too_large", 503)
|
||||
batches.append(batch)
|
||||
batch = []
|
||||
batch.append(part)
|
||||
if batch:
|
||||
batches.append(batch)
|
||||
if _archive_size(unit, fmt, policy) > ARCHIVE_MAX_BYTES:
|
||||
raise ExportError("record_too_large", 503)
|
||||
batch.extend(unit)
|
||||
if batch:
|
||||
batches.append(batch)
|
||||
return batches
|
||||
|
||||
|
||||
def _build_archives(directory, manifest):
|
||||
result = []
|
||||
for fmt in manifest["supported_formats"]:
|
||||
batches = _archive_batches(manifest["parts"], fmt, manifest["export_policy"])
|
||||
for index, parts in enumerate(batches, 1):
|
||||
_check_capacity(directory)
|
||||
name = f"{SOURCE_GROUP_EXPORT_FILE_STEMS[manifest['source_group']]}-{fmt}-archive-{index:06d}.zip"
|
||||
years = sorted({p["year"] for p in parts})
|
||||
period = str(years[0]) if len(years) == 1 else f"{years[0]}-{years[-1]}"
|
||||
name = f"{SOURCE_GROUP_EXPORT_FILE_STEMS[manifest['source_group']]}-{period}-{fmt}-archive-{index:06d}.zip"
|
||||
path = directory / name
|
||||
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED) as archive:
|
||||
for variant, member in _archive_entries(parts, fmt):
|
||||
@@ -682,7 +701,7 @@ def _build_archives(directory, manifest):
|
||||
"archive_number": index,
|
||||
"archives_count": len(batches),
|
||||
"format": fmt,
|
||||
"years": sorted({p["year"] for p in parts}),
|
||||
"years": years,
|
||||
"file_name": name,
|
||||
"mime_type": "application/zip",
|
||||
"size_bytes": size,
|
||||
@@ -958,6 +977,7 @@ def build_source(group):
|
||||
_check_capacity(staging)
|
||||
manifest = {
|
||||
"version": 5,
|
||||
"archive_packing": ARCHIVE_PACKING,
|
||||
"export_policy": policy,
|
||||
"partitioning": "year",
|
||||
"exported_records_count": sum(p["records_count"] for p in parts),
|
||||
|
||||
@@ -1339,3 +1339,140 @@ class SourceExportCatalogTest(APITestCase):
|
||||
"is_stale"
|
||||
]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _packing_part(year, number, size):
|
||||
return {
|
||||
"part_id": str(uuid4()),
|
||||
"dataset_key": f"records_{year}",
|
||||
"year": year,
|
||||
"part_number": number,
|
||||
"records_count": 1,
|
||||
"variants": [
|
||||
{
|
||||
"artifact_id": str(uuid4()),
|
||||
"format": "json",
|
||||
"file_name": f"records-{year}-{number:06d}.json",
|
||||
"mime_type": "application/json",
|
||||
"size_bytes": size,
|
||||
"sha256": "0" * 64,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def test_twenty_mb_years_are_combined_and_named_by_range(self):
|
||||
from organizations import source_export_catalog as exports
|
||||
|
||||
parts = [self._packing_part(year, 1, 20_000_000) for year in range(2021, 2026)]
|
||||
directory = Path(self.directory.name)
|
||||
for part in parts:
|
||||
# Opaque sparse payloads exercise actual ZIP size, not serializer sizing.
|
||||
with (directory / part["variants"][0]["file_name"]).open("wb") as handle:
|
||||
handle.truncate(20_000_000)
|
||||
manifest = {
|
||||
"source_group": GROUP,
|
||||
"parts": parts,
|
||||
"supported_formats": ["json"],
|
||||
"export_policy": exports._export_policy(),
|
||||
}
|
||||
archives = exports._build_archives(directory, manifest)
|
||||
self.assertEqual(
|
||||
[a["years"] for a in archives], [[2021, 2022, 2023, 2024], [2025]]
|
||||
)
|
||||
self.assertIn("-2021-2024-json-archive-000001.zip", archives[0]["file_name"])
|
||||
self.assertIn("-2025-json-archive-000002.zip", archives[1]["file_name"])
|
||||
for metadata in archives:
|
||||
self.assertLessEqual(metadata["size_bytes"], 99_000_000)
|
||||
with zipfile.ZipFile(directory / metadata["file_name"]) as archive:
|
||||
self.assertIsNone(archive.testzip())
|
||||
self.assertEqual(
|
||||
{
|
||||
int(n.split("/")[0])
|
||||
for n in archive.namelist()
|
||||
if n != "manifest.json"
|
||||
},
|
||||
set(metadata["years"]),
|
||||
)
|
||||
|
||||
def test_year_that_fits_alone_is_not_split_across_archives(self):
|
||||
from organizations import source_export_catalog as exports
|
||||
|
||||
parts = [
|
||||
self._packing_part(2021, 1, 4000),
|
||||
self._packing_part(2022, 1, 2000),
|
||||
self._packing_part(2022, 2, 2000),
|
||||
]
|
||||
policy = exports._export_policy()
|
||||
cap = exports._archive_size(parts[:2], "json", policy)
|
||||
self.assertLessEqual(exports._archive_size(parts[1:], "json", policy), cap)
|
||||
self.assertGreater(exports._archive_size(parts, "json", policy), cap)
|
||||
with patch.object(exports, "ARCHIVE_MAX_BYTES", cap):
|
||||
batches = exports._archive_batches(parts, "json", policy)
|
||||
self.assertEqual(batches, [[parts[0]], parts[1:]])
|
||||
|
||||
def test_year_fit_is_calculated_separately_for_each_format(self):
|
||||
from organizations import source_export_catalog as exports
|
||||
|
||||
parts = [
|
||||
self._packing_part(2021, 1, 4000),
|
||||
self._packing_part(2022, 1, 2000),
|
||||
self._packing_part(2022, 2, 2000),
|
||||
]
|
||||
for part in parts:
|
||||
part["variants"].append(
|
||||
{
|
||||
**part["variants"][0],
|
||||
"format": "csv",
|
||||
"file_name": part["variants"][0]["file_name"].replace(
|
||||
".json", ".csv"
|
||||
),
|
||||
"mime_type": "text/csv",
|
||||
"size_bytes": 1000,
|
||||
}
|
||||
)
|
||||
policy = exports._export_policy()
|
||||
cap = exports._archive_size(parts[:2], "json", policy)
|
||||
with patch.object(exports, "ARCHIVE_MAX_BYTES", cap):
|
||||
self.assertEqual(
|
||||
exports._archive_batches(parts, "json", policy), [[parts[0]], parts[1:]]
|
||||
)
|
||||
self.assertEqual(exports._archive_batches(parts, "csv", policy), [parts])
|
||||
|
||||
def test_only_oversized_year_can_span_multiple_archives(self):
|
||||
from organizations import source_export_catalog as exports
|
||||
|
||||
previous = self._packing_part(2021, 1, 2000)
|
||||
annual = [self._packing_part(2022, i, 3500) for i in range(1, 4)]
|
||||
policy = exports._export_policy()
|
||||
cap = exports._archive_size(annual[:2], "json", policy)
|
||||
self.assertGreater(exports._archive_size(annual, "json", policy), cap)
|
||||
with patch.object(exports, "ARCHIVE_MAX_BYTES", cap):
|
||||
batches = exports._archive_batches([previous, *annual], "json", policy)
|
||||
self.assertEqual(batches, [[previous, annual[0]], annual[1:]])
|
||||
self.assertTrue(
|
||||
all(
|
||||
exports._archive_size(batch, "json", policy) <= cap for batch in batches
|
||||
)
|
||||
)
|
||||
|
||||
def test_previous_packing_is_stale_but_its_archive_ticket_still_works(self):
|
||||
self.records(3)
|
||||
generation = build_source(GROUP)
|
||||
self.assertEqual(generation.manifest.pop("archive_packing"), "whole_year")
|
||||
generation.save(update_fields=["manifest"])
|
||||
self.assertTrue(
|
||||
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
|
||||
"is_stale"
|
||||
]
|
||||
)
|
||||
ticket = issue_ticket(self.user, self.selector(generation, "source_archive"))
|
||||
with zipfile.ZipFile(
|
||||
BytesIO(self.bytes(consume_ticket(ticket["ticket"])))
|
||||
) as archive:
|
||||
self.assertIsNone(archive.testzip())
|
||||
build_source(GROUP)
|
||||
self.assertFalse(
|
||||
next(s for s in catalog()["sources"] if s["source_group"] == GROUP)[
|
||||
"is_stale"
|
||||
]
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user