fix: keep fitting export years together and name ZIP year ranges
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m48s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 2m51s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m21s

This commit is contained in:
Aleksandr Meshchryakov
2026-11-03 14:48:13 +01:00
parent f0a0266495
commit d6d1c8112a
4 changed files with 194 additions and 15 deletions

View File

@@ -74,6 +74,7 @@ IDLE_BUILD = {
"error_code": None,
}
ARCHIVE_MAX_BYTES = 99_000_000
ARCHIVE_PACKING = "whole_year"
CHUNK_SIZE = 1024 * 1024
HEARTBEAT_RENEW_SECONDS = 30
TICKET_PATTERN = re.compile(r"se1_[A-Za-z0-9_-]{43}")
@@ -210,7 +211,8 @@ def summary(entry, state=None, generation=None, revision=None, policy=None):
stale = bool(
manifest
and (
manifest.get("export_policy") != (policy or _export_policy())
manifest.get("archive_packing") != ARCHIVE_PACKING
or manifest.get("export_policy") != (policy or _export_policy())
or build["state"] == "failed"
or generation.created_at < timezone.now() - timedelta(hours=36)
or generation.data_revision
@@ -599,6 +601,7 @@ def _archive_manifest(parts, fmt, policy=None):
return {
"version": 5,
"partitioning": "year",
"archive_packing": ARCHIVE_PACKING,
"supported_formats": [fmt],
"records_count": sum(p["records_count"] for p in parts),
"export_policy": policy or _export_policy(),
@@ -637,25 +640,41 @@ def _archive_size(parts, fmt, policy=None):
return len(buffer.getvalue()) + payload_bytes
def _build_archives(directory, manifest):
result = []
for fmt in manifest["supported_formats"]:
batches, batch = [], []
for part in manifest["parts"]:
if (
_archive_size([*batch, part], fmt, manifest["export_policy"])
> ARCHIVE_MAX_BYTES
):
def _archive_batches(parts, fmt, policy):
"""Keep a year together when it fits alone; split only oversized years."""
from itertools import groupby
batches, batch = [], []
for _, annual in groupby(parts, key=lambda part: part["year"]):
annual = list(annual)
units = (
[annual]
if _archive_size(annual, fmt, policy) <= ARCHIVE_MAX_BYTES
else [[part] for part in annual]
)
for unit in units:
if _archive_size([*batch, *unit], fmt, policy) > ARCHIVE_MAX_BYTES:
if not batch:
raise ExportError("record_too_large", 503)
batches.append(batch)
batch = []
batch.append(part)
if batch:
batches.append(batch)
if _archive_size(unit, fmt, policy) > ARCHIVE_MAX_BYTES:
raise ExportError("record_too_large", 503)
batch.extend(unit)
if batch:
batches.append(batch)
return batches
def _build_archives(directory, manifest):
result = []
for fmt in manifest["supported_formats"]:
batches = _archive_batches(manifest["parts"], fmt, manifest["export_policy"])
for index, parts in enumerate(batches, 1):
_check_capacity(directory)
name = f"{SOURCE_GROUP_EXPORT_FILE_STEMS[manifest['source_group']]}-{fmt}-archive-{index:06d}.zip"
years = sorted({p["year"] for p in parts})
period = str(years[0]) if len(years) == 1 else f"{years[0]}-{years[-1]}"
name = f"{SOURCE_GROUP_EXPORT_FILE_STEMS[manifest['source_group']]}-{period}-{fmt}-archive-{index:06d}.zip"
path = directory / name
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED) as archive:
for variant, member in _archive_entries(parts, fmt):
@@ -682,7 +701,7 @@ def _build_archives(directory, manifest):
"archive_number": index,
"archives_count": len(batches),
"format": fmt,
"years": sorted({p["year"] for p in parts}),
"years": years,
"file_name": name,
"mime_type": "application/zip",
"size_bytes": size,
@@ -958,6 +977,7 @@ def build_source(group):
_check_capacity(staging)
manifest = {
"version": 5,
"archive_packing": ARCHIVE_PACKING,
"export_policy": policy,
"partitioning": "year",
"exported_records_count": sum(p["records_count"] for p in parts),