feat: partition all source exports by year and enable nightly builds

This commit is contained in:
Aleksandr Meshchryakov
2026-10-22 21:45:29 +02:00
parent 63872e86fe
commit 2002e80196
13 changed files with 672 additions and 87 deletions

View File

@@ -25400,6 +25400,13 @@
"all_available"
]
},
"partitioning": {
"type": "string",
"enum": [
"year",
"none"
]
},
"source_slug": {
"type": "string"
},
@@ -25420,6 +25427,12 @@
"maximum": 9007199254740991,
"nullable": true
},
"exported_records_count": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991,
"nullable": true
},
"parts_count": {
"type": "integer",
"minimum": 0,
@@ -25462,11 +25475,13 @@
"generated_at",
"data_as_of",
"data_scope",
"partitioning",
"source_slug",
"title",
"order",
"export_enabled",
"records_count",
"exported_records_count",
"parts_count",
"supported_formats",
"default_format",
@@ -25491,7 +25506,7 @@
"schema_version": {
"type": "string",
"enum": [
"1"
"2"
]
},
"catalog_revision": {
@@ -25574,6 +25589,12 @@
"key": {
"type": "string"
},
"year": {
"type": "integer",
"minimum": 1,
"maximum": 9999,
"nullable": true
},
"title": {
"type": "string"
},
@@ -25593,6 +25614,7 @@
},
"required": [
"key",
"year",
"title",
"export_schema_version",
"records_count",
@@ -25649,6 +25671,12 @@
"type": "string",
"format": "uuid"
},
"year": {
"type": "integer",
"minimum": 1,
"maximum": 9999,
"nullable": true
},
"dataset_key": {
"type": "string"
},
@@ -25688,6 +25716,7 @@
},
"required": [
"part_id",
"year",
"dataset_key",
"title",
"part_number",
@@ -25809,6 +25838,13 @@
"all_available"
]
},
"partitioning": {
"type": "string",
"enum": [
"year",
"none"
]
},
"current_generation_id": {
"type": "string",
"format": "uuid",
@@ -25838,6 +25874,7 @@
"generated_at",
"data_as_of",
"data_scope",
"partitioning",
"current_generation_id",
"description",
"datasets",

View File

@@ -1,4 +1,4 @@
"""Prepare new all-available export generations without changing annual exports."""
"""Prepare source exports partitioned by year without changing legacy annual exports."""
from django.core.management.base import BaseCommand, CommandError
from organizations.models import SourceExportGeneration
@@ -11,7 +11,7 @@ from organizations.source_export_catalog import (
class Command(BaseCommand):
help = "Собрать all_available каталог и файлы существующих источников"
help = "Собрать файлы всех источников по доступным годам"
def add_arguments(self, parser):
parser.add_argument("--source-group", choices=list(registry()))

View File

@@ -0,0 +1,56 @@
"""Enable nightly preparation of all sources partitioned by calendar year."""
from django.db import migrations
from django.utils import timezone
NAME = "organizations:source-export-catalog:nightly-msk"
def schedule_changed(apps):
# Historical migration models do not run PeriodicTask's save/signals.
apps.get_model("django_celery_beat", "PeriodicTasks").objects.update_or_create(
ident=1, defaults={"last_update": timezone.now()}
)
def enable(apps, schema_editor):
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
Task = apps.get_model("django_celery_beat", "PeriodicTask")
cron, _ = Cron.objects.get_or_create(
minute="30",
hour="6",
day_of_week="*",
day_of_month="*",
month_of_year="*",
timezone="Europe/Moscow",
)
Task.objects.update_or_create(
name=NAME,
defaults={
"task": "organizations.tasks.refresh_source_export_catalog",
"enabled": True,
"one_off": False,
"expires": None,
"expire_seconds": None,
"start_time": None,
"args": "[]",
"kwargs": "{}",
"crontab": cron,
"interval": None,
"solar": None,
"clocked": None,
"description": "Nightly preparation of all 22 sources, with separate files for every available calendar year.",
},
)
schedule_changed(apps)
def disable(apps, schema_editor):
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
name=NAME
).update(enabled=False)
schedule_changed(apps)
class Migration(migrations.Migration):
dependencies = [("organizations", "0020_source_export_catalog_schedule")]
operations = [migrations.RunPython(enable, reverse_code=disable)]

View File

@@ -217,6 +217,12 @@ def summary(entry, state=None, generation=None, revision=None):
"data_as_of": manifest["data_as_of"] if manifest else None,
"data_scope": "all_available",
"records_count": manifest["records_count"] if manifest else None,
"exported_records_count": manifest.get(
"exported_records_count", manifest["records_count"]
)
if manifest
else None,
"partitioning": manifest.get("partitioning", "none") if manifest else "year",
"parts_count": len(manifest["parts"]) if manifest else None,
"supported_formats": formats,
"default_format": formats[0] if formats else None,
@@ -251,7 +257,7 @@ def catalog():
# is_stale may change solely with age; revision tracks actual persisted metadata.
persisted = [{k: v for k, v in item.items() if k != "is_stale"} for item in sources]
return {
"schema_version": "1",
"schema_version": "2",
"catalog_revision": hashlib.sha256(_json_bytes(persisted)).hexdigest(),
"sources": sources,
}
@@ -309,6 +315,7 @@ def detail(group, query):
"generated_at",
"data_as_of",
"data_scope",
"partitioning",
)
}
data.update(
@@ -317,8 +324,13 @@ def detail(group, query):
if state and state.current_generation
else None,
"description": manifest.get("description", _description(entry)),
"datasets": manifest.get("datasets", []),
"files": parts[(page - 1) * size : page * size],
"datasets": [
{"year": None, **dataset} for dataset in manifest.get("datasets", [])
],
"files": [
{"year": None, **part}
for part in parts[(page - 1) * size : page * size]
],
}
)
return data, {
@@ -331,7 +343,7 @@ def detail(group, query):
}
def _description(entry, example=None):
def _description(entry, example=None, example_key="records"):
return {
"composition": entry["composition"],
"origin": entry["origin"],
@@ -342,7 +354,7 @@ def _description(entry, example=None):
"предметные поля источника",
]
),
"example_dataset_key": "records" if example else None,
"example_dataset_key": example_key if example else None,
"example_is_synthetic": False,
"example_record": example,
}
@@ -417,6 +429,32 @@ def _build_lock(group):
_build_state(group, owner, lock_owner=None, lock_expires_at=None)
def _record_year(record):
"""Use the canonical record-date year; missing dates use the ingestion year."""
if not record.record_date:
return timezone.localtime(
record.created_at, timezone.get_default_timezone()
).year
match = re.search(r"(?<!\d)([0-9]{4})(?!\d)", record.record_date)
return int(match[1]) if match and int(match[1]) else None
def _year_key(year):
return f"records_{year:04d}" if year else "records_undated"
def _year_rows(record, row, financial):
if not financial or not row["financial_lines"]:
yield _record_year(record), row
return
years = {}
for line in row["financial_lines"]:
year = line["year"] if 1 <= line["year"] <= 9999 else None
years.setdefault(year, []).append(line)
for year, lines in years.items():
yield year, {**row, "financial_lines": lines}
def _spool(group, path):
"""Use a disk-backed sort, bounded RAM and one repeatable-read snapshot.
@@ -425,12 +463,14 @@ def _spool(group, path):
"""
import sqlite3
count, headers, example = 0, set(), None
count, headers, example, example_key = 0, set(), None, None
ordered = path.with_suffix(".sqlite3")
db = sqlite3.connect(ordered)
try:
db.execute("PRAGMA journal_mode=OFF")
db.execute("CREATE TABLE rows (source TEXT, kind TEXT, uid TEXT, data TEXT)")
db.execute(
"CREATE TABLE rows (partition TEXT, source TEXT, kind TEXT, uid TEXT, data TEXT)"
)
with _source_export_read_snapshot():
revision = get_source_data_cache_version()
as_of = _timestamp()
@@ -471,27 +511,37 @@ def _spool(group, path):
record, include_financial_lines=group == FINANCIAL_SOURCE_GROUP
)
headers.update(key for key in row if key.startswith("payload."))
encoded = _json_bytes(row).decode("utf-8")
if example is None and len(encoded.encode("utf-8")) <= 32768:
# Example is the actual JSON file projection, including OKPO grouping.
example_row = dict(row)
okpo = example_row.pop("ОКПО", None) or example_row.pop(
"okpo", None
for year, annual_row in _year_rows(
record, row, group == FINANCIAL_SOURCE_GROUP
):
encoded = _json_bytes({"year": year, "row": annual_row}).decode(
"utf-8"
)
candidate = {"okpo": okpo or None, group: [example_row]}
if len(_json_bytes(candidate)) <= 32768:
example = candidate
rows.append(
(record.source, record.record_type, str(record.uid), encoded)
)
if example is None and len(encoded.encode("utf-8")) <= 32768:
example_row = dict(annual_row)
okpo = example_row.pop("ОКПО", None) or example_row.pop(
"okpo", None
)
candidate = {"okpo": okpo or None, group: [example_row]}
if len(_json_bytes(candidate)) <= 32768:
example, example_key = candidate, _year_key(year)
rows.append(
(
_year_key(year),
record.source,
record.record_type,
str(record.uid),
encoded,
)
)
count += 1
_check_capacity(path.parent)
db.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", rows)
count += len(rows)
db.executemany("INSERT INTO rows VALUES (?, ?, ?, ?, ?)", rows)
db.commit()
db.execute("CREATE INDEX row_order ON rows(source, kind, uid)")
db.execute("CREATE INDEX row_order ON rows(partition, source, kind, uid)")
with path.open("w", encoding="utf-8") as output:
for (data,) in db.execute(
"SELECT data FROM rows ORDER BY source, kind, uid"
"SELECT data FROM rows ORDER BY partition, source, kind, uid"
):
output.write(data + "\n")
if group == SourceGroup.SANCTIONS:
@@ -502,16 +552,18 @@ def _spool(group, path):
*SOURCE_RECORD_EXPORT_FIELDS,
*sorted(headers),
]
return fields, count, example, as_of, revision
return fields, count, example, as_of, revision, example_key
finally:
db.close()
ordered.unlink(missing_ok=True)
def _render_part(group, spool, directory, headers, count, formats, serial):
def _render_part(group, spool, directory, headers, count, formats, serial, year=None):
"""Render then split every variant together when actual bytes exceed the cap."""
max_bytes = min(_limit("MAX_FILE_BYTES", 512 * 1024 * 1024), 512 * 1024 * 1024)
stem = SOURCE_GROUP_EXPORT_FILE_STEMS[group]
stem = SOURCE_GROUP_EXPORT_FILE_STEMS[group] + (
f"-{year:04d}" if year else "-undated"
)
files = []
for fmt in formats:
_check_capacity(directory)
@@ -544,7 +596,7 @@ def _render_part(group, spool, directory, headers, count, formats, serial):
_json_bytes(row).decode("utf-8") + "\n"
)
parts = _render_part(
group, halves[0], directory, headers, count // 2, formats, serial
group, halves[0], directory, headers, count // 2, formats, serial, year
)
parts.extend(
_render_part(
@@ -555,6 +607,7 @@ def _render_part(group, spool, directory, headers, count, formats, serial):
count - count // 2,
formats,
serial + len(parts),
year,
)
)
return parts
@@ -578,7 +631,8 @@ def _render_part(group, spool, directory, headers, count, formats, serial):
return [
{
"part_id": str(uuid4()),
"dataset_key": "records",
"dataset_key": _year_key(year),
"year": year,
"part_number": serial,
"parts_count": 0,
"title": "",
@@ -589,7 +643,7 @@ def _render_part(group, spool, directory, headers, count, formats, serial):
]
def _render_parts(group, staging, spool, headers, formats):
def _render_year_parts(group, staging, annual_rows, headers, formats, year):
max_rows = min(
_limit("ROWS_PER_PART", 100000),
_limit(
@@ -606,29 +660,28 @@ def _render_parts(group, staging, spool, headers, formats):
part_spool = staging / ".part.jsonl"
output = part_spool.open("w", encoding="utf-8")
try:
with spool.open(encoding="utf-8") as source:
for line in source:
size = len(line.encode("utf-8"))
if batch_count and (
batch_count >= max_rows or batch_bytes + size > target
):
output.close()
parts.extend(
_render_part(
group,
part_spool,
staging,
headers,
batch_count,
formats,
len(parts) + 1,
)
for row in annual_rows:
line = _json_bytes(row).decode("utf-8") + "\n"
size = len(line.encode("utf-8"))
if batch_count and (batch_count >= max_rows or batch_bytes + size > target):
output.close()
parts.extend(
_render_part(
group,
part_spool,
staging,
headers,
batch_count,
formats,
len(parts) + 1,
year,
)
output = part_spool.open("w", encoding="utf-8")
batch_count, batch_bytes = 0, 0
output.write(line)
batch_count += 1
batch_bytes += size
)
output = part_spool.open("w", encoding="utf-8")
batch_count, batch_bytes = 0, 0
output.write(line)
batch_count += 1
batch_bytes += size
output.close()
if batch_count:
parts.extend(
@@ -640,6 +693,7 @@ def _render_parts(group, staging, spool, headers, formats):
batch_count,
formats,
len(parts) + 1,
year,
)
)
finally:
@@ -648,6 +702,51 @@ def _render_parts(group, staging, spool, headers, formats):
return parts
def _render_parts(group, staging, spool, headers, formats):
from itertools import groupby
parts = []
with spool.open(encoding="utf-8") as source:
envelopes = (json.loads(line) for line in source)
for year, rows in groupby(envelopes, key=lambda item: item["year"]):
annual = _render_year_parts(
group, staging, (item["row"] for item in rows), headers, formats, year
)
title = _entry(group)["title"] + (
f" — {year:04d} год" if year else " — год не определён"
)
for part in annual:
part["parts_count"] = len(annual)
part["title"] = title + (
f" — часть {part['part_number']} из {len(annual)}"
if len(annual) > 1
else ""
)
parts.extend(annual)
return parts
def _annual_datasets(parts, title):
from itertools import groupby
datasets = []
for key, grouped in groupby(parts, key=lambda p: p["dataset_key"]):
rows = list(grouped)
year = rows[0]["year"]
datasets.append(
{
"key": key,
"year": year,
"title": title
+ (f" — {year:04d} год" if year else " — год не определён"),
"export_schema_version": "2",
"records_count": sum(p["records_count"] for p in rows),
"parts_count": len(rows),
}
)
return datasets
def build_source(group):
entry = _entry(group)
if not entry["export_enabled"]:
@@ -665,20 +764,15 @@ def build_source(group):
raise OSError("Export disk reserve exhausted")
staging.mkdir(mode=0o700)
spool = staging / ".rows.jsonl"
headers, count, example, as_of, revision = _spool(group, spool)
headers, count, example, as_of, revision, example_key = _spool(group, spool)
formats = ("json",) if group == FINANCIAL_SOURCE_GROUP else FORMATS
parts = _render_parts(group, staging, spool, headers, formats)
spool.unlink()
for part in parts:
part["parts_count"] = len(parts)
part["title"] = entry["title"] + (
f" — часть {part['part_number']} из {len(parts)}"
if len(parts) > 1
else ""
)
_check_capacity(staging)
manifest = {
"version": 3,
"version": 4,
"partitioning": "year",
"exported_records_count": sum(p["records_count"] for p in parts),
"data_scope": "all_available",
"source_group": group,
"generation_id": str(uid),
@@ -686,16 +780,8 @@ def build_source(group):
"data_as_of": as_of,
"records_count": count,
"supported_formats": list(formats) if count else [],
"description": _description(entry, example),
"datasets": [
{
"key": "records",
"title": entry["title"],
"export_schema_version": "1",
"records_count": count,
"parts_count": len(parts),
}
],
"description": _description(entry, example, example_key),
"datasets": _annual_datasets(parts, entry["title"]),
"parts": parts,
}
(staging / "manifest.json").write_bytes(_json_bytes(manifest))
@@ -915,7 +1001,9 @@ def _stream_zip(opened, manifest):
) as archive:
for artifact, handle in opened:
with archive.open(
"records/" + artifact["file_name"], "w", force_zip64=True
artifact.get("archive_path", "records/" + artifact["file_name"]),
"w",
force_zip64=True,
) as target:
yield from sink.drain()
transferred = 0
@@ -1092,6 +1180,24 @@ def _admission_error(generation, lease, now):
def _prepare_stream(generation, lease, payload, stack, now):
opened = _open_artifacts(generation, payload["artifacts"], stack)
if generation.manifest.get("partitioning") == "year":
years = {
v["artifact_id"]: f"{p['year']:04d}" if p["year"] else "undated"
for p in generation.manifest["parts"]
for v in p["variants"]
}
opened = [
(
{
**artifact,
"archive_path": years[artifact["artifact_id"]]
+ "/"
+ artifact["file_name"],
},
handle,
)
for artifact, handle in opened
]
lease.kind, lease.expires_at = "stream", now + timedelta(minutes=10)
lease.save(update_fields=["kind", "expires_at"])
manifest = {

View File

@@ -128,7 +128,7 @@ def refresh_organization_data_snapshots_for_parser_batch(
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_export_catalog(self, source_group=None):
"""Independent group publication; contention retries the latest canonical data."""
"""Build all sources by year; independently publish each complete group."""
from organizations.source_export_catalog import (
ExportError,
build_source,