feat: partition all source exports by year and enable nightly builds

This commit is contained in:
Aleksandr Meshchryakov
2026-10-22 21:45:29 +02:00
parent 63872e86fe
commit 2002e80196
13 changed files with 672 additions and 87 deletions

View File

@@ -57,3 +57,11 @@ export async function exportSelection(selector: SourceExportFileTicketRequest |
}
return '';
}
import type { SourceExportPart, SourceExportSourceSummary } from './generated/schemas';
export function sourceExportYear(part: SourceExportPart, source: SourceExportSourceSummary): string {
const year: number | null = part.year;
const partitioning: 'year' | 'none' = source.partitioning;
const projections: number | null = source.exported_records_count;
return `${partitioning}:${year ?? 'undated'}:${projections}`;
}

View File

@@ -7,6 +7,7 @@ export function addSourceExportCatalog(document) {
const date = { type: 'string', format: 'date-time' };
const integer = { type: 'integer', minimum: 0, maximum: 9007199254740991 };
const positive = { ...integer, minimum: 1 };
const year = { type: 'integer', minimum: 1, maximum: 9999, nullable: true };
const bool = { type: 'boolean' };
const enumeration = values => ({ type: 'string', enum: values });
const format = enumeration(['xlsx', 'csv', 'json']);
@@ -20,19 +21,19 @@ export function addSourceExportCatalog(document) {
source_group: group, generation_id: nullable(uuid),
state: enumeration(['ready', 'empty', 'not_ready', 'building', 'unavailable']),
unavailable_reason: nullable(enumeration(['disabled', 'adapter_missing', 'build_failed', 'storage_unavailable'])),
generated_at: nullable(date), data_as_of: nullable(date), data_scope: enumeration(['all_available']),
generated_at: nullable(date), data_as_of: nullable(date), data_scope: enumeration(['all_available']), partitioning: enumeration(['year', 'none']),
};
schemas.SourceExportSourceSummary = object({ ...snapshot, source_slug: string, title: string, order: integer, export_enabled: bool,
records_count: nullable(integer), parts_count: nullable(integer), supported_formats: { ...array(format), uniqueItems: true },
records_count: nullable(integer), exported_records_count: nullable(integer), parts_count: nullable(integer), supported_formats: { ...array(format), uniqueItems: true },
default_format: nullable(format), is_stale: bool, last_build: ref('SourceExportBuildSummary') });
const envelope = (data, meta = nil) => object({ success: { type: 'boolean', enum: [true] }, data, errors: nil, meta });
schemas.SourceExportCatalogResponse = envelope(object({ schema_version: enumeration(['1']), catalog_revision: string, sources: array(ref('SourceExportSourceSummary')) }));
schemas.SourceExportCatalogResponse = envelope(object({ schema_version: enumeration(['2']), catalog_revision: string, sources: array(ref('SourceExportSourceSummary')) }));
schemas.SourceExportDescription = object({ composition: string, origin: string, fields: string, example_dataset_key: nullable(string), example_is_synthetic: bool,
example_record: { type: 'object', additionalProperties: true, nullable: true } });
schemas.SourceExportDataset = object({ key: string, title: string, export_schema_version: string, records_count: integer, parts_count: integer });
schemas.SourceExportDataset = object({ key: string, year, title: string, export_schema_version: string, records_count: integer, parts_count: integer });
schemas.SourceExportArtifact = object({ artifact_id: uuid, format, file_name: { type: 'string', pattern: '^[A-Za-z0-9._-]+$' }, mime_type: string,
size_bytes: positive, sha256: { type: 'string', pattern: '^[0-9a-f]{64}$' } });
schemas.SourceExportPart = object({ part_id: uuid, dataset_key: string, title: string, part_number: positive, parts_count: positive, records_count: positive,
schemas.SourceExportPart = object({ part_id: uuid, year, dataset_key: string, title: string, part_number: positive, parts_count: positive, records_count: positive,
default_format: format, variants: { ...array(ref('SourceExportArtifact')), minItems: 1 } });
schemas.SourceExportPagination = object({ page: positive, page_size: { ...positive, maximum: 200 }, total_count: integer, total_pages: positive, has_next: bool, has_previous: bool });
schemas.SourceExportSourceDetail = object({ ...snapshot, current_generation_id: nullable(uuid), description: ref('SourceExportDescription'), datasets: array(ref('SourceExportDataset')), files: array(ref('SourceExportPart')) });