feat: partition all source exports by year and enable nightly builds
This commit is contained in:
@@ -7,6 +7,7 @@ export function addSourceExportCatalog(document) {
|
||||
const date = { type: 'string', format: 'date-time' };
|
||||
const integer = { type: 'integer', minimum: 0, maximum: 9007199254740991 };
|
||||
const positive = { ...integer, minimum: 1 };
|
||||
const year = { type: 'integer', minimum: 1, maximum: 9999, nullable: true };
|
||||
const bool = { type: 'boolean' };
|
||||
const enumeration = values => ({ type: 'string', enum: values });
|
||||
const format = enumeration(['xlsx', 'csv', 'json']);
|
||||
@@ -20,19 +21,19 @@ export function addSourceExportCatalog(document) {
|
||||
source_group: group, generation_id: nullable(uuid),
|
||||
state: enumeration(['ready', 'empty', 'not_ready', 'building', 'unavailable']),
|
||||
unavailable_reason: nullable(enumeration(['disabled', 'adapter_missing', 'build_failed', 'storage_unavailable'])),
|
||||
generated_at: nullable(date), data_as_of: nullable(date), data_scope: enumeration(['all_available']),
|
||||
generated_at: nullable(date), data_as_of: nullable(date), data_scope: enumeration(['all_available']), partitioning: enumeration(['year', 'none']),
|
||||
};
|
||||
schemas.SourceExportSourceSummary = object({ ...snapshot, source_slug: string, title: string, order: integer, export_enabled: bool,
|
||||
records_count: nullable(integer), parts_count: nullable(integer), supported_formats: { ...array(format), uniqueItems: true },
|
||||
records_count: nullable(integer), exported_records_count: nullable(integer), parts_count: nullable(integer), supported_formats: { ...array(format), uniqueItems: true },
|
||||
default_format: nullable(format), is_stale: bool, last_build: ref('SourceExportBuildSummary') });
|
||||
const envelope = (data, meta = nil) => object({ success: { type: 'boolean', enum: [true] }, data, errors: nil, meta });
|
||||
schemas.SourceExportCatalogResponse = envelope(object({ schema_version: enumeration(['1']), catalog_revision: string, sources: array(ref('SourceExportSourceSummary')) }));
|
||||
schemas.SourceExportCatalogResponse = envelope(object({ schema_version: enumeration(['2']), catalog_revision: string, sources: array(ref('SourceExportSourceSummary')) }));
|
||||
schemas.SourceExportDescription = object({ composition: string, origin: string, fields: string, example_dataset_key: nullable(string), example_is_synthetic: bool,
|
||||
example_record: { type: 'object', additionalProperties: true, nullable: true } });
|
||||
schemas.SourceExportDataset = object({ key: string, title: string, export_schema_version: string, records_count: integer, parts_count: integer });
|
||||
schemas.SourceExportDataset = object({ key: string, year, title: string, export_schema_version: string, records_count: integer, parts_count: integer });
|
||||
schemas.SourceExportArtifact = object({ artifact_id: uuid, format, file_name: { type: 'string', pattern: '^[A-Za-z0-9._-]+$' }, mime_type: string,
|
||||
size_bytes: positive, sha256: { type: 'string', pattern: '^[0-9a-f]{64}$' } });
|
||||
schemas.SourceExportPart = object({ part_id: uuid, dataset_key: string, title: string, part_number: positive, parts_count: positive, records_count: positive,
|
||||
schemas.SourceExportPart = object({ part_id: uuid, year, dataset_key: string, title: string, part_number: positive, parts_count: positive, records_count: positive,
|
||||
default_format: format, variants: { ...array(ref('SourceExportArtifact')), minItems: 1 } });
|
||||
schemas.SourceExportPagination = object({ page: positive, page_size: { ...positive, maximum: 200 }, total_count: integer, total_pages: positive, has_next: bool, has_previous: bool });
|
||||
schemas.SourceExportSourceDetail = object({ ...snapshot, current_generation_id: nullable(uuid), description: ref('SourceExportDescription'), datasets: array(ref('SourceExportDataset')), files: array(ref('SourceExportPart')) });
|
||||
|
||||
Reference in New Issue
Block a user