feat: add versioned all-available source export catalog
This commit is contained in:
@@ -3,6 +3,7 @@ import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import converter from 'swagger2openapi';
|
||||
import { addSourceExportCatalog } from './source-export-catalog.mjs';
|
||||
import validator from 'oas-validator';
|
||||
|
||||
const base = fileURLToPath(new URL('../..', import.meta.url));
|
||||
@@ -236,6 +237,7 @@ function removeEmptyRequired(value) {
|
||||
if (Array.isArray(value.required) && value.required.length === 0) delete value.required;
|
||||
for (const child of Object.values(value)) removeEmptyRequired(child);
|
||||
}
|
||||
addSourceExportCatalog(document);
|
||||
removeEmptyRequired(document);
|
||||
|
||||
const encoded = JSON.stringify(document, null, 2) + '\n';
|
||||
|
||||
26
tools/openapi/check-source-export-runtime.mjs
Normal file
26
tools/openapi/check-source-export-runtime.mjs
Normal file
@@ -0,0 +1,26 @@
|
||||
/** Validate actual new-scope responses using the generated client validators. */
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import {
|
||||
V2OrganizationSourceRecordsExportCatalogList200Response,
|
||||
V2OrganizationSourceRecordsExportCatalogRead200Response,
|
||||
V2OrganizationSourceRecordsExportTicket201Response,
|
||||
V2OrganizationSourceRecordsExportCatalogList400Response,
|
||||
} from './generated-zod/contract.ts';
|
||||
const validators = {
|
||||
catalog: V2OrganizationSourceRecordsExportCatalogList200Response,
|
||||
detail: V2OrganizationSourceRecordsExportCatalogRead200Response,
|
||||
'file-ticket': V2OrganizationSourceRecordsExportTicket201Response,
|
||||
'archive-ticket': V2OrganizationSourceRecordsExportTicket201Response,
|
||||
invalid: V2OrganizationSourceRecordsExportCatalogList400Response,
|
||||
};
|
||||
const cases = JSON.parse(readFileSync(process.argv[2], 'utf8'));
|
||||
const seen = new Set();
|
||||
for (const fixture of cases) {
|
||||
if (seen.has(fixture.name) || !validators[fixture.name]) throw new Error('Unexpected fixture');
|
||||
seen.add(fixture.name);
|
||||
const value = validators[fixture.name].parse(fixture.body);
|
||||
if (!isDeepStrictEqual(value, fixture.body)) throw new Error(`Lossy generated validator: ${fixture.name}`);
|
||||
}
|
||||
if (seen.size !== Object.keys(validators).length) throw new Error('Missing fixtures');
|
||||
console.log(`Source export generated Zod: ${seen.size} actual API responses passed`);
|
||||
@@ -39,3 +39,21 @@ export function total(response: OrganizationSourceRecordListResponse): number {
|
||||
}
|
||||
|
||||
export const existingOrder: V2OrganizationSourceRecordsListOrdering = '-payload__united_states';
|
||||
|
||||
import { v2OrganizationSourceRecordsExportTicket } from './generated/client';
|
||||
import type { SourceExportFileTicketRequest, SourceExportArchiveTicketRequest, OrganizationSourceRecordExportRequest } from './generated/schemas';
|
||||
|
||||
// All three selectors must remain callable through the actual generated function.
|
||||
export async function exportSelection(selector: SourceExportFileTicketRequest | SourceExportArchiveTicketRequest | OrganizationSourceRecordExportRequest): Promise<string> {
|
||||
const response = await v2OrganizationSourceRecordsExportTicket(selector);
|
||||
if (response.status === 201) {
|
||||
const ticket: string = response.data.ticket;
|
||||
if ('selection_kind' in response.data) {
|
||||
const format: 'xlsx' | 'csv' | 'json' = response.data.format;
|
||||
const size: number | null = response.data.size_bytes;
|
||||
return `${ticket}:${format}:${size}`;
|
||||
}
|
||||
return ticket;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
@@ -13,7 +13,14 @@ export default defineConfig({
|
||||
output: {
|
||||
mode: 'single', target: './generated-zod/contract.ts',
|
||||
client: 'zod', clean: true,
|
||||
override: { zod: { generateEachHttpStatus: true } },
|
||||
override: {
|
||||
zod: { generateEachHttpStatus: true },
|
||||
operations: {
|
||||
v2_organization_source_records_export_ticket: {
|
||||
zod: { strict: { body: true, response: true } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
76
tools/openapi/source-export-catalog.mjs
Normal file
76
tools/openapi/source-export-catalog.mjs
Normal file
@@ -0,0 +1,76 @@
|
||||
/** Typed all-available contract, composed with generated legacy DRF operations. */
|
||||
export function addSourceExportCatalog(document) {
|
||||
const schemas = document.components.schemas;
|
||||
const ref = name => ({ $ref: `#/components/schemas/${name}` });
|
||||
const string = { type: 'string' };
|
||||
const uuid = { type: 'string', format: 'uuid' };
|
||||
const date = { type: 'string', format: 'date-time' };
|
||||
const integer = { type: 'integer', minimum: 0, maximum: 9007199254740991 };
|
||||
const positive = { ...integer, minimum: 1 };
|
||||
const bool = { type: 'boolean' };
|
||||
const enumeration = values => ({ type: 'string', enum: values });
|
||||
const format = enumeration(['xlsx', 'csv', 'json']);
|
||||
const nullable = schema => ({ ...schema, nullable: true });
|
||||
const nil = { type: 'object', nullable: true, enum: [null] };
|
||||
const array = items => ({ type: 'array', items });
|
||||
const object = properties => ({ type: 'object', additionalProperties: false, properties, required: Object.keys(properties) });
|
||||
const group = structuredClone(schemas.OrganizationSourceRecordExportRequest.properties.sources.items);
|
||||
schemas.SourceExportBuildSummary = object({ state: enumeration(['idle', 'queued', 'running', 'success', 'failed']), started_at: nullable(date), finished_at: nullable(date), error_code: nullable(string) });
|
||||
const snapshot = {
|
||||
source_group: group, generation_id: nullable(uuid),
|
||||
state: enumeration(['ready', 'empty', 'not_ready', 'building', 'unavailable']),
|
||||
unavailable_reason: nullable(enumeration(['disabled', 'adapter_missing', 'build_failed', 'storage_unavailable'])),
|
||||
generated_at: nullable(date), data_as_of: nullable(date), data_scope: enumeration(['all_available']),
|
||||
};
|
||||
schemas.SourceExportSourceSummary = object({ ...snapshot, source_slug: string, title: string, order: integer, export_enabled: bool,
|
||||
records_count: nullable(integer), parts_count: nullable(integer), supported_formats: { ...array(format), uniqueItems: true },
|
||||
default_format: nullable(format), is_stale: bool, last_build: ref('SourceExportBuildSummary') });
|
||||
const envelope = (data, meta = nil) => object({ success: { type: 'boolean', enum: [true] }, data, errors: nil, meta });
|
||||
schemas.SourceExportCatalogResponse = envelope(object({ schema_version: enumeration(['1']), catalog_revision: string, sources: array(ref('SourceExportSourceSummary')) }));
|
||||
schemas.SourceExportDescription = object({ composition: string, origin: string, fields: string, example_dataset_key: nullable(string), example_is_synthetic: bool,
|
||||
example_record: { type: 'object', additionalProperties: true, nullable: true } });
|
||||
schemas.SourceExportDataset = object({ key: string, title: string, export_schema_version: string, records_count: integer, parts_count: integer });
|
||||
schemas.SourceExportArtifact = object({ artifact_id: uuid, format, file_name: { type: 'string', pattern: '^[A-Za-z0-9._-]+$' }, mime_type: string,
|
||||
size_bytes: positive, sha256: { type: 'string', pattern: '^[0-9a-f]{64}$' } });
|
||||
schemas.SourceExportPart = object({ part_id: uuid, dataset_key: string, title: string, part_number: positive, parts_count: positive, records_count: positive,
|
||||
default_format: format, variants: { ...array(ref('SourceExportArtifact')), minItems: 1 } });
|
||||
schemas.SourceExportPagination = object({ page: positive, page_size: { ...positive, maximum: 200 }, total_count: integer, total_pages: positive, has_next: bool, has_previous: bool });
|
||||
schemas.SourceExportSourceDetail = object({ ...snapshot, current_generation_id: nullable(uuid), description: ref('SourceExportDescription'), datasets: array(ref('SourceExportDataset')), files: array(ref('SourceExportPart')) });
|
||||
schemas.SourceExportSourceDetailResponse = envelope(ref('SourceExportSourceDetail'), object({ pagination: ref('SourceExportPagination') }));
|
||||
schemas.SourceExportAuthenticationError = object({ detail: string });
|
||||
schemas.SourceExportError = object({ detail: string, code: string, field: nullable(string), retryable: bool, request_id: uuid });
|
||||
schemas.SourceExportFileTicketRequest = object({ selection_kind: enumeration(['file']), source_group: group, generation_id: uuid, part_id: uuid, format });
|
||||
schemas.SourceExportArchiveTicketRequest = object({ selection_kind: enumeration(['source_archive']), source_group: group, generation_id: uuid, format });
|
||||
schemas.OrganizationSourceRecordExportRequest.additionalProperties = false;
|
||||
schemas.SourceRecordExportTicket.additionalProperties = false;
|
||||
schemas.SourceRecordExportTicket.required = Object.keys(schemas.SourceRecordExportTicket.properties);
|
||||
schemas.SourceExportSelectionTicket = object({ ticket: { type: 'string', minLength: 47, maxLength: 47 }, file_name: string, expires_in: { type: 'integer', enum: [300] },
|
||||
selection_kind: enumeration(['file', 'source_archive']), source_group: group, generation_id: uuid, format, mime_type: string, size_bytes: nullable(positive), parts_count: positive });
|
||||
schemas.SourceRecordExportDownloadRequest = object({ ticket: { type: 'string', minLength: 1, maxLength: 128 } });
|
||||
const base = '/api/v2/organization-source-records/';
|
||||
const json = schema => ({ 'application/json': { schema } });
|
||||
const response = (schema, description = 'Success') => ({ description, content: json(schema) });
|
||||
const errorResponses = Object.fromEntries([400,403,404,409,410,429,503,500].map(code => [code, response(ref('SourceExportError'), 'Export request failed')]));
|
||||
for (const [endpoint, schema] of [['export-catalog/', 'SourceExportCatalogResponse'], ['export-catalog/{source_group}/', 'SourceExportSourceDetailResponse']]) {
|
||||
const operation = document.paths[base + endpoint].get;
|
||||
operation.responses = { ...errorResponses, 200: response(ref(schema)), 401: response(ref('SourceExportAuthenticationError'), 'Authentication required') };
|
||||
operation.parameters = endpoint.includes('{') ? [
|
||||
{ name: 'source_group', in: 'path', required: true, schema: group },
|
||||
{ name: 'generation_id', in: 'query', schema: uuid },
|
||||
{ name: 'page', in: 'query', schema: { ...positive, default: 1 } },
|
||||
{ name: 'page_size', in: 'query', schema: { ...positive, maximum: 200, default: 50 } },
|
||||
] : [];
|
||||
}
|
||||
const ticket = document.paths[base + 'export-ticket/'].post;
|
||||
ticket.requestBody = { required: true, content: json({ oneOf: ['OrganizationSourceRecordExportRequest', 'SourceExportFileTicketRequest', 'SourceExportArchiveTicketRequest'].map(ref) }) };
|
||||
ticket.responses = { ...errorResponses, 201: response({ oneOf: ['SourceRecordExportTicket', 'SourceExportSelectionTicket'].map(ref) }), 401: response(ref('SourceExportAuthenticationError'), 'Authentication required') };
|
||||
const download = document.paths[base + 'export-download/'].post;
|
||||
download.requestBody = { required: true, content: { 'application/x-www-form-urlencoded': { schema: ref('SourceRecordExportDownloadRequest') } } };
|
||||
download.responses = { ...errorResponses, 200: { description: 'Pinned artifact or streaming ZIP64', content: Object.fromEntries([
|
||||
'application/zip', 'application/json', 'text/csv', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
].map(mime => [mime, { schema: { type: 'string', format: 'binary' } }])) } };
|
||||
for (const [status, legacy] of [[400, 'SourceRecordExportInvalidTicket'], [410, 'SourceRecordExportTicketExpired'], [503, 'SourceRecordExportUnavailable']]) {
|
||||
download.responses[status] = response({ oneOf: [ref(legacy), ref('SourceExportError')] }, 'Legacy or all-available export failure');
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user