feat: add versioned all-available source export catalog

This commit is contained in:
Aleksandr Meshchryakov
2026-10-22 21:30:18 +02:00
parent 1ffdcd0aaf
commit 63872e86fe
28 changed files with 5754 additions and 240 deletions

View File

@@ -3,6 +3,7 @@ import { readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import converter from 'swagger2openapi';
import { addSourceExportCatalog } from './source-export-catalog.mjs';
import validator from 'oas-validator';
const base = fileURLToPath(new URL('../..', import.meta.url));
@@ -236,6 +237,7 @@ function removeEmptyRequired(value) {
if (Array.isArray(value.required) && value.required.length === 0) delete value.required;
for (const child of Object.values(value)) removeEmptyRequired(child);
}
addSourceExportCatalog(document);
removeEmptyRequired(document);
const encoded = JSON.stringify(document, null, 2) + '\n';

View File

@@ -0,0 +1,26 @@
/** Validate actual new-scope responses using the generated client validators. */
import { readFileSync } from 'node:fs';
import { isDeepStrictEqual } from 'node:util';
import {
V2OrganizationSourceRecordsExportCatalogList200Response,
V2OrganizationSourceRecordsExportCatalogRead200Response,
V2OrganizationSourceRecordsExportTicket201Response,
V2OrganizationSourceRecordsExportCatalogList400Response,
} from './generated-zod/contract.ts';
const validators = {
catalog: V2OrganizationSourceRecordsExportCatalogList200Response,
detail: V2OrganizationSourceRecordsExportCatalogRead200Response,
'file-ticket': V2OrganizationSourceRecordsExportTicket201Response,
'archive-ticket': V2OrganizationSourceRecordsExportTicket201Response,
invalid: V2OrganizationSourceRecordsExportCatalogList400Response,
};
const cases = JSON.parse(readFileSync(process.argv[2], 'utf8'));
const seen = new Set();
for (const fixture of cases) {
if (seen.has(fixture.name) || !validators[fixture.name]) throw new Error('Unexpected fixture');
seen.add(fixture.name);
const value = validators[fixture.name].parse(fixture.body);
if (!isDeepStrictEqual(value, fixture.body)) throw new Error(`Lossy generated validator: ${fixture.name}`);
}
if (seen.size !== Object.keys(validators).length) throw new Error('Missing fixtures');
console.log(`Source export generated Zod: ${seen.size} actual API responses passed`);

View File

@@ -39,3 +39,21 @@ export function total(response: OrganizationSourceRecordListResponse): number {
}
export const existingOrder: V2OrganizationSourceRecordsListOrdering = '-payload__united_states';
import { v2OrganizationSourceRecordsExportTicket } from './generated/client';
import type { SourceExportFileTicketRequest, SourceExportArchiveTicketRequest, OrganizationSourceRecordExportRequest } from './generated/schemas';
// All three selectors must remain callable through the actual generated function.
export async function exportSelection(selector: SourceExportFileTicketRequest | SourceExportArchiveTicketRequest | OrganizationSourceRecordExportRequest): Promise<string> {
const response = await v2OrganizationSourceRecordsExportTicket(selector);
if (response.status === 201) {
const ticket: string = response.data.ticket;
if ('selection_kind' in response.data) {
const format: 'xlsx' | 'csv' | 'json' = response.data.format;
const size: number | null = response.data.size_bytes;
return `${ticket}:${format}:${size}`;
}
return ticket;
}
return '';
}

View File

@@ -13,7 +13,14 @@ export default defineConfig({
output: {
mode: 'single', target: './generated-zod/contract.ts',
client: 'zod', clean: true,
override: { zod: { generateEachHttpStatus: true } },
override: {
zod: { generateEachHttpStatus: true },
operations: {
v2_organization_source_records_export_ticket: {
zod: { strict: { body: true, response: true } },
},
},
},
},
},
});

View File

@@ -0,0 +1,76 @@
/** Typed all-available contract, composed with generated legacy DRF operations. */
export function addSourceExportCatalog(document) {
const schemas = document.components.schemas;
const ref = name => ({ $ref: `#/components/schemas/${name}` });
const string = { type: 'string' };
const uuid = { type: 'string', format: 'uuid' };
const date = { type: 'string', format: 'date-time' };
const integer = { type: 'integer', minimum: 0, maximum: 9007199254740991 };
const positive = { ...integer, minimum: 1 };
const bool = { type: 'boolean' };
const enumeration = values => ({ type: 'string', enum: values });
const format = enumeration(['xlsx', 'csv', 'json']);
const nullable = schema => ({ ...schema, nullable: true });
const nil = { type: 'object', nullable: true, enum: [null] };
const array = items => ({ type: 'array', items });
const object = properties => ({ type: 'object', additionalProperties: false, properties, required: Object.keys(properties) });
const group = structuredClone(schemas.OrganizationSourceRecordExportRequest.properties.sources.items);
schemas.SourceExportBuildSummary = object({ state: enumeration(['idle', 'queued', 'running', 'success', 'failed']), started_at: nullable(date), finished_at: nullable(date), error_code: nullable(string) });
const snapshot = {
source_group: group, generation_id: nullable(uuid),
state: enumeration(['ready', 'empty', 'not_ready', 'building', 'unavailable']),
unavailable_reason: nullable(enumeration(['disabled', 'adapter_missing', 'build_failed', 'storage_unavailable'])),
generated_at: nullable(date), data_as_of: nullable(date), data_scope: enumeration(['all_available']),
};
schemas.SourceExportSourceSummary = object({ ...snapshot, source_slug: string, title: string, order: integer, export_enabled: bool,
records_count: nullable(integer), parts_count: nullable(integer), supported_formats: { ...array(format), uniqueItems: true },
default_format: nullable(format), is_stale: bool, last_build: ref('SourceExportBuildSummary') });
const envelope = (data, meta = nil) => object({ success: { type: 'boolean', enum: [true] }, data, errors: nil, meta });
schemas.SourceExportCatalogResponse = envelope(object({ schema_version: enumeration(['1']), catalog_revision: string, sources: array(ref('SourceExportSourceSummary')) }));
schemas.SourceExportDescription = object({ composition: string, origin: string, fields: string, example_dataset_key: nullable(string), example_is_synthetic: bool,
example_record: { type: 'object', additionalProperties: true, nullable: true } });
schemas.SourceExportDataset = object({ key: string, title: string, export_schema_version: string, records_count: integer, parts_count: integer });
schemas.SourceExportArtifact = object({ artifact_id: uuid, format, file_name: { type: 'string', pattern: '^[A-Za-z0-9._-]+$' }, mime_type: string,
size_bytes: positive, sha256: { type: 'string', pattern: '^[0-9a-f]{64}$' } });
schemas.SourceExportPart = object({ part_id: uuid, dataset_key: string, title: string, part_number: positive, parts_count: positive, records_count: positive,
default_format: format, variants: { ...array(ref('SourceExportArtifact')), minItems: 1 } });
schemas.SourceExportPagination = object({ page: positive, page_size: { ...positive, maximum: 200 }, total_count: integer, total_pages: positive, has_next: bool, has_previous: bool });
schemas.SourceExportSourceDetail = object({ ...snapshot, current_generation_id: nullable(uuid), description: ref('SourceExportDescription'), datasets: array(ref('SourceExportDataset')), files: array(ref('SourceExportPart')) });
schemas.SourceExportSourceDetailResponse = envelope(ref('SourceExportSourceDetail'), object({ pagination: ref('SourceExportPagination') }));
schemas.SourceExportAuthenticationError = object({ detail: string });
schemas.SourceExportError = object({ detail: string, code: string, field: nullable(string), retryable: bool, request_id: uuid });
schemas.SourceExportFileTicketRequest = object({ selection_kind: enumeration(['file']), source_group: group, generation_id: uuid, part_id: uuid, format });
schemas.SourceExportArchiveTicketRequest = object({ selection_kind: enumeration(['source_archive']), source_group: group, generation_id: uuid, format });
schemas.OrganizationSourceRecordExportRequest.additionalProperties = false;
schemas.SourceRecordExportTicket.additionalProperties = false;
schemas.SourceRecordExportTicket.required = Object.keys(schemas.SourceRecordExportTicket.properties);
schemas.SourceExportSelectionTicket = object({ ticket: { type: 'string', minLength: 47, maxLength: 47 }, file_name: string, expires_in: { type: 'integer', enum: [300] },
selection_kind: enumeration(['file', 'source_archive']), source_group: group, generation_id: uuid, format, mime_type: string, size_bytes: nullable(positive), parts_count: positive });
schemas.SourceRecordExportDownloadRequest = object({ ticket: { type: 'string', minLength: 1, maxLength: 128 } });
const base = '/api/v2/organization-source-records/';
const json = schema => ({ 'application/json': { schema } });
const response = (schema, description = 'Success') => ({ description, content: json(schema) });
const errorResponses = Object.fromEntries([400,403,404,409,410,429,503,500].map(code => [code, response(ref('SourceExportError'), 'Export request failed')]));
for (const [endpoint, schema] of [['export-catalog/', 'SourceExportCatalogResponse'], ['export-catalog/{source_group}/', 'SourceExportSourceDetailResponse']]) {
const operation = document.paths[base + endpoint].get;
operation.responses = { ...errorResponses, 200: response(ref(schema)), 401: response(ref('SourceExportAuthenticationError'), 'Authentication required') };
operation.parameters = endpoint.includes('{') ? [
{ name: 'source_group', in: 'path', required: true, schema: group },
{ name: 'generation_id', in: 'query', schema: uuid },
{ name: 'page', in: 'query', schema: { ...positive, default: 1 } },
{ name: 'page_size', in: 'query', schema: { ...positive, maximum: 200, default: 50 } },
] : [];
}
const ticket = document.paths[base + 'export-ticket/'].post;
ticket.requestBody = { required: true, content: json({ oneOf: ['OrganizationSourceRecordExportRequest', 'SourceExportFileTicketRequest', 'SourceExportArchiveTicketRequest'].map(ref) }) };
ticket.responses = { ...errorResponses, 201: response({ oneOf: ['SourceRecordExportTicket', 'SourceExportSelectionTicket'].map(ref) }), 401: response(ref('SourceExportAuthenticationError'), 'Authentication required') };
const download = document.paths[base + 'export-download/'].post;
download.requestBody = { required: true, content: { 'application/x-www-form-urlencoded': { schema: ref('SourceRecordExportDownloadRequest') } } };
download.responses = { ...errorResponses, 200: { description: 'Pinned artifact or streaming ZIP64', content: Object.fromEntries([
'application/zip', 'application/json', 'text/csv', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
].map(mime => [mime, { schema: { type: 'string', format: 'binary' } }])) } };
for (const [status, legacy] of [[400, 'SourceRecordExportInvalidTicket'], [410, 'SourceRecordExportTicketExpired'], [503, 'SourceRecordExportUnavailable']]) {
download.responses[status] = response({ oneOf: [ref(legacy), ref('SourceExportError')] }, 'Legacy or all-available export failure');
}
}