// Validates the contribution data: models/.json registrations and // their artifacts under outputs/. Runs in CI for every PR and push, so a // malformed or undocumented contribution fails before it can ship. import { existsSync, readFileSync, readdirSync } from 'node:fs' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' const root = join(dirname(fileURLToPath(import.meta.url)), '..') const modelsDir = join(root, 'models') const outputsDir = join(root, 'outputs') const cases = JSON.parse(readFileSync(join(root, 'cases.json'), 'utf8')) as Array<{ id: string kind: string }> const caseKind = new Map(cases.map((c) => [c.id, c.kind])) const ID_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/ const errors: string[] = [] const isBilingual = (v: unknown): boolean => { if (!v || typeof v !== 'object') return false const { zh, en } = v as { zh?: unknown; en?: unknown } return typeof zh === 'string' && zh.trim() !== '' && typeof en === 'string' && en.trim() !== '' } const modelIds = new Set() for (const f of readdirSync(modelsDir).filter((f) => f.endsWith('.json'))) { const id = f.replace(/\.json$/, '') modelIds.add(id) if (!ID_RE.test(id)) { errors.push(`models/${f}: id "${id}" must be lowercase letters/digits/dashes only (it doubles as the outputs/ folder name)`) } type RunJson = { note?: unknown; file?: unknown; contributor?: unknown } let def: { label?: unknown; vendor?: unknown; order?: unknown; runs?: Record } try { def = JSON.parse(readFileSync(join(modelsDir, f), 'utf8')) } catch (e) { errors.push(`models/${f}: invalid JSON (${e})`) continue } if (typeof def.label !== 'string' || !def.label.trim()) errors.push(`models/${f}: "label" is required`) if (typeof def.vendor !== 'string' || !def.vendor.trim()) errors.push(`models/${f}: "vendor" is required`) if (def.order !== undefined && typeof def.order !== 'number') errors.push(`models/${f}: "order" must be a number`) for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) { if (!caseKind.has(caseId)) { errors.push(`models/${f}: unknown case "${caseId}" (known: ${[...caseKind.keys()].join(', ')})`) continue } const variants = Array.isArray(runRaw) ? runRaw : [runRaw] if (!variants.length) errors.push(`models/${f}: runs.${caseId} is an empty array`) const seenFiles = new Set() variants.forEach((run, i) => { const where = variants.length > 1 ? `runs.${caseId}[${i}]` : `runs.${caseId}` if (!isBilingual(run?.note)) { errors.push(`models/${f}: ${where} needs a bilingual "note" ({ "zh", "en" }) documenting provenance (tool, effort level, one-shot or fixed)`) } if (run?.file !== undefined && typeof run.file !== 'string') { errors.push(`models/${f}: ${where}.file must be a string`) } if (run?.contributor !== undefined && (typeof run.contributor !== 'string' || !run.contributor.trim())) { errors.push(`models/${f}: ${where}.contributor must be a non-empty GitHub username`) } const file = (typeof run?.file === 'string' && run.file) || `${caseId}.${caseKind.get(caseId)}` if (seenFiles.has(file)) { errors.push(`models/${f}: ${where} resolves to the same artifact "${file}" as another variant — extra variants must set a distinct "file"`) } seenFiles.add(file) if (!existsSync(join(outputsDir, id, file))) { errors.push(`models/${f}: ${where} declared but artifact missing: outputs/${id}/${file}`) } }) } } for (const entry of readdirSync(outputsDir, { withFileTypes: true })) { if (entry.isDirectory() && !modelIds.has(entry.name)) { errors.push(`outputs/${entry.name}/ exists but models/${entry.name}.json is missing`) } } // Every file under outputs// must be declared by a run — loose artifacts // would otherwise ship without provenance. for (const id of modelIds) { const dir = join(outputsDir, id) if (!existsSync(dir)) continue const def = JSON.parse(readFileSync(join(modelsDir, `${id}.json`), 'utf8')) as { runs?: Record> } const declared = new Set() for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) { for (const run of Array.isArray(runRaw) ? runRaw : [runRaw]) { declared.add(run?.file ?? `${caseId}.${caseKind.get(caseId)}`) } } for (const file of readdirSync(dir)) { if (!declared.has(file)) { errors.push(`outputs/${id}/${file} is not declared by any run in models/${id}.json`) } } } if (errors.length) { console.error(`Data validation failed (${errors.length}):\n- ${errors.join('\n- ')}`) process.exit(1) } console.log(`Data OK: ${modelIds.size} models, ${caseKind.size} cases validated`)