Become the data-only repo: site and worker move to private nagi-bench-site
The showcase app and the arena backend now live in the private nagi-bench-site repo and deploy to Cloudflare Pages, fetching this repo's models/, outputs/ and cases.json at build time. CI here is a standalone data validation (same rules as before, no build), plus an optional repository_dispatch that triggers an instant site rebuild when SITE_DISPATCH_TOKEN is configured — otherwise the site's 6-hour cron picks new data up. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ee081a4e34
commit
664a7115c7
5 files changed
+151
-71
No files matched your search
@@ -0,0 +1,110 @@
|
||||
// Validates the contribution data: models/<model-id>.json registrations and
|
||||
// their artifacts under outputs/. Runs in CI for every PR and push, so a
|
||||
// malformed or undocumented contribution fails before it can ship.
|
||||
import { existsSync, readFileSync, readdirSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const modelsDir = join(root, 'models')
|
||||
const outputsDir = join(root, 'outputs')
|
||||
|
||||
const cases = JSON.parse(readFileSync(join(root, 'cases.json'), 'utf8')) as Array<{
|
||||
id: string
|
||||
kind: string
|
||||
}>
|
||||
const caseKind = new Map(cases.map((c) => [c.id, c.kind]))
|
||||
|
||||
const ID_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/
|
||||
const errors: string[] = []
|
||||
|
||||
const isBilingual = (v: unknown): boolean => {
|
||||
if (!v || typeof v !== 'object') return false
|
||||
const { zh, en } = v as { zh?: unknown; en?: unknown }
|
||||
return typeof zh === 'string' && zh.trim() !== '' && typeof en === 'string' && en.trim() !== ''
|
||||
}
|
||||
|
||||
const modelIds = new Set<string>()
|
||||
for (const f of readdirSync(modelsDir).filter((f) => f.endsWith('.json'))) {
|
||||
const id = f.replace(/\.json$/, '')
|
||||
modelIds.add(id)
|
||||
if (!ID_RE.test(id)) {
|
||||
errors.push(`models/${f}: id "${id}" must be lowercase letters/digits/dashes only (it doubles as the outputs/ folder name)`)
|
||||
}
|
||||
|
||||
type RunJson = { note?: unknown; file?: unknown; contributor?: unknown }
|
||||
let def: { label?: unknown; vendor?: unknown; order?: unknown; runs?: Record<string, RunJson | RunJson[]> }
|
||||
try {
|
||||
def = JSON.parse(readFileSync(join(modelsDir, f), 'utf8'))
|
||||
} catch (e) {
|
||||
errors.push(`models/${f}: invalid JSON (${e})`)
|
||||
continue
|
||||
}
|
||||
|
||||
if (typeof def.label !== 'string' || !def.label.trim()) errors.push(`models/${f}: "label" is required`)
|
||||
if (typeof def.vendor !== 'string' || !def.vendor.trim()) errors.push(`models/${f}: "vendor" is required`)
|
||||
if (def.order !== undefined && typeof def.order !== 'number') errors.push(`models/${f}: "order" must be a number`)
|
||||
|
||||
for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) {
|
||||
if (!caseKind.has(caseId)) {
|
||||
errors.push(`models/${f}: unknown case "${caseId}" (known: ${[...caseKind.keys()].join(', ')})`)
|
||||
continue
|
||||
}
|
||||
const variants = Array.isArray(runRaw) ? runRaw : [runRaw]
|
||||
if (!variants.length) errors.push(`models/${f}: runs.${caseId} is an empty array`)
|
||||
const seenFiles = new Set<string>()
|
||||
variants.forEach((run, i) => {
|
||||
const where = variants.length > 1 ? `runs.${caseId}[${i}]` : `runs.${caseId}`
|
||||
if (!isBilingual(run?.note)) {
|
||||
errors.push(`models/${f}: ${where} needs a bilingual "note" ({ "zh", "en" }) documenting provenance (tool, effort level, one-shot or fixed)`)
|
||||
}
|
||||
if (run?.file !== undefined && typeof run.file !== 'string') {
|
||||
errors.push(`models/${f}: ${where}.file must be a string`)
|
||||
}
|
||||
if (run?.contributor !== undefined && (typeof run.contributor !== 'string' || !run.contributor.trim())) {
|
||||
errors.push(`models/${f}: ${where}.contributor must be a non-empty GitHub username`)
|
||||
}
|
||||
const file = (typeof run?.file === 'string' && run.file) || `${caseId}.${caseKind.get(caseId)}`
|
||||
if (seenFiles.has(file)) {
|
||||
errors.push(`models/${f}: ${where} resolves to the same artifact "${file}" as another variant — extra variants must set a distinct "file"`)
|
||||
}
|
||||
seenFiles.add(file)
|
||||
if (!existsSync(join(outputsDir, id, file))) {
|
||||
errors.push(`models/${f}: ${where} declared but artifact missing: outputs/${id}/${file}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for (const entry of readdirSync(outputsDir, { withFileTypes: true })) {
|
||||
if (entry.isDirectory() && !modelIds.has(entry.name)) {
|
||||
errors.push(`outputs/${entry.name}/ exists but models/${entry.name}.json is missing`)
|
||||
}
|
||||
}
|
||||
|
||||
// Every file under outputs/<id>/ must be declared by a run — loose artifacts
|
||||
// would otherwise ship without provenance.
|
||||
for (const id of modelIds) {
|
||||
const dir = join(outputsDir, id)
|
||||
if (!existsSync(dir)) continue
|
||||
const def = JSON.parse(readFileSync(join(modelsDir, `${id}.json`), 'utf8')) as {
|
||||
runs?: Record<string, { file?: string } | Array<{ file?: string }>>
|
||||
}
|
||||
const declared = new Set<string>()
|
||||
for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) {
|
||||
for (const run of Array.isArray(runRaw) ? runRaw : [runRaw]) {
|
||||
declared.add(run?.file ?? `${caseId}.${caseKind.get(caseId)}`)
|
||||
}
|
||||
}
|
||||
for (const file of readdirSync(dir)) {
|
||||
if (!declared.has(file)) {
|
||||
errors.push(`outputs/${id}/${file} is not declared by any run in models/${id}.json`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.error(`Data validation failed (${errors.length}):\n- ${errors.join('\n- ')}`)
|
||||
process.exit(1)
|
||||
}
|
||||
console.log(`Data OK: ${modelIds.size} models, ${caseKind.size} cases validated`)
|
||||
Reference in new issue
Block a user