nagi-bench/scripts/validate-data.ts

// Validates the contribution data: models/<model-id>.json registrations and
// their artifacts under outputs/. Runs in CI for every PR and push, so a
// malformed or undocumented contribution fails before it can ship.
import { existsSync, readFileSync, readdirSync } from 'node:fs'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'

const root = join(dirname(fileURLToPath(import.meta.url)), '..')
const modelsDir = join(root, 'models')
const outputsDir = join(root, 'outputs')

const cases = JSON.parse(readFileSync(join(root, 'cases.json'), 'utf8')) as Array<{
  id: string
  kind: string
}>
const caseKind = new Map(cases.map((c) => [c.id, c.kind]))

const ID_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/
const errors: string[] = []

const isBilingual = (v: unknown): boolean => {
  if (!v || typeof v !== 'object') return false
  const { zh, en } = v as { zh?: unknown; en?: unknown }
  return typeof zh === 'string' && zh.trim() !== '' && typeof en === 'string' && en.trim() !== ''
}

const modelIds = new Set<string>()
for (const f of readdirSync(modelsDir).filter((f) => f.endsWith('.json'))) {
  const id = f.replace(/\.json$/, '')
  modelIds.add(id)
  if (!ID_RE.test(id)) {
    errors.push(`models/${f}: id "${id}" must be lowercase letters/digits/dashes only (it doubles as the outputs/ folder name)`)
  }

  type RunJson = { note?: unknown; file?: unknown; contributor?: unknown }
  let def: { label?: unknown; vendor?: unknown; order?: unknown; runs?: Record<string, RunJson | RunJson[]> }
  try {
    def = JSON.parse(readFileSync(join(modelsDir, f), 'utf8'))
  } catch (e) {
    errors.push(`models/${f}: invalid JSON (${e})`)
    continue
  }

  if (typeof def.label !== 'string' || !def.label.trim()) errors.push(`models/${f}: "label" is required`)
  if (typeof def.vendor !== 'string' || !def.vendor.trim()) errors.push(`models/${f}: "vendor" is required`)
  if (def.order !== undefined && typeof def.order !== 'number') errors.push(`models/${f}: "order" must be a number`)

  for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) {
    if (!caseKind.has(caseId)) {
      errors.push(`models/${f}: unknown case "${caseId}" (known: ${[...caseKind.keys()].join(', ')})`)
      continue
    }
    const variants = Array.isArray(runRaw) ? runRaw : [runRaw]
    if (!variants.length) errors.push(`models/${f}: runs.${caseId} is an empty array`)
    const seenFiles = new Set<string>()
    variants.forEach((run, i) => {
      const where = variants.length > 1 ? `runs.${caseId}[${i}]` : `runs.${caseId}`
      if (!isBilingual(run?.note)) {
        errors.push(`models/${f}: ${where} needs a bilingual "note" ({ "zh", "en" }) documenting provenance (tool, effort level, one-shot or fixed)`)
      }
      if (run?.file !== undefined && typeof run.file !== 'string') {
        errors.push(`models/${f}: ${where}.file must be a string`)
      }
      if (run?.contributor !== undefined && (typeof run.contributor !== 'string' || !run.contributor.trim())) {
        errors.push(`models/${f}: ${where}.contributor must be a non-empty GitHub username`)
      }
      const file = (typeof run?.file === 'string' && run.file) || `${caseId}.${caseKind.get(caseId)}`
      if (seenFiles.has(file)) {
        errors.push(`models/${f}: ${where} resolves to the same artifact "${file}" as another variant — extra variants must set a distinct "file"`)
      }
      seenFiles.add(file)
      if (!existsSync(join(outputsDir, id, file))) {
        errors.push(`models/${f}: ${where} declared but artifact missing: outputs/${id}/${file}`)
      }
    })
  }
}

for (const entry of readdirSync(outputsDir, { withFileTypes: true })) {
  if (entry.isDirectory() && !modelIds.has(entry.name)) {
    errors.push(`outputs/${entry.name}/ exists but models/${entry.name}.json is missing`)
  }
}

// Every file under outputs/<id>/ must be declared by a run — loose artifacts
// would otherwise ship without provenance.
for (const id of modelIds) {
  const dir = join(outputsDir, id)
  if (!existsSync(dir)) continue
  const def = JSON.parse(readFileSync(join(modelsDir, `${id}.json`), 'utf8')) as {
    runs?: Record<string, { file?: string } | Array<{ file?: string }>>
  }
  const declared = new Set<string>()
  for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) {
    for (const run of Array.isArray(runRaw) ? runRaw : [runRaw]) {
      declared.add(run?.file ?? `${caseId}.${caseKind.get(caseId)}`)
    }
  }
  for (const file of readdirSync(dir)) {
    if (!declared.has(file)) {
      errors.push(`outputs/${id}/${file} is not declared by any run in models/${id}.json`)
    }
  }
}

if (errors.length) {
  console.error(`Data validation failed (${errors.length}):\n- ${errors.join('\n- ')}`)
  process.exit(1)
}
console.log(`Data OK: ${modelIds.size} models, ${caseKind.size} cases validated`)