diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 32dd6e3..7feffe4 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -14,5 +14,6 @@ - [ ] 每条 run 的 `note` 双语（zh + en）填写了上面的来历信息 / bilingual `note` documents the provenance above - [ ] `contributor` 是我的 GitHub 用户名 / `contributor` is my GitHub username - [ ] 同一模型在不同 Harness 或思考配额下是**新建一个 JSON 条目**，不是塞进现有条目 / a different harness or effort = a NEW json entry, not a variant +- [ ] 已运行 `bun scripts/update-registry.ts` 更新 README Registry 表 / ran `bun scripts/update-registry.ts` to refresh the README Registry tables diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 063c37f..bca6c79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 + - name: Check README registry + run: bun scripts/update-registry.ts --check - name: Validate contribution data run: bun scripts/validate-data.ts diff --git a/README.en.md b/README.en.md index 583a122..ab92c9c 100644 --- a/README.en.md +++ b/README.en.md @@ -24,10 +24,13 @@ One-shot LLM eval cases by NAGI STUDIO: same prompt, different model x harness x ## Registry + +> This table is generated from `models/*.json` by `bun scripts/update-registry.ts`; CI checks that it stays current. + | Model | Vendor | Harness x Effort | Runs | |---|---|---|---| | Claude Fable 5 | Anthropic | Claude Web App · Max
Claude Code · Max
Cursor · High | 06 | -| Claude Opus 4.8 | Anthropic | Claude Code · Max | 02 | +| Claude Opus 4.8 | Anthropic | Claude Code · Max | 04 | | Claude Opus 4.7 | Anthropic | Cursor · Max | 02 | | Claude Opus 4.6 | Anthropic | Cursor · Max | 02 | | Claude Opus 4.5 | Anthropic | Cursor · Thinking | 02 | @@ -38,8 +41,9 @@ One-shot LLM eval cases by NAGI STUDIO: same prompt, different model x harness x | GPT-5.3 Codex | OpenAI | Cursor · xhigh | 02 | | GPT-5.2 | OpenAI | Cursor · xhigh | 02 | | Gemini 3.1 Pro | Google | AntiGravity · High
Cursor · Default
Google AI Studio · High | 04 | -| Gemini 3.5 Flash | Google | AntiGravity · High
Google AI Studio · High
Cursor · Default | 04 | +| Gemini 3.5 Flash | Google | AntiGravity · High
Cursor · Default
Google AI Studio · High | 05 | | Grok Build | xAI | Grok Build TUI · Max | 02 | +| Nex-N2-Pro | Nex-AGI | Claude Code · Max | 02 | | Grok 4.3 | xAI | Cursor · Default | 02 | | Composer 2.5 | Cursor | Cursor · Max
Cursor · Default
Grok Build TUI · Default | 05 | | Mistral Medium 3.5 | Mistral AI | Vibe · Thinking | 02 | @@ -47,14 +51,13 @@ One-shot LLM eval cases by NAGI STUDIO: same prompt, different model x harness x | Doubao Seed 2.0 Pro | ByteDance | Doubao Web · Pro Mode | 01 | | Doubao Seed 2.0 Mini | ByteDance | Doubao Web · Fast Mode | 01 | | MiMo v2.5 Pro | Xiaomi | Claude Code · Max | 02 | -| Kimi K2.6 | Moonshot AI | Kimi Code · Thinking | 02 | | Kimi K2.7-Code | Moonshot AI | Kimi Code · Thinking | 03 | +| Kimi K2.6 | Moonshot AI | Kimi Code · Thinking | 02 | | MiniMax M3 | MiniMax | MiniMax Code Web · Thinking | 02 | | Qwen3.7-Max | Alibaba | Qoder · Default | 02 | Pending: GLM-5.1 · GLM-5.2 (PRs welcome) - -> This table is a hand-maintained snapshot; the site's registry section is generated live from `models/*.json` and is authoritative. + ## Arena blind battles & community board @@ -69,6 +72,7 @@ outputs//. raw model artifacts (HTML / SVG), filena models/.json agent registry: label / vendor / harness / effort / artifactDir / order / run notes cases.json case definitions (bilingual prompts, maintainer-owned) scripts/validate-data.ts data validation (CI runs it on every PR) +scripts/update-registry.ts README Registry table generation (CI checks freshness) ``` > The site and vote-backend source lives in the private `nagi-bench-site` repo, deployed on Cloudflare Pages; this repo is the site's data source. @@ -76,6 +80,7 @@ scripts/validate-data.ts data validation (CI runs it on every PR) ## Validation ```bash +bun scripts/update-registry.ts # updates README Registry tables from models/*.json bun scripts/validate-data.ts # the same data validation CI runs ``` @@ -85,7 +90,7 @@ bun scripts/validate-data.ts # the same data validation CI runs > > - **The unit you contribute is one Agent = a model × the harness you run in.** The `` (lowercase letters / digits / dashes) encodes that pairing — e.g. `gpt-5-5-pro`, `claude-fable-5-cc` (cc = Claude Code). The same model in a different harness is a different Agent with a different id. The `` is the vote and leaderboard identity; do not rename it just to reorganize files. > - **There are two cases right now**, defined in [`cases.json`](./cases.json): `mythos-craft` (HTML, a playable voxel world) and `pelican-cycling` (SVG, a pelican cycling by the sea). The prompt lives in `cases.json` — **use it verbatim, do not paraphrase**. -> - **To find what's missing**: read `cases.json` (all cases) and `models/*.json` (what each Agent has run) — a `` absent from an Agent's json is an open slot; any row in the Registry table above whose "Runs" count is below the number of cases is an Agent missing a case; a brand-new Agent can fill both. +> - **To find what's missing**: read `cases.json` (all cases) and `models/*.json` (what each Agent has run) — a `` absent from an Agent's json is an open slot; any row in the script-generated, CI-checked Registry table above whose "Runs" count is below the number of cases is an Agent missing a case; a brand-new Agent can fill both. > - **Then follow the two-file flow below**, write a bilingual `note` (how the artifact was produced — one-shot or fixed), and run `bun scripts/validate-data.ts` until it passes before opening a PR. Contributions are data-only — no code changes needed: diff --git a/README.md b/README.md index b0ca0c9..b4dbba5 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,13 @@ NAGI STUDIO 的 LLM 测评案例集：同一段提示词，不同「模型 × Ha ## 已测组合 / Registry + +> 此表由 `bun scripts/update-registry.ts` 从 `models/*.json` 自动生成；CI 会检查它是否最新。 + | 模型 | 厂商 | 运行环境（Harness）× 思考配额 | 产出 | |---|---|---|---| | Claude Fable 5 | Anthropic | Claude Web App · Max
Claude Code · Max
Cursor · High | 06 | -| Claude Opus 4.8 | Anthropic | Claude Code · Max | 02 | +| Claude Opus 4.8 | Anthropic | Claude Code · Max | 04 | | Claude Opus 4.7 | Anthropic | Cursor · Max | 02 | | Claude Opus 4.6 | Anthropic | Cursor · Max | 02 | | Claude Opus 4.5 | Anthropic | Cursor · Thinking | 02 | @@ -38,8 +41,9 @@ NAGI STUDIO 的 LLM 测评案例集：同一段提示词，不同「模型 × Ha | GPT-5.3 Codex | OpenAI | Cursor · xhigh | 02 | | GPT-5.2 | OpenAI | Cursor · xhigh | 02 | | Gemini 3.1 Pro | Google | AntiGravity · High
Cursor · Default
Google AI Studio · High | 04 | -| Gemini 3.5 Flash | Google | AntiGravity · High
Google AI Studio · High
Cursor · Default | 04 | +| Gemini 3.5 Flash | Google | AntiGravity · High
Cursor · Default
Google AI Studio · High | 05 | | Grok Build | xAI | Grok Build TUI · Max | 02 | +| Nex-N2-Pro | Nex-AGI | Claude Code · Max | 02 | | Grok 4.3 | xAI | Cursor · Default | 02 | | Composer 2.5 | Cursor | Cursor · Max
Cursor · Default
Grok Build TUI · Default | 05 | | Mistral Medium 3.5 | Mistral AI | Vibe · Thinking | 02 | @@ -47,14 +51,13 @@ NAGI STUDIO 的 LLM 测评案例集：同一段提示词，不同「模型 × Ha | Doubao Seed 2.0 Pro | ByteDance | Doubao Web · Pro Mode | 01 | | Doubao Seed 2.0 Mini | ByteDance | Doubao Web · Fast Mode | 01 | | MiMo v2.5 Pro | Xiaomi | Claude Code · Max | 02 | -| Kimi K2.6 | Moonshot AI | Kimi Code · Thinking | 02 | | Kimi K2.7-Code | Moonshot AI | Kimi Code · Thinking | 03 | +| Kimi K2.6 | Moonshot AI | Kimi Code · Thinking | 02 | | MiniMax M3 | MiniMax | MiniMax Code Web · Thinking | 02 | | Qwen3.7-Max | Alibaba | Qoder · Default | 02 | 待测：GLM-5.1 · GLM-5.2（欢迎 PR 补充） - -> 此表为人工维护的快照；站点上的「模型阵容」表由 `models/*.json` 实时生成，以站点为准。 + ## Arena 盲评与社区榜单 @@ -69,6 +72,7 @@ outputs//. 模型产出原文件（HTML / SVG）， models/.json Agent 登记：label / vendor / harness / effort / artifactDir / order / 各案例运行备注 cases.json 案例定义：双语标题与提示词（维护者维护） scripts/validate-data.ts 数据校验（CI 对每个 PR 自动执行） +scripts/update-registry.ts README Registry 表生成（CI 检查是否最新） ``` > 站点与投票后端的源码在私有仓库 `nagi-bench-site` 中，部署于 Cloudflare Pages；本仓库是站点的数据源。 @@ -76,6 +80,7 @@ scripts/validate-data.ts 数据校验（CI 对每个 PR 自动执行 ## 本地校验 / Validation ```bash +bun scripts/update-registry.ts # 从 models/*.json 更新 README Registry 表 bun scripts/validate-data.ts # 与 CI 相同的数据校验 ``` @@ -85,7 +90,7 @@ bun scripts/validate-data.ts # 与 CI 相同的数据校验 > > - **你贡献的单位是一个 Agent = 模型 × 你所在的 Harness。** ``（小写字母 / 数字 / 连字符）就编码这个组合，如 `gpt-5-5-pro`、`claude-fable-5-cc`（cc = Claude Code）。同一模型换个 Harness 就是另一个 Agent、另一个 id。`` 是投票与榜单身份，不要为了整理文件夹而改它。 > - **当前有两个案例**，定义在 [`cases.json`](./cases.json)：`mythos-craft`（HTML，可玩体素世界）与 `pelican-cycling`（SVG，海边骑车的鹈鹕）。提示词以 `cases.json` 为准，**逐字使用、不要改写**。 -> - **想知道还缺哪些**：读 `cases.json`（全部案例）与 `models/*.json`（每个 Agent 跑过的案例）——某个 Agent 的 json 里缺某个 `` 就是一个空位；上面 Registry 表里「产出」数小于案例总数的行，就是还缺案例的 Agent；全新 Agent 则两个案例都可补。 +> - **想知道还缺哪些**：读 `cases.json`（全部案例）与 `models/*.json`（每个 Agent 跑过的案例）——某个 Agent 的 json 里缺某个 `` 就是一个空位；上面由脚本生成并由 CI 检查的 Registry 表里，「产出」数小于案例总数的行，就是还缺案例的 Agent；全新 Agent 则两个案例都可补。 > - **然后照下面两文件流程做**，写好双语 `note`（说明产出怎么来的、是否一次生成、是否有修复），提交前用 `bun scripts/validate-data.ts` 自检通过再发 PR。 贡献**不需要改任何代码**，只涉及两类文件： diff --git a/scripts/update-registry.ts b/scripts/update-registry.ts new file mode 100644 index 0000000..8a59606 --- /dev/null +++ b/scripts/update-registry.ts @@ -0,0 +1,139 @@ +// Generates the README Registry tables from models/*.json. CI runs this in +// --check mode so the public README cannot drift from the source data. +import { readFileSync, readdirSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = join(dirname(fileURLToPath(import.meta.url)), '..') +const modelsDir = join(root, 'models') +const checkOnly = process.argv.includes('--check') + +const START = '' +const END = '' + +type RunJson = { file?: unknown } +type ModelJson = { + label?: unknown + vendor?: unknown + harness?: unknown + effort?: unknown + order?: unknown + runs?: Record +} + +type HarnessEntry = { + label: string + order: number +} + +type RegistryGroup = { + label: string + vendor: string + order: number + runs: number + harnesses: Map +} + +function requiredString(value: unknown, field: string, file: string): string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`models/${file}: "${field}" is required`) + return value.trim() +} + +function runCount(runs: ModelJson['runs']): number { + return Object.values(runs ?? {}).reduce((sum, run) => sum + (Array.isArray(run) ? run.length : 1), 0) +} + +function harnessLabel(def: ModelJson): string { + const harness = typeof def.harness === 'string' ? def.harness.trim() : '' + const effort = typeof def.effort === 'string' ? def.effort.trim() : '' + if (!harness && !effort) return 'Default' + if (!harness) return effort + return effort ? `${harness} · ${effort}` : harness +} + +function escapeCell(value: string): string { + return value.replace(/\|/g, '\\|').replace(/\n/g, ' ') +} + +function registryGroups(): RegistryGroup[] { + const groups = new Map() + for (const file of readdirSync(modelsDir).filter((name) => name.endsWith('.json')).sort()) { + const def = JSON.parse(readFileSync(join(modelsDir, file), 'utf8')) as ModelJson + const label = requiredString(def.label, 'label', file) + const vendor = requiredString(def.vendor, 'vendor', file) + const order = typeof def.order === 'number' ? def.order : Number.MAX_SAFE_INTEGER + const key = `${label}\0${vendor}` + let group = groups.get(key) + if (!group) { + group = { label, vendor, order, runs: 0, harnesses: new Map() } + groups.set(key, group) + } + + group.order = Math.min(group.order, order) + group.runs += runCount(def.runs) + const harness = harnessLabel(def) + const existing = group.harnesses.get(harness) + if (!existing || order < existing.order) group.harnesses.set(harness, { label: harness, order }) + } + + return [...groups.values()].sort((a, b) => a.order - b.order || a.label.localeCompare(b.label)) +} + +function table(lang: 'zh' | 'en'): string { + const groups = registryGroups() + const tested = groups.filter((group) => group.runs > 0) + const pending = groups.filter((group) => group.runs === 0).map((group) => escapeCell(group.label)) + const note = + lang === 'zh' + ? '> 此表由 `bun scripts/update-registry.ts` 从 `models/*.json` 自动生成；CI 会检查它是否最新。' + : '> This table is generated from `models/*.json` by `bun scripts/update-registry.ts`; CI checks that it stays current.' + const header = + lang === 'zh' + ? '| 模型 | 厂商 | 运行环境（Harness）× 思考配额 | 产出 |' + : '| Model | Vendor | Harness x Effort | Runs |' + + const rows = tested.map((group) => { + const harnesses = [...group.harnesses.values()] + .sort((a, b) => a.order - b.order || a.label.localeCompare(b.label)) + .map((entry) => escapeCell(entry.label)) + .join('
') + return `| ${escapeCell(group.label)} | ${escapeCell(group.vendor)} | ${harnesses} | ${String(group.runs).padStart(2, '0')} |` + }) + + const lines = [START, note, '', header, '|---|---|---|---|', ...rows] + if (pending.length) { + lines.push( + '', + lang === 'zh' + ? `待测：${pending.join(' · ')}（欢迎 PR 补充）` + : `Pending: ${pending.join(' · ')} (PRs welcome)`, + ) + } + lines.push(END) + return lines.join('\n') +} + +function replaceRegistry(path: string, generated: string): boolean { + const original = readFileSync(path, 'utf8') + const start = original.indexOf(START) + const end = original.indexOf(END) + if (start === -1 || end === -1 || end < start) { + throw new Error(`${path.replace(`${root}/`, '')}: missing ${START} / ${END} markers`) + } + const next = `${original.slice(0, start)}${generated}${original.slice(end + END.length)}` + if (next === original) return false + if (!checkOnly) writeFileSync(path, next) + return true +} + +const changed = [ + replaceRegistry(join(root, 'README.md'), table('zh')), + replaceRegistry(join(root, 'README.en.md'), table('en')), +].some(Boolean) + +if (checkOnly && changed) { + console.error('Registry tables are stale. Run: bun scripts/update-registry.ts') + process.exit(1) +} + +console.log(changed ? 'Registry tables updated' : 'Registry tables OK')