From 664a7115c7f0a80894b496411c9bda7d7030d8b6 Mon Sep 17 00:00:00 2001 From: Jesse Zhang <101612750+Nagi-ovo@users.noreply.github.com> Date: Wed, 10 Jun 2026 22:35:53 +0100 Subject: [PATCH] Become the data-only repo: site and worker move to private nagi-bench-site MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The showcase app and the arena backend now live in the private nagi-bench-site repo and deploy to Cloudflare Pages, fetching this repo's models/, outputs/ and cases.json at build time. CI here is a standalone data validation (same rules as before, no build), plus an optional repository_dispatch that triggers an instant site rebuild when SITE_DISPATCH_TOKEN is configured — otherwise the site's 6-hour cron picks new data up. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 34 ++++++++--- .github/workflows/deploy.yml | 46 --------------- README.en.md | 16 +++-- README.md | 16 +++-- scripts/validate-data.ts | 110 +++++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 71 deletions(-) delete mode 100644 .github/workflows/deploy.yml create mode 100644 scripts/validate-data.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c2a3fb..063c37f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,19 +2,39 @@ name: CI on: pull_request: + push: + branches: [main] permissions: contents: read jobs: - build: + validate: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - - name: Install dependencies - run: bun install - working-directory: site - - name: Type-check and build - run: bun run build - working-directory: site + - name: Validate contribution data + run: bun scripts/validate-data.ts + + trigger-site-rebuild: + # The site lives in the private nagi-bench-site repo (Cloudflare Pages). + # With SITE_DISPATCH_TOKEN set the rebuild is instant; without it the + # site's 6-hour cron picks the new data up anyway. + if: github.event_name == 'push' + needs: validate + runs-on: ubuntu-latest + steps: + - name: Dispatch data-updated to nagi-bench-site + env: + TOKEN: ${{ secrets.SITE_DISPATCH_TOKEN }} + run: | + if [ -z "$TOKEN" ]; then + echo "SITE_DISPATCH_TOKEN not set; relying on the site's scheduled rebuild." + exit 0 + fi + curl -sf -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/nagi-studio/nagi-bench-site/dispatches \ + -d '{"event_type":"data-updated"}' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index de8b4d1..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Deploy site to GitHub Pages - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - - name: Install dependencies - run: bun install --frozen-lockfile - working-directory: site - - name: Build - run: bun run build - working-directory: site - env: - # Optional arena backend (see worker/README.md); empty = local-only votes. - VITE_ARENA_API: ${{ vars.VITE_ARENA_API }} - VITE_TURNSTILE_SITE_KEY: ${{ vars.VITE_TURNSTILE_SITE_KEY }} - - uses: actions/configure-pages@v5 - - uses: actions/upload-pages-artifact@v3 - with: - path: site/dist - - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - id: deployment - uses: actions/deploy-pages@v4 diff --git a/README.en.md b/README.en.md index ab2a7c9..e28ec9e 100644 --- a/README.en.md +++ b/README.en.md @@ -49,18 +49,16 @@ Pending: Mistral Medium 3.5 · GLM-5.1 · Qwen3.7-Max (PRs welcome) ``` outputs//. raw model artifacts (HTML / SVG), filename = case id models/.json combination registry: label / vendor / harness / effort / order / run notes -site/ showcase app (Vite + React + Tailwind v4 + GSAP, bun) -site/src/data/cases.json case definitions (bilingual prompts, maintainer-owned) -.github/workflows/ deploy.yml deploys; ci.yml validates data + builds on PRs +cases.json case definitions (bilingual prompts, maintainer-owned) +scripts/validate-data.ts data validation (CI runs it on every PR) ``` -## Development +> The site and vote-backend source lives in the private `nagi-bench-site` repo, deployed on Cloudflare Pages; this repo is the site's data source. + +## Validation ```bash -cd site -bun install -bun run dev # syncs outputs/ and serves http://localhost:5173/ -bun run build # data validation + type-check + build (site/dist) +bun scripts/validate-data.ts # the same data validation CI runs ``` ## Contributing a run @@ -72,4 +70,4 @@ Contributions are data-only — no code changes needed: CI-enforced rules: dash-only lowercase ids matching the `outputs/` folder name; a **required bilingual provenance `note`** per run (which harness, what effort, one-shot or fixed); every declared run must have its artifact; multiple versions of the same combination x case go in an array with distinct `file` names (e.g. `pelican-cycling-2.svg`). Set `contributor` to your GitHub username — the site shows your avatar linked to your profile next to the run. Fill in `harness` and `effort` truthfully for new combinations: the case page renders Harness / Effort metadata chips from these fields and auto-matches brand icons for the model, vendor and harness (via [lobe-icons](https://github.com/lobehub/lobe-icons)) — contributors never touch icons. -PRs get data validation + a build from CI; merges to `main` deploy automatically. +PRs get data validation from CI; merges to `main` rebuild the site automatically (usually instant, at most 6 hours). diff --git a/README.md b/README.md index dda23c5..0ca88e3 100644 --- a/README.md +++ b/README.md @@ -49,18 +49,16 @@ NAGI STUDIO 的 LLM 测评案例集：同一段提示词，不同「模型 × Ha ``` outputs//. 模型产出原文件（HTML / SVG），文件名 = 案例 id models/.json 组合登记：label / vendor / harness / effort / order / 各案例运行备注 -site/ 展示站点（Vite + React + Tailwind v4 + GSAP，bun 驱动） -site/src/data/cases.json 案例定义：双语标题与提示词（维护者维护） -.github/workflows/ deploy.yml 自动部署；ci.yml 对 PR 跑数据校验 + 构建 +cases.json 案例定义：双语标题与提示词（维护者维护） +scripts/validate-data.ts 数据校验（CI 对每个 PR 自动执行） ``` -## 本地开发 / Development +> 站点与投票后端的源码在私有仓库 `nagi-bench-site` 中，部署于 Cloudflare Pages；本仓库是站点的数据源。 + +## 本地校验 / Validation ```bash -cd site -bun install -bun run dev # 同步 outputs/ 并启动 http://localhost:5173/ -bun run build # 数据校验 + 类型检查 + 产物构建（输出到 site/dist） +bun scripts/validate-data.ts # 与 CI 相同的数据校验 ``` ## 贡献一个模型产出 / Contributing a run @@ -97,4 +95,4 @@ bun run build # 数据校验 + 类型检查 + 产物构建（输出到 site/d - `contributor` 填你的 GitHub 用户名，站点会在产出旁展示你的头像并链接到主页； - 新组合的 `harness`（运行环境）与 `effort`（思考配额）请如实填写：站点会据此在测评页生成「运行环境 / 思考配额」metadata 徽章，并自动为模型、厂商、Harness 匹配品牌 icon（来自 [lobe-icons](https://github.com/lobehub/lobe-icons)），贡献者无需处理任何图标。 -提 PR 后 CI 会跑数据校验 + 构建；合入 `main` 自动部署。 +提 PR 后 CI 自动校验数据；合入 `main` 后站点自动重建（通常即时，最长 6 小时）。 diff --git a/scripts/validate-data.ts b/scripts/validate-data.ts new file mode 100644 index 0000000..42552ba --- /dev/null +++ b/scripts/validate-data.ts @@ -0,0 +1,110 @@ +// Validates the contribution data: models/.json registrations and +// their artifacts under outputs/. Runs in CI for every PR and push, so a +// malformed or undocumented contribution fails before it can ship. +import { existsSync, readFileSync, readdirSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = join(dirname(fileURLToPath(import.meta.url)), '..') +const modelsDir = join(root, 'models') +const outputsDir = join(root, 'outputs') + +const cases = JSON.parse(readFileSync(join(root, 'cases.json'), 'utf8')) as Array<{ + id: string + kind: string +}> +const caseKind = new Map(cases.map((c) => [c.id, c.kind])) + +const ID_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/ +const errors: string[] = [] + +const isBilingual = (v: unknown): boolean => { + if (!v || typeof v !== 'object') return false + const { zh, en } = v as { zh?: unknown; en?: unknown } + return typeof zh === 'string' && zh.trim() !== '' && typeof en === 'string' && en.trim() !== '' +} + +const modelIds = new Set() +for (const f of readdirSync(modelsDir).filter((f) => f.endsWith('.json'))) { + const id = f.replace(/\.json$/, '') + modelIds.add(id) + if (!ID_RE.test(id)) { + errors.push(`models/${f}: id "${id}" must be lowercase letters/digits/dashes only (it doubles as the outputs/ folder name)`) + } + + type RunJson = { note?: unknown; file?: unknown; contributor?: unknown } + let def: { label?: unknown; vendor?: unknown; order?: unknown; runs?: Record } + try { + def = JSON.parse(readFileSync(join(modelsDir, f), 'utf8')) + } catch (e) { + errors.push(`models/${f}: invalid JSON (${e})`) + continue + } + + if (typeof def.label !== 'string' || !def.label.trim()) errors.push(`models/${f}: "label" is required`) + if (typeof def.vendor !== 'string' || !def.vendor.trim()) errors.push(`models/${f}: "vendor" is required`) + if (def.order !== undefined && typeof def.order !== 'number') errors.push(`models/${f}: "order" must be a number`) + + for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) { + if (!caseKind.has(caseId)) { + errors.push(`models/${f}: unknown case "${caseId}" (known: ${[...caseKind.keys()].join(', ')})`) + continue + } + const variants = Array.isArray(runRaw) ? runRaw : [runRaw] + if (!variants.length) errors.push(`models/${f}: runs.${caseId} is an empty array`) + const seenFiles = new Set() + variants.forEach((run, i) => { + const where = variants.length > 1 ? `runs.${caseId}[${i}]` : `runs.${caseId}` + if (!isBilingual(run?.note)) { + errors.push(`models/${f}: ${where} needs a bilingual "note" ({ "zh", "en" }) documenting provenance (tool, effort level, one-shot or fixed)`) + } + if (run?.file !== undefined && typeof run.file !== 'string') { + errors.push(`models/${f}: ${where}.file must be a string`) + } + if (run?.contributor !== undefined && (typeof run.contributor !== 'string' || !run.contributor.trim())) { + errors.push(`models/${f}: ${where}.contributor must be a non-empty GitHub username`) + } + const file = (typeof run?.file === 'string' && run.file) || `${caseId}.${caseKind.get(caseId)}` + if (seenFiles.has(file)) { + errors.push(`models/${f}: ${where} resolves to the same artifact "${file}" as another variant — extra variants must set a distinct "file"`) + } + seenFiles.add(file) + if (!existsSync(join(outputsDir, id, file))) { + errors.push(`models/${f}: ${where} declared but artifact missing: outputs/${id}/${file}`) + } + }) + } +} + +for (const entry of readdirSync(outputsDir, { withFileTypes: true })) { + if (entry.isDirectory() && !modelIds.has(entry.name)) { + errors.push(`outputs/${entry.name}/ exists but models/${entry.name}.json is missing`) + } +} + +// Every file under outputs// must be declared by a run — loose artifacts +// would otherwise ship without provenance. +for (const id of modelIds) { + const dir = join(outputsDir, id) + if (!existsSync(dir)) continue + const def = JSON.parse(readFileSync(join(modelsDir, `${id}.json`), 'utf8')) as { + runs?: Record> + } + const declared = new Set() + for (const [caseId, runRaw] of Object.entries(def.runs ?? {})) { + for (const run of Array.isArray(runRaw) ? runRaw : [runRaw]) { + declared.add(run?.file ?? `${caseId}.${caseKind.get(caseId)}`) + } + } + for (const file of readdirSync(dir)) { + if (!declared.has(file)) { + errors.push(`outputs/${id}/${file} is not declared by any run in models/${id}.json`) + } + } +} + +if (errors.length) { + console.error(`Data validation failed (${errors.length}):\n- ${errors.join('\n- ')}`) + process.exit(1) +} +console.log(`Data OK: ${modelIds.size} models, ${caseKind.size} cases validated`)