Remove empty artifacts from Cursor runs
This commit is contained in:
1 parent
cb79f09c88
commit
7a1571f46b
10 files changed
+9
-46
No files matched your search
+2
-3
@@ -29,11 +29,10 @@ One-shot LLM eval cases by NAGI STUDIO: same prompt, different model x harness x
|
|||||||
| Claude Fable 5 | Anthropic | Claude Web App · Max<br>Claude Code · Max<br>Cursor · High | 06 |
|
| Claude Fable 5 | Anthropic | Claude Web App · Max<br>Claude Code · Max<br>Cursor · High | 06 |
|
||||||
| Claude Opus 4.8 | Anthropic | Claude Code · Max | 02 |
|
| Claude Opus 4.8 | Anthropic | Claude Code · Max | 02 |
|
||||||
| Claude Opus 4.7 | Anthropic | Cursor · Max | 02 |
|
| Claude Opus 4.7 | Anthropic | Cursor · Max | 02 |
|
||||||
| Claude Opus 4.6 | Anthropic | Cursor · Max | 02 |
|
| Claude Opus 4.6 | Anthropic | Cursor · Max | 01 |
|
||||||
| Claude Sonnet 4.6 | Anthropic | Cursor · High | 02 |
|
|
||||||
| GPT-5.5 Pro | OpenAI | ChatGPT Web · Extended Pro | 02 |
|
| GPT-5.5 Pro | OpenAI | ChatGPT Web · Extended Pro | 02 |
|
||||||
| GPT-5.5 | OpenAI | Codex CLI · xhigh | 02 |
|
| GPT-5.5 | OpenAI | Codex CLI · xhigh | 02 |
|
||||||
| Gemini 3.1 Pro | Google | AntiGravity · High<br>Cursor · Default<br>Google AI Studio · High | 05 |
|
| Gemini 3.1 Pro | Google | AntiGravity · High<br>Cursor · Default<br>Google AI Studio · High | 04 |
|
||||||
| Gemini 3.5 Flash | Google | AntiGravity · High<br>Google AI Studio · High<br>Cursor · Default | 04 |
|
| Gemini 3.5 Flash | Google | AntiGravity · High<br>Google AI Studio · High<br>Cursor · Default | 04 |
|
||||||
| Grok Build | xAI | Grok Build TUI · Max | 02 |
|
| Grok Build | xAI | Grok Build TUI · Max | 02 |
|
||||||
| Composer 2.5 | Cursor | Cursor · Max<br>Cursor · Default<br>Grok Build TUI · Default | 05 |
|
| Composer 2.5 | Cursor | Cursor · Max<br>Cursor · Default<br>Grok Build TUI · Default | 05 |
|
||||||
|
|||||||
@@ -29,11 +29,10 @@ NAGI STUDIO 的 LLM 测评案例集:同一段提示词,不同「模型 × Ha
|
|||||||
| Claude Fable 5 | Anthropic | Claude Web App · Max<br>Claude Code · Max<br>Cursor · High | 06 |
|
| Claude Fable 5 | Anthropic | Claude Web App · Max<br>Claude Code · Max<br>Cursor · High | 06 |
|
||||||
| Claude Opus 4.8 | Anthropic | Claude Code · Max | 02 |
|
| Claude Opus 4.8 | Anthropic | Claude Code · Max | 02 |
|
||||||
| Claude Opus 4.7 | Anthropic | Cursor · Max | 02 |
|
| Claude Opus 4.7 | Anthropic | Cursor · Max | 02 |
|
||||||
| Claude Opus 4.6 | Anthropic | Cursor · Max | 02 |
|
| Claude Opus 4.6 | Anthropic | Cursor · Max | 01 |
|
||||||
| Claude Sonnet 4.6 | Anthropic | Cursor · High | 02 |
|
|
||||||
| GPT-5.5 Pro | OpenAI | ChatGPT Web · Extended Pro | 02 |
|
| GPT-5.5 Pro | OpenAI | ChatGPT Web · Extended Pro | 02 |
|
||||||
| GPT-5.5 | OpenAI | Codex CLI · xhigh | 02 |
|
| GPT-5.5 | OpenAI | Codex CLI · xhigh | 02 |
|
||||||
| Gemini 3.1 Pro | Google | AntiGravity · High<br>Cursor · Default<br>Google AI Studio · High | 05 |
|
| Gemini 3.1 Pro | Google | AntiGravity · High<br>Cursor · Default<br>Google AI Studio · High | 04 |
|
||||||
| Gemini 3.5 Flash | Google | AntiGravity · High<br>Google AI Studio · High<br>Cursor · Default | 04 |
|
| Gemini 3.5 Flash | Google | AntiGravity · High<br>Google AI Studio · High<br>Cursor · Default | 04 |
|
||||||
| Grok Build | xAI | Grok Build TUI · Max | 02 |
|
| Grok Build | xAI | Grok Build TUI · Max | 02 |
|
||||||
| Composer 2.5 | Cursor | Cursor · Max<br>Cursor · Default<br>Grok Build TUI · Default | 05 |
|
| Composer 2.5 | Cursor | Cursor · Max<br>Cursor · Default<br>Grok Build TUI · Default | 05 |
|
||||||
|
|||||||
@@ -12,13 +12,6 @@
|
|||||||
"en": "Generated in one shot by Claude Opus 4.6 (Max effort) inside Cursor; unmodified original output."
|
"en": "Generated in one shot by Claude Opus 4.6 (Max effort) inside Cursor; unmodified original output."
|
||||||
},
|
},
|
||||||
"contributor": "Nagi-ovo"
|
"contributor": "Nagi-ovo"
|
||||||
},
|
|
||||||
"pelican-cycling": {
|
|
||||||
"note": {
|
|
||||||
"zh": "在 Cursor 中以 Claude Opus 4.6(Max 思考配额)一次生成;本页为未经修改的原始产出。",
|
|
||||||
"en": "Generated in one shot by Claude Opus 4.6 (Max effort) inside Cursor; unmodified original output."
|
|
||||||
},
|
|
||||||
"contributor": "Nagi-ovo"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
{
|
|
||||||
"label": "Claude Sonnet 4.6",
|
|
||||||
"vendor": "Anthropic",
|
|
||||||
"harness": "Cursor",
|
|
||||||
"effort": "High",
|
|
||||||
"artifactDir": "claude-sonnet-4-6/cursor-high",
|
|
||||||
"order": 18,
|
|
||||||
"runs": {
|
|
||||||
"mythos-craft": {
|
|
||||||
"note": {
|
|
||||||
"zh": "在 Cursor 中以 Claude Sonnet 4.6(High 思考配额)一次生成;本页为未经修改的原始产出。",
|
|
||||||
"en": "Generated in one shot by Claude Sonnet 4.6 (High effort) inside Cursor; unmodified original output."
|
|
||||||
},
|
|
||||||
"contributor": "Nagi-ovo"
|
|
||||||
},
|
|
||||||
"pelican-cycling": {
|
|
||||||
"note": {
|
|
||||||
"zh": "在 Cursor 中以 Claude Sonnet 4.6(High 思考配额)一次生成;本页为未经修改的原始产出。",
|
|
||||||
"en": "Generated in one shot by Claude Sonnet 4.6 (High effort) inside Cursor; unmodified original output."
|
|
||||||
},
|
|
||||||
"contributor": "Nagi-ovo"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -12,13 +12,6 @@
|
|||||||
"en": "Generated in one shot by Gemini 3.1 Pro inside Cursor with default settings; unmodified original output."
|
"en": "Generated in one shot by Gemini 3.1 Pro inside Cursor with default settings; unmodified original output."
|
||||||
},
|
},
|
||||||
"contributor": "Nagi-ovo"
|
"contributor": "Nagi-ovo"
|
||||||
},
|
|
||||||
"pelican-cycling": {
|
|
||||||
"note": {
|
|
||||||
"zh": "在 Cursor 中以 Gemini 3.1 Pro 默认设置一次生成;本页为未经修改的原始产出。",
|
|
||||||
"en": "Generated in one shot by Gemini 3.1 Pro inside Cursor with default settings; unmodified original output."
|
|
||||||
},
|
|
||||||
"contributor": "Nagi-ovo"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Whitespace-only changes.
Whitespace-only changes.
Whitespace-only changes.
Whitespace-only changes.
@@ -1,7 +1,7 @@
|
|||||||
// Validates the contribution data: models/<model-id>.json registrations and
|
// Validates the contribution data: models/<model-id>.json registrations and
|
||||||
// their artifacts under outputs/. Runs in CI for every PR and push, so a
|
// their artifacts under outputs/. Runs in CI for every PR and push, so a
|
||||||
// malformed or undocumented contribution fails before it can ship.
|
// malformed or undocumented contribution fails before it can ship.
|
||||||
import { existsSync, readFileSync, readdirSync } from 'node:fs'
|
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'
|
||||||
import { dirname, join } from 'node:path'
|
import { dirname, join } from 'node:path'
|
||||||
import { fileURLToPath } from 'node:url'
|
import { fileURLToPath } from 'node:url'
|
||||||
|
|
||||||
@@ -99,8 +99,11 @@ for (const f of readdirSync(modelsDir).filter((f) => f.endsWith('.json'))) {
|
|||||||
seenFiles.add(file)
|
seenFiles.add(file)
|
||||||
const artifact = `${artifactDir}/${file}`
|
const artifact = `${artifactDir}/${file}`
|
||||||
declaredArtifacts.add(artifact)
|
declaredArtifacts.add(artifact)
|
||||||
if (!existsSync(join(outputsDir, artifactDir, file))) {
|
const artifactPath = join(outputsDir, artifactDir, file)
|
||||||
|
if (!existsSync(artifactPath)) {
|
||||||
errors.push(`models/${f}: ${where} declared but artifact missing: outputs/${artifact}`)
|
errors.push(`models/${f}: ${where} declared but artifact missing: outputs/${artifact}`)
|
||||||
|
} else if (statSync(artifactPath).size === 0) {
|
||||||
|
errors.push(`models/${f}: ${where} artifact is empty: outputs/${artifact}`)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user