skillgauntlet 1.2.0 → 1.3.1

package/lib/automerge.js

@@ -0,0 +1,107 @@
+// The decision rules of the submission auto-merge, as pure functions: no network, no files, no clock.
+// scripts/automerge.mjs gathers the facts (through the GitHub API, as data) and applies the verdict.
+//
+// A pull request is merged automatically only when every rule here holds. Anything else is not an
+// error: it is sent to a maintainer with the reasons written on the pull request.
+import { FILES, LIMITS } from './submission.js';
+
+export const POLICY = {
+  fileBytes: LIMITS.fileBytes,        // per file (the same cap the validator enforces)
+  totalBytes: 10 * 1024 * 1024,       // all four files of one submission
+  maxFiles: FILES.length,
+  perAccountPerDay: 5,                // auto-merged submissions per account in the last 24 hours
+  minAccountAgeDays: 30,              // younger accounts go to manual review
+  trustedAssociations: ['OWNER', 'MEMBER', 'COLLABORATOR'], // skip the account-age and recent-failure rules only
+  claContext: 'license/cla',
+  label: 'needs-review',
+};
+
+const DIR_RE = new RegExp(`^(submissions/[a-z0-9][a-z0-9._-]*/[a-z0-9][a-z0-9._-]*/[0-9]{8}T[0-9]{6}Z-[A-Za-z0-9][A-Za-z0-9-]*)/(${FILES.map((f) => f.replace('.', '\\.')).join('|')})$`);
+
+// The text goes into a pull-request comment: no @mentions, one line, bounded.
+const safe = (s) => String(s).replace(/@/g, '@​').replace(/[\r\n`]+/g, ' ').slice(0, 200);
+
+// changed: [{ filename, status }] from the pull request's file list.
+// entries: { [path]: { type, size } } from the contents API at the head commit (type is 'file' for a plain
+// file, 'symlink' / 'submodule' / 'dir' otherwise). Returns { ok, reasons, dir }.
+export function gateFiles(changed, entries = {}) {
+  const reasons = [];
+  const dirs = new Set();
+  if (!changed.length) reasons.push('the pull request lists no changed files');
+  if (changed.length > POLICY.maxFiles) reasons.push(`it changes ${changed.length} files; a submission is exactly ${POLICY.maxFiles}`);
+  let total = 0;
+  const seen = new Set();
+  for (const { filename, status } of changed) {
+    const name = String(filename);
+    if (name.split('/').some((seg) => seg === '..' || seg === '.' || seg === '') || name.includes('\\')) {
+      reasons.push(`\`${safe(name)}\` is not a plain relative path`);
+      continue;
+    }
+    const m = DIR_RE.exec(name);
+    if (!m) { reasons.push(`\`${safe(name)}\` is outside a submission directory or is not one of ${FILES.join(', ')}`); continue; }
+    dirs.add(m[1]);
+    seen.add(m[2]);
+    if (status !== 'added') { reasons.push(`\`${safe(name)}\` is ${safe(status)}, not added`); continue; }
+    const e = entries[name];
+    if (!e) { reasons.push(`\`${safe(name)}\` could not be read at the head commit`); continue; }
+    if (e.type !== 'file') { reasons.push(`\`${safe(name)}\` is a ${safe(e.type)}, not a regular file`); continue; }
+    if (!Number.isInteger(e.size) || e.size < 0) { reasons.push(`\`${safe(name)}\` has no usable size`); continue; }
+    if (e.size > POLICY.fileBytes) reasons.push(`\`${safe(name)}\` is ${e.size} bytes; the limit is ${POLICY.fileBytes}`);
+    total += e.size;
+  }
+  if (dirs.size > 1) reasons.push(`it touches ${dirs.size} submission directories; automatic merging takes exactly one`);
+  if (dirs.size === 1 && !reasons.length) {
+    const missing = FILES.filter((f) => !seen.has(f));
+    if (missing.length) reasons.push(`missing file(s): ${missing.join(', ')}`);
+  }
+  if (total > POLICY.totalBytes) reasons.push(`the files total ${total} bytes; the limit is ${POLICY.totalBytes}`);
+  return { ok: reasons.length === 0 && dirs.size === 1, reasons, dir: dirs.size === 1 ? [...dirs][0] : null };
+}
+
+// account: { createdAt, association, mergedLastDay, flaggedLastDay } (counts exclude this pull request).
+export function gateAccount(account, now, policy = POLICY) {
+  const reasons = [];
+  if (account.mergedLastDay >= policy.perAccountPerDay) {
+    reasons.push(`this account already has ${account.mergedLastDay} auto-merged submissions in the last 24 hours (limit ${policy.perAccountPerDay})`);
+  }
+  if (!policy.trustedAssociations.includes(account.association)) {
+    const created = Date.parse(account.createdAt);
+    if (!Number.isFinite(created)) reasons.push('the account creation date is unknown');
+    else if ((now - created) / 86400000 < policy.minAccountAgeDays) reasons.push(`the account is younger than ${policy.minAccountAgeDays} days`);
+    if (account.flaggedLastDay > 0) reasons.push('a submission from this account was refused in the last 24 hours');
+  }
+  return { ok: reasons.length === 0, reasons };
+}
+
+// facts: {
+//   validatedSha, currentSha  – the head commit the validation read, and the head right now
+//   files: gateFiles() result, validation: { ok, errors }, cla: 'success' | other | null,
+//   account: gateAccount() result
+// }
+// Returns { merge, reasons, waitForCla }. waitForCla marks a refusal that is only "not yet": the maintainer or
+// contributor fixes it by signing, and no needs-review label is warranted.
+export function decide({ validatedSha, currentSha, files, validation, cla, account }) {
+  const reasons = [];
+  if (!validatedSha || validatedSha !== currentSha) {
+    return { merge: false, reasons: ['the pull request changed after it was checked; the next run will check the new commit'], stale: true, waitForCla: false };
+  }
+  if (!files.ok) reasons.push(...files.reasons);
+  if (files.ok && !validation.ok) reasons.push(...validation.errors.map((e) => `check: ${e}`));
+  if (files.ok && validation.ok && validation.dir && validation.dir !== files.dir) reasons.push('the validated directory differs from the changed directory');
+  const claOk = cla === 'success';
+  if (!claOk) reasons.push(`the \`${POLICY.claContext}\` check is ${cla || 'missing'}; sign the CLA (see the comment from the CLA bot) and a maintainer can re-run this`);
+  if (!account.ok) reasons.push(...account.reasons);
+  const onlyCla = !claOk && reasons.length === 1;
+  return { merge: reasons.length === 0, reasons, stale: false, waitForCla: onlyCla };
+}
+
+export function renderVerdict({ merge, reasons, stale }, { dir } = {}) {
+  const marker = '<!-- skillgauntlet-automerge -->';
+  if (merge) return `${marker}\n### ✅ Merged automatically\n\n\`${safe(dir)}\` passed every check and was merged as a **community single sample**.\n`;
+  if (stale) return null;
+  return `${marker}\n### ⏸️ Not merged automatically\n\n${reasons.map((r) => `- ${r}`).join('\n')}\n\nA maintainer will look at this by hand. Nothing is wrong with asking: this is a safety rule, not a verdict on your results.\n`;
+}
+
+export function mergeTitle(dir) {
+  return `Add submission ${dir.replace(/^submissions\//, '')}`;
+}

package/lib/chart.js

@@ -31,9 +31,12 @@ function staticLines(result) {
   for (const g of result.groups) {
     const mark = g.verdict === 'pass' ? green('✓') : g.verdict === 'warn' ? yellow('!') : red('✗');
     L.push(`  ${mark} ${g.id}  ${t(g.titleKey)}`);
-    // A group that is not clean says which checks and why: a letter and a mark alone cannot be acted on.
-    for (const c of g.checks.filter((x) => x.status === 'warn' || x.status === 'fail')) {
-      L.push(`      ${c.status === 'fail' ? red('✗') : yellow('!')} ${c.id} ${t(c.titleKey)}${dim(` — ${cut(t(c.detailKey, c.detailVars), 70)}`)}`);
+    // Every check of the group with its reason: a letter and a mark alone cannot be acted on. Clean ones are dimmed.
+    for (const c of g.checks) {
+      const flagged = c.status === 'warn' || c.status === 'fail';
+      const sym = c.status === 'fail' ? red('✗') : c.status === 'warn' ? yellow('!') : c.status === 'info' ? dim('·') : green('✓');
+      const text = `${c.id} ${t(c.titleKey)} — ${cut(t(c.detailKey, c.detailVars), 70)}`;
+      L.push(`      ${sym} ${flagged ? text : dim(text)}`);
       for (const e of (c.evidence || []).slice(0, 2)) L.push(`          ${dim(`${cut(`${e.file || ''}:${e.line}`, 56)}  ${cut(e.match, 40)}`)}`);
     }
   }

package/lib/codewhale.js

@@ -65,6 +65,8 @@ function telemetryOff(bin, env) {
 
 // The verdict for one stream line, or null to keep reading.
 export function judgeLine(ev, skillName, rec) {
+  if (ev.type === 'content' && typeof ev.content === 'string') rec.said = `${rec.said || ''}${ev.content}`.slice(0, 2000);
+  if (ev.type === 'tool_use') rec.firstInput = rec.firstInput || String(ev.input?.name ?? ev.input?.query ?? ev.input?.command ?? JSON.stringify(ev.input ?? '')).slice(0, 300);
   if (ev.type === 'tool_use' && ev.name === 'load_skill') {
     rec.tools.push('load_skill');
     rec.firstTool = rec.firstTool || 'load_skill';

package/lib/execute.js

@@ -23,7 +23,7 @@ export async function executeTrials({ run, trials, doTrial, workers = 2, streakS
       if (idx >= trials.length) return;
       const { q, i } = trials[idx];
       const rec = await doTrial({ q, i });
-      const { init, ...stored } = rec;
+      const { init, said, firstInput, ...stored } = rec;
       const row = { ...stored, queryId: q.id, run: i, query: q.query, shouldTrigger: q.shouldTrigger };
       if (init && !run.meta.tools) {
         run.patch({
@@ -35,6 +35,7 @@ export async function executeTrials({ run, trials, doTrial, workers = 2, streakS
         });
       }
       run.append(row);
+      if (said || firstInput) run.appendAnswer({ queryId: q.id, run: i, said: said || '', firstInput: firstInput || '' });
       state.done += 1;
       state.streak = FAILED.has(row.verdict) ? state.streak + 1 : 0;
       if (state.streak >= streakStop && !state.stopped) {

package/lib/i18n.js

@@ -13,6 +13,8 @@ const MESSAGES = {
     srcInstalledHint: '扫 .agents/skills、.claude/skills、.codex/skills',
     srcPath: '另一个本地目录',
     srcRepo: 'GitHub 仓库（未安装）',
+    srcRepoHint: 'owner/repo · 可分享结果',
+    sourceNote: '提示：想把测试结果分享到公开榜单，最省事的是选“GitHub 仓库”：程序会自动知道技能来自哪个仓库的哪个提交。已安装的技能和本地目录也可以分享，但前提是它在带 GitHub origin 的 git 仓库里、改动已提交并推送；否则要自己补上仓库地址和提交，结果页会给出命令。',
     noInstalled: '本机没有找到已安装的技能。',
     foundInstalled: '找到 {n} 个已安装的技能。',
     scopeQ: '测哪些？',
@@ -36,7 +38,7 @@ const MESSAGES = {
     needDeepseekKey: '要用 DeepSeek 模型，请先在环境变量里设 DEEPSEEK_API_KEY（只放变量，不要把密钥写进命令行或文件）。',
     hostCodewhale: 'Codewhale',
     hostCodewhaleHint: '能测触发率（原 DeepSeek-TUI，用 DeepSeek 模型）',
-    codewhaleNote: 'Codewhale 的技能调用在事件流里是名为 load_skill 的工具调用：第一个 input.name 等于被测技能才算触发（查询和列目录不算）。试跑在一个只装了被测技能的临时 HOME 里进行，只开放 load_skill 这一个工具；用 DEEPSEEK_API_KEY，花 DeepSeek 现金。这是 Codewhale 的数字，不与其他宿主的合并比较。需要 codewhale 在 PATH 里（npm install -g codewhale --allow-scripts=codewhale）。',
+    codewhaleNote: 'Codewhale 的技能调用在事件流里是名为 load_skill 的工具调用：第一个 input.name 等于被测技能才算触发（查询和列目录不算）。试跑在一个只装了被测技能的临时 HOME 里进行，只开放 load_skill 这一个工具；用 DEEPSEEK_API_KEY。费用取决于后端和你的计划。这是 Codewhale 的数字，不与其他宿主的合并比较。需要 codewhale 在 PATH 里（npm install -g codewhale --allow-scripts=codewhale）。',
     hostCodex: 'Codex',
     hostCodexHint: '触发不可观测：只测静态与激活探针',
     codexNote: 'Codex 把技能当作宿主侧上下文注入，事件流里没有"技能被调用"这一项（只有 command_execution / file_change / mcp_tool_call 等）。所以这里不给触发率——给了就是编的。替代方案是激活探针：同一请求装与不装各跑一次，比首个动作与改动文件。',
@@ -67,9 +69,7 @@ const MESSAGES = {
     depthDeepHint: '~10 分钟 · 加只读探索与配对增益',
     providerQ: '用哪个后端？',
     provClaude: 'Claude Code 订阅',
-    provClaudeHint: '占周额度',
     provDeepseek: 'DeepSeek',
-    provDeepseekHint: '按 DeepSeek 用量计费',
     provManual: '手动输入模型名',
     modelQ: '模型',
     modelOptionalQ: '模型（直接回车 = 默认模型）',
@@ -90,7 +90,6 @@ const MESSAGES = {
     modelExpensive: '最贵',
     needBaseUrl: '需要先设置后端：\n  ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic\n  ANTHROPIC_AUTH_TOKEN=<你的 key>',
     backendLine: '后端：{backend}　模型：{model}',
-    estimate: '即将运行：{skills} 个技能 × {trials} 次试跑 ≈ 每个 {minutes} 分钟，合计约 {totalMinutes} 分钟\n费用取决于后端和计划：订阅走额度，DeepSeek 按量计费（这里不估金额）',
     confirmRun: '继续？',
     cancelled: '已取消。免费的静态层不需要确认：npx skillgauntlet static <目录>',
     staticDone: '静态层：{verdicts}',
@@ -132,6 +131,13 @@ const MESSAGES = {
     'chart.otherSkill': '改用别的技能：{name}',
     'chart.more': '…还有 {n} 题，见完整报告',
     'chart.wrongTitle': '不该触发却触发的 {n} 次',
+    modelUnknown: '{host} 的模型清单里没有“{model}”。它现在支持：{list}。请重新选一个，或选“默认模型”。',
+    updateAvail: '有新版本 {latest}（你现在用的是 {current}）。',
+    updateQ: '要更新吗？',
+    updateNow: '更新并重新启动',
+    updateSkip: '先用当前版本（这个版本不再提醒）',
+    updating: '正在下载新版 {latest}…',
+    updateFailed: '更新没有成功：{why}。继续使用当前版本 {current}。',
     waitCloning: '正在下载仓库（大仓库可能要几十秒）…',
     waitAnalyze: '正在做静态检查…',
     waitModels: '正在读取 {host} 的模型列表…',
@@ -146,7 +152,7 @@ const MESSAGES = {
     'chart.probeMixed': '读法：第一步和产出只有一项不同，信号弱；只跑了一次，不能下结论。',
     'chart.static': '静态检查（不调用模型）',
     'chart.staticLegend': '✓ 通过  ! 警告（值得看一眼）  ✗ 失败（建议处理）；每项的完整说明见完整报告',
-    noSubmitNoSource: '这个技能不是从公开的 GitHub 仓库测的（本地目录或已安装的技能），榜单需要知道它来自哪个仓库的哪个提交，所以不能提交。想提交，请用“GitHub 仓库”这个来源再测一次。',
+    noSubmitNoSource: '这次的结果暂时不能自动提交：程序看不出这个技能来自哪个公开仓库的哪个提交。\n有两种办法：\n  1. 如果技能在带 GitHub origin 的 git 仓库里，把改动提交并推送到 GitHub，再对同一目录重测（榜单会核对公开仓库里的 SKILL.md 与你测的一致）。\n  2. 你知道它的仓库和提交，直接用下面的命令补上（--subdir 是技能在仓库里的子目录，在仓库根目录就省略）：\n  {cmd}',
     'chart.trigger': '触发率：该用这个技能的时候，它被用上了吗',
     'chart.verdict': '结论',
     'chart.grade.well': '触发准确',
@@ -166,7 +172,7 @@ const MESSAGES = {
     'chart.gainLegend': '每项是一条程序能判分的要求，条越长越符合',
     'chart.full': '完整报告：{path}',
     progressLine: '试跑',
-    draftFirst: '接下来先为每个技能出题（每个技能 1 次模型调用），出完会告诉你确切的试跑次数，你同意后才开始试跑。费用取决于后端和你的计划（订阅走额度，DeepSeek 按用量计费）。',
+    draftFirst: '接下来先为每个技能出题（每个技能 1 次模型调用），出完会告诉你确切的试跑次数，你同意后才开始试跑。费用取决于后端和你的计划。',
     trialsQ: '{q} 道题 × {runs} 次 = {n} 次试跑，约 {minutes} 分钟。',
     trialsGo: '开始试跑',
     trialsAll: '开始，后面的技能不再逐个询问',
@@ -329,6 +335,36 @@ const MESSAGES = {
     'poll.body': '1) Claude Code　2) Codex　3) 其他　—— 回一个数字就行。',
     'poll.why': '为什么问：触发率只在支持显式技能调用的宿主上可测。Codex 的事件流里没有 skill item，那一列是空的，不是没测。所以读者用什么宿主，决定这份报告对谁有用、下一步该补哪一层。答案只用作信号，不做比例统计：回帖的人是自我选择的，算不出比例。',
     'trial.title': '触发率',
+    'trial.answersTitle': '逐题详情：模型说了什么、先做了什么',
+    'trial.stopNote': '试跑在模型做出第一个动作时就停止（那时已能判定有没有用上技能），所以这里只到“第一步”，不是完整回答；也不会让模型真的去执行命令。',
+    'trial.runN': '第 {n} 轮：',
+    'trial.saidFirst': '动作之前它先说：',
+    'trial.saidNothing': '动作之前没说任何话，直接行动。',
+    'trial.glance': '一眼看结果',
+    'trial.glanceRecall': '应触发的题：触发了  ',
+    'trial.glanceFpr': '不该触发的题：误触发',
+    'trial.legend': '读法：“期望”是出题时的设定；“实际”是模型真的做了什么；✓ 表示两者一致，✗ 表示不一致（该触发却没触发，或不该触发却触发了）。',
+    'trial.byClause': '题目与结果对照（按触发条款）',
+    'trial.clauseTitle': '条款：{clause}',
+    'trial.noClause': '未归类',
+    'trial.hExpect': '期望',
+    'trial.hActual': '实际',
+    'trial.hMatch': '一致',
+    'trial.hQuery': '题目',
+    'trial.hFirst': '模型先做了什么',
+    'trial.hNote': '出题说明',
+    'trial.actTriggered': '触发了',
+    'trial.actNot': '没触发',
+    'trial.actPartial': '部分触发 {k}/{n}',
+    'trial.actWrong': '误触发 {k}/{n}',
+    'trial.borrows': '（借用的词：{words}）',
+    'trial.didSkill': '调用了技能 {name}',
+    'trial.otherSkill': '改用别的技能 {name}',
+    'trial.noToolRun': '没调用工具',
+    'trial.missTitle': '该触发却没触发的 {n} 次：模型改做了什么',
+    'trial.hFirstTool': '先做的事',
+    'trial.hCount': '次数',
+    'trial.allZero': '一次都没触发：更可能是宿主没读到这个技能（路径、安装或权限），而不是技能写得差；请先核对技能装在宿主能读到的位置。',
     'trial.modelLine': '模型：{model}（{backend}）；宿主：{host}；每题 {runs} 轮，共 {trials} 次试跑',
     'trial.hostUnknown': '未记录',
     'trial.recall': '召回率：{p}（95% CI {lo}–{hi}）',
@@ -373,6 +409,8 @@ const MESSAGES = {
     srcInstalledHint: 'scans .agents/skills, .claude/skills, .codex/skills',
     srcPath: 'Another local directory',
     srcRepo: 'GitHub repository (not installed)',
+    srcRepoHint: 'owner/repo · results can be shared',
+    sourceNote: 'Tip: to share the results on the public leaderboard, the easiest is "GitHub repository": the program then knows which repository and commit the skill came from. An installed skill or a local folder can be shared too, but only if it is in a git repository with a GitHub origin and its changes are committed and pushed; otherwise you supply the repository and commit yourself, and the results page gives you the command.',
     noInstalled: 'No installed skills found on this machine.',
     foundInstalled: 'Found {n} installed skills.',
     scopeQ: 'Which ones?',
@@ -396,7 +434,7 @@ const MESSAGES = {
     needDeepseekKey: 'To use a DeepSeek model, set DEEPSEEK_API_KEY in the environment first (a variable only; keep the key out of the command line and files).',
     hostCodewhale: 'Codewhale',
     hostCodewhaleHint: 'trigger rate is measurable (formerly DeepSeek-TUI, runs DeepSeek models)',
-    codewhaleNote: 'In Codewhale a skill call is a tool call named load_skill: the first one whose input.name equals the tested skill counts as a trigger (a query or a catalogue listing does not). Trials run in a throw-away HOME holding only the tested skill, with load_skill as the only tool; it uses DEEPSEEK_API_KEY and spends DeepSeek cash. The number is a Codewhale number and is not merged with another host. It needs codewhale on PATH (npm install -g codewhale --allow-scripts=codewhale).',
+    codewhaleNote: 'In Codewhale a skill call is a tool call named load_skill: the first one whose input.name equals the tested skill counts as a trigger (a query or a catalogue listing does not). Trials run in a throw-away HOME holding only the tested skill, with load_skill as the only tool; it uses DEEPSEEK_API_KEY. Cost depends on the backend and your plan. The number is a Codewhale number and is not merged with another host. It needs codewhale on PATH (npm install -g codewhale --allow-scripts=codewhale).',
     hostCodex: 'Codex',
     hostCodexHint: 'trigger is not observable: static plus an activation probe',
     codexNote: 'Codex injects a skill as host-side context; the event stream has no "skill was used" item (only command_execution, file_change, mcp_tool_call). A trigger rate here would have to be invented, so instead there is an activation probe: the same request with and without the skill, comparing the first action and the files changed.',
@@ -427,9 +465,7 @@ const MESSAGES = {
     depthDeepHint: '~10 min · adds read-only exploration and paired gain',
     providerQ: 'Which backend?',
     provClaude: 'Claude Code subscription',
-    provClaudeHint: 'spends weekly quota',
     provDeepseek: 'DeepSeek',
-    provDeepseekHint: 'billed by DeepSeek usage',
     provManual: 'Type a model name',
     modelQ: 'Model',
     modelOptionalQ: 'Model (press Enter for the default)',
@@ -450,7 +486,6 @@ const MESSAGES = {
     modelExpensive: 'most expensive',
     needBaseUrl: 'Set the backend first:\n  ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic\n  ANTHROPIC_AUTH_TOKEN=<your key>',
     backendLine: 'Backend: {backend}   model: {model}',
-    estimate: 'About to run: {skills} skill(s) × {trials} trials ≈ {minutes} min each, {totalMinutes} min in total\nCost depends on the backend and plan: a subscription spends quota, DeepSeek bills per use (no amount is estimated here)',
     confirmRun: 'Continue?',
     cancelled: 'Cancelled. The free static layer needs no confirmation: npx skillgauntlet static <dir>',
     staticDone: 'Static: {verdicts}',
@@ -492,6 +527,13 @@ const MESSAGES = {
     'chart.otherSkill': 'used another skill: {name}',
     'chart.more': '…{n} more, see the full report',
     'chart.wrongTitle': '{n} times it triggered when it should not have',
+    modelUnknown: '{host} does not list a model called "{model}". It currently supports: {list}. Pick another, or choose the default model.',
+    updateAvail: 'A newer version {latest} is available (you are on {current}).',
+    updateQ: 'Update?',
+    updateNow: 'Update and restart',
+    updateSkip: 'Keep this version (do not ask about this one again)',
+    updating: 'Downloading {latest}…',
+    updateFailed: 'The update did not work: {why}. Continuing with {current}.',
     waitCloning: 'Downloading the repository (a big one can take tens of seconds)…',
     waitAnalyze: 'Running the static checks…',
     waitModels: 'Reading the model list from {host}…',
@@ -506,7 +548,7 @@ const MESSAGES = {
     'chart.probeMixed': 'Reading: only one of the first step and the output differs, a weak signal; one run each is not a conclusion.',
     'chart.static': 'Static checks (no model calls)',
     'chart.staticLegend': '✓ pass  ! warning (worth a look)  ✗ fail (should be handled); each check is explained in the full report',
-    noSubmitNoSource: 'This skill was not tested from a public GitHub repository (a local folder or an installed skill). The leaderboard needs to know which repository and commit it came from, so it cannot be submitted. To submit, test it again using the "GitHub repository" source.',
+    noSubmitNoSource: 'These results cannot be submitted automatically yet: the program cannot tell which public repository and commit this skill came from.\nTwo ways forward:\n  1. If the skill is in a git repository with a GitHub origin, commit and push your changes, then test the same folder again (the leaderboard checks that SKILL.md in the public repository matches what you tested).\n  2. If you know its repository and commit, add them with the command below (--subdir is the skill\'s folder inside the repository; leave it out at the repository root):\n  {cmd}',
     'chart.trigger': 'Trigger rate: is the skill used when it should be?',
     'chart.verdict': 'Verdict',
     'chart.grade.well': 'triggers accurately',
@@ -526,7 +568,7 @@ const MESSAGES = {
     'chart.gainLegend': 'Each item is a requirement a program can grade; a longer bar means more of it is met',
     'chart.full': 'Full report: {path}',
     progressLine: 'Trials',
-    draftFirst: 'Next, questions are drafted for each skill (one model call per skill). You will then be told the exact number of trials and nothing is run until you agree. Cost depends on the backend and your plan (a subscription spends quota, DeepSeek bills per use).',
+    draftFirst: 'Next, questions are drafted for each skill (one model call per skill). You will then be told the exact number of trials and nothing is run until you agree. Cost depends on the backend and your plan.',
     trialsQ: '{q} questions × {runs} run(s) = {n} trials, about {minutes} min.',
     trialsGo: 'Start the trials',
     trialsAll: 'Start, and do not ask again for the next skills',
@@ -685,6 +727,36 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'poll.body': '1) Claude Code　2) Codex　3) Other — reply with a single digit.',
     'poll.why': 'Why ask: a trigger rate is only measurable on a host that exposes explicit skill invocation. Codex has no skill item in its event stream, so its column is empty rather than unmeasured. Which host a reader uses therefore decides who the report serves and which layer to build next. The answers are read as a signal, never as a proportion: whoever replies has selected themselves, so no ratio can be computed from it.',
     'trial.title': 'Trigger rate',
+    'trial.answersTitle': 'Per question: what the model said and did first',
+    'trial.stopNote': 'A trial stops at the model\'s first action (by then it is known whether the skill was used), so this goes only up to the first step. It is not a complete answer, and the model is never let run its commands.',
+    'trial.runN': 'Run {n}:',
+    'trial.saidFirst': 'Before acting it said:',
+    'trial.saidNothing': 'It said nothing before acting.',
+    'trial.glance': 'At a glance',
+    'trial.glanceRecall': 'Should trigger: triggered   ',
+    'trial.glanceFpr': 'Should not: triggered       ',
+    'trial.legend': 'How to read: "Expected" is what the question was written to do; "Actual" is what the model really did; ✓ means they agree, ✗ that they do not (a miss, or a false trigger).',
+    'trial.byClause': 'Questions and results, by trigger clause',
+    'trial.clauseTitle': 'Clause: {clause}',
+    'trial.noClause': 'Unassigned',
+    'trial.hExpect': 'Expected',
+    'trial.hActual': 'Actual',
+    'trial.hMatch': 'Agree',
+    'trial.hQuery': 'Question',
+    'trial.hFirst': 'What the model did first',
+    'trial.hNote': 'Why it was written',
+    'trial.actTriggered': 'triggered',
+    'trial.actNot': 'did not trigger',
+    'trial.actPartial': 'triggered {k}/{n}',
+    'trial.actWrong': 'false trigger {k}/{n}',
+    'trial.borrows': '(borrowed words: {words})',
+    'trial.didSkill': 'invoked skill {name}',
+    'trial.otherSkill': 'used another skill: {name}',
+    'trial.noToolRun': 'no tool',
+    'trial.missTitle': 'The {n} misses: what the model did instead',
+    'trial.hFirstTool': 'First thing it did',
+    'trial.hCount': 'Count',
+    'trial.allZero': 'Nothing triggered at all: more likely the host could not read this skill (path, install or permissions) than that the skill is badly written. Check that it sits where the host can read it.',
     'trial.modelLine': 'Model: {model} ({backend}); host: {host}; {runs} runs per query, {trials} trials',
     'trial.hostUnknown': 'not recorded',
     'trial.recall': 'Recall: {p} (95% CI {lo}–{hi})',
@@ -728,6 +800,8 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     srcInstalledHint: '.agents/skills、.claude/skills、.codex/skills を走査',
     srcPath: '別のローカルディレクトリ',
     srcRepo: 'GitHub リポジトリ（未インストール）',
+    srcRepoHint: 'owner/repo · 結果を共有できます',
+    sourceNote: 'ヒント：結果を公開リーダーボードで共有するなら「GitHub リポジトリ」が最も簡単です。スキルがどのリポジトリのどのコミットかをプログラムが自動で把握します。インストール済みのスキルやローカルフォルダも共有できますが、GitHub の origin を持つ git リポジトリにあり、変更がコミット・プッシュ済みであることが条件です。そうでなければリポジトリとコミットを自分で指定します（結果画面にコマンドが表示されます）。',
     noInstalled: 'このマシンにインストール済みのスキルが見つかりません。',
     foundInstalled: 'インストール済みのスキルが {n} 件見つかりました。',
     scopeQ: 'どれをテストしますか？',
@@ -753,7 +827,7 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     needDeepseekKey: 'DeepSeek のモデルを使うには、先に環境変数 DEEPSEEK_API_KEY を設定してください（変数にのみ置き、キーをコマンドラインやファイルに書かないでください）。',
     hostCodewhale: 'Codewhale',
     hostCodewhaleHint: '発動率を測定できます（旧 DeepSeek-TUI、DeepSeek のモデルで動作）',
-    codewhaleNote: 'Codewhale では、スキルの呼び出しは load_skill という名前のツール呼び出しです。最初の呼び出しの input.name が対象スキルと一致した場合のみ発動とみなします（検索や一覧表示は含みません）。試行は対象スキルだけを入れた使い捨ての HOME で行い、使えるツールは load_skill のみです。DEEPSEEK_API_KEY を使い、DeepSeek の現金がかかります。これは Codewhale の数値であり、他のホストの数値とは合算・比較しません。codewhale が PATH 上に必要です（npm install -g codewhale --allow-scripts=codewhale）。',
+    codewhaleNote: 'Codewhale では、スキルの呼び出しは load_skill という名前のツール呼び出しです。最初の呼び出しの input.name が対象スキルと一致した場合のみ発動とみなします（検索や一覧表示は含みません）。試行は対象スキルだけを入れた使い捨ての HOME で行い、使えるツールは load_skill のみです。DEEPSEEK_API_KEY を使います。費用はバックエンドとプランによります。これは Codewhale の数値であり、他のホストの数値とは合算・比較しません。codewhale が PATH 上に必要です（npm install -g codewhale --allow-scripts=codewhale）。',
     codexNote: 'Codex はスキルをホスト側のコンテキストとして注入するため、イベントストリームに「スキルが使われた」という項目がありません（command_execution / file_change / mcp_tool_call などのみ）。ここで発動率を出すと作り話になるので出しません。代わりに有効化プローブを使います：同じリクエストをスキルあり・なしで 1 回ずつ実行し、最初の動作と変更されたファイルを比べます。',
     probeQ: '有効化プローブを実行しますか？（2 回実行）',
     probeRun: 'プローブ実行中（スキルあり、次にスキルなし）…',
@@ -782,9 +856,7 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     depthDeepHint: '約 10 分 · 読み取り専用の探索とペア比較の効果測定を追加',
     providerQ: 'どのバックエンドを使いますか？',
     provClaude: 'Claude Code サブスクリプション',
-    provClaudeHint: '週間の利用枠を消費',
     provDeepseek: 'DeepSeek',
-    provDeepseekHint: 'DeepSeek の従量課金',
     provManual: 'モデル名を手入力',
     modelQ: 'モデル',
     modelOptionalQ: 'モデル（Enter で既定のモデル）',
@@ -805,7 +877,6 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     modelExpensive: '最も高価',
     needBaseUrl: '先にバックエンドを設定してください：\n  ANTHROPIC_BASE_URL=https://api.deepseek.com/anthropic\n  ANTHROPIC_AUTH_TOKEN=<あなたの key>',
     backendLine: 'バックエンド：{backend}　モデル：{model}',
-    estimate: '実行予定：{skills} 件のスキル × {trials} 回の試行 ≈ 1 件あたり {minutes} 分、合計約 {totalMinutes} 分\n費用はバックエンドとプランによります：サブスクリプションは利用枠を消費し、DeepSeek は従量課金です（ここでは金額を見積もりません）',
     confirmRun: '続行しますか？',
     cancelled: 'キャンセルしました。無料の静的チェックは確認不要です：npx skillgauntlet static <ディレクトリ>',
     staticDone: '静的チェック：{verdicts}',
@@ -900,6 +971,13 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'chart.otherSkill': '別のスキルを使用：{name}',
     'chart.more': '…ほか {n} 件は詳細レポートへ',
     'chart.wrongTitle': '発動すべきでないのに発動した {n} 回',
+    modelUnknown: '{host} のモデル一覧に「{model}」はありません。現在対応：{list}。別のものを選ぶか、既定のモデルを選んでください。',
+    updateAvail: '新しいバージョン {latest} があります（現在は {current}）。',
+    updateQ: '更新しますか？',
+    updateNow: '更新して再起動',
+    updateSkip: '今のバージョンのまま使う（このバージョンは再通知しない）',
+    updating: '{latest} をダウンロード中…',
+    updateFailed: '更新に失敗しました：{why}。現在の {current} で続行します。',
     waitCloning: 'リポジトリをダウンロード中（大きいものは数十秒かかります）…',
     waitAnalyze: '静的チェックを実行中…',
     waitModels: '{host} のモデル一覧を読み込み中…',
@@ -914,7 +992,7 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'chart.probeMixed': '読み方：最初の動作と成果物のどちらか一方だけが異なり、弱い信号です。各 1 回では結論は出せません。',
     'chart.static': '静的チェック（モデル呼び出しなし）',
     'chart.staticLegend': '✓ 合格  ! 警告（確認推奨）  ✗ 不合格（対処推奨）。各項目の説明は詳細レポートに',
-    noSubmitNoSource: 'このスキルは公開 GitHub リポジトリからテストされていません（ローカルフォルダまたはインストール済み）。リーダーボードにはどのリポジトリのどのコミットかが必要なため提出できません。提出するには「GitHub リポジトリ」から選んでもう一度テストしてください。',
+    noSubmitNoSource: '今回の結果はまだ自動提出できません：このスキルがどの公開リポジトリのどのコミットか、プログラムには分かりません。\n方法は 2 つあります：\n  1. スキルが GitHub の origin を持つ git リポジトリにあるなら、変更をコミットして GitHub にプッシュし、同じフォルダをもう一度テストします（リーダーボードは公開リポジトリの SKILL.md がテストした内容と一致するか確認します）。\n  2. リポジトリとコミットが分かっているなら、次のコマンドで補います（--subdir はリポジトリ内のスキルのフォルダ。リポジトリ直下なら省略）：\n  {cmd}',
     'chart.trigger': '発動率：使うべき時にスキルが使われるか',
     'chart.verdict': '評価',
     'chart.grade.well': '正確に発動',
@@ -934,7 +1012,7 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'chart.gainLegend': '各項目はプログラムが採点できる要件です。バーが長いほど満たしています',
     'chart.full': '詳細レポート：{path}',
     progressLine: '試行',
-    draftFirst: 'まず各スキルのテスト用質問を作成します（スキルごとにモデル呼び出し 1 回）。その後、試行回数を正確にお知らせし、同意いただいてから試行を開始します。費用はバックエンドとプランによります（サブスクリプションは利用枠を消費、DeepSeek は従量課金）。',
+    draftFirst: 'まず各スキルのテスト用質問を作成します（スキルごとにモデル呼び出し 1 回）。その後、試行回数を正確にお知らせし、同意いただいてから試行を開始します。費用はバックエンドとプランによります。',
     trialsQ: '質問 {q} 件 × {runs} 回 = {n} 回の試行、約 {minutes} 分。',
     trialsGo: '試行を開始する',
     trialsAll: '開始し、以降のスキルでは確認しない',
@@ -1040,6 +1118,36 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'poll.body': '1) Claude Code　2) Codex　3) その他　—— 数字 1 つで答えてください。',
     'poll.why': '質問の理由：発動率は、スキルの明示的な呼び出しを公開しているホストでしか測れません。Codex のイベントストリームには skill 項目がないため、その列は「未測定」ではなく空欄です。読者がどのホストを使っているかで、このレポートが誰の役に立つか、次にどの層を作るべきかが決まります。回答はシグナルとしてのみ読み、割合としては扱いません。回答する人は自己選択であり、そこから比率は計算できないためです。',
     'trial.title': '発動率',
+    'trial.answersTitle': '質問ごとの詳細：モデルが言ったこと・最初にしたこと',
+    'trial.stopNote': '試行はモデルの最初の動作で止まります（その時点でスキルが使われたか分かるため）。ここは最初のステップまでで、完全な回答ではなく、モデルにコマンドを実行させることもありません。',
+    'trial.runN': '{n} 回目：',
+    'trial.saidFirst': '動作の前に言ったこと：',
+    'trial.saidNothing': '動作の前には何も言わず、すぐ行動しました。',
+    'trial.glance': '結果の概要',
+    'trial.glanceRecall': '発動すべき質問：発動  ',
+    'trial.glanceFpr': '発動すべきでない質問：誤発動',
+    'trial.legend': '読み方：「期待」は質問の設計意図、「実際」はモデルが実際に行ったこと。✓ は一致、✗ は不一致（未発動、または誤発動）。',
+    'trial.byClause': '質問と結果の対照（発動条件ごと）',
+    'trial.clauseTitle': '条件：{clause}',
+    'trial.noClause': '未分類',
+    'trial.hExpect': '期待',
+    'trial.hActual': '実際',
+    'trial.hMatch': '一致',
+    'trial.hQuery': '質問',
+    'trial.hFirst': 'モデルが最初にしたこと',
+    'trial.hNote': '出題の意図',
+    'trial.actTriggered': '発動',
+    'trial.actNot': '発動せず',
+    'trial.actPartial': '一部発動 {k}/{n}',
+    'trial.actWrong': '誤発動 {k}/{n}',
+    'trial.borrows': '（借用した語：{words}）',
+    'trial.didSkill': 'スキル {name} を呼び出し',
+    'trial.otherSkill': '別のスキルを使用：{name}',
+    'trial.noToolRun': 'ツールなし',
+    'trial.missTitle': '発動すべきなのに発動しなかった {n} 回：代わりに何をしたか',
+    'trial.hFirstTool': '最初にしたこと',
+    'trial.hCount': '回数',
+    'trial.allZero': '一度も発動しませんでした。スキルの出来が悪いというより、ホストがこのスキルを読めていない（パス・インストール・権限）可能性が高いです。ホストが読める場所にあるか確認してください。',
     'trial.modelLine': 'モデル：{model}（{backend}）；ホスト：{host}；1 問あたり {runs} 回、計 {trials} 回の試行',
     'trial.hostUnknown': '記録なし',
     'trial.recall': '再現率（発動すべき質問で実際に発動した割合）：{p}（95% 信頼区間 {lo}–{hi}）',

package/lib/models.js

@@ -36,3 +36,11 @@ export function hostModels(host, bin, exec = run) {
   } catch { /* the caller falls back to typing */ }
   return [];
 }
+
+// A model the user typed that the host does not list would fail only after the question-drafting call started. Names the
+// package turns into a DeepSeek model of its own (deepseek-*) are exempt: they are not in the host's list by design.
+// Returns the host's current list when the name is not in it, else null (also null when the list could not be read).
+export function unknownModel(model, known) {
+  if (!model || !known?.length || known.includes(model) || /^deepseek/i.test(model)) return null;
+  return known;
+}

package/lib/report.js

@@ -39,7 +39,98 @@ export function staticMarkdown(result) {
   return L.join('\n');
 }
 
-export function trialSection(trial) {
+const bar = (v, w = 20) => {
+  const n = Math.round(Math.min(1, Math.max(0, v ?? 0)) * w);
+  return '█'.repeat(n) + '░'.repeat(w - n);
+};
+const cell = (x) => String(x ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ').trim();
+
+// What the model did in one run, in words.
+function didWhat(r) {
+  if (r.verdict === 'triggered') return t('trial.didSkill', { name: r.invokedSkill || '?' });
+  if (r.verdict === 'other_skill') return t('trial.otherSkill', { name: r.invokedSkill || '?' });
+  return r.firstTool || t('trial.noToolRun');
+}
+
+// One row per question, grouped under the trigger clause it was written for: what was expected, what happened, whether
+// they agree, what the model did instead, and why the question was written (for a near miss, the words it borrows).
+function trialDetail(trial, records, answers) {
+  const byQuery = new Map();
+  for (const r of records) {
+    if (!byQuery.has(r.queryId)) byQuery.set(r.queryId, []);
+    byQuery.get(r.queryId).push(r);
+  }
+  const L = [];
+  const pos = trial.questions.filter((q) => q.shouldTrigger);
+  const neg = trial.questions.filter((q) => !q.shouldTrigger);
+  const hitP = pos.reduce((a, q) => a + q.hits, 0);
+  const nP = pos.reduce((a, q) => a + q.n, 0);
+  const hitN = neg.reduce((a, q) => a + q.hits, 0);
+  const nN = neg.reduce((a, q) => a + q.n, 0);
+  L.push('', `### ${t('trial.glance')}`, '', '```');
+  L.push(`${t('trial.glanceRecall')}  ${bar(trial.recall)} ${pct(trial.recall)}  (${hitP}/${nP})`);
+  L.push(`${t('trial.glanceFpr')}  ${bar(trial.fpr)} ${pct(trial.fpr)}  (${hitN}/${nN})`);
+  L.push('```', '', t('trial.legend'));
+
+  // group by clause, in the order the clauses were drafted
+  const clauses = [];
+  for (const q of trial.queries || []) if (!clauses.includes(q.clause || '')) clauses.push(q.clause || '');
+  L.push('', `### ${t('trial.byClause')}`);
+  for (const clause of clauses) {
+    const qs = (trial.queries || []).filter((q) => (q.clause || '') === clause);
+    L.push('', `#### ${clause ? t('trial.clauseTitle', { clause: cell(clause) }) : t('trial.noClause')}`, '');
+    L.push(`| ${t('trial.hExpect')} | ${t('trial.hActual')} | ${t('trial.hMatch')} | ${t('trial.hQuery')} | ${t('trial.hFirst')} | ${t('trial.hNote')} |`);
+    L.push('|---|---|---|---|---|---|');
+    for (const q of [...qs.filter((x) => x.shouldTrigger), ...qs.filter((x) => !x.shouldTrigger)]) {
+      const runs = byQuery.get(q.id) || [];
+      const hits = runs.filter((r) => r.verdict === 'triggered').length;
+      const n = runs.length;
+      let actual;
+      let ok;
+      if (!n) { actual = '—'; ok = '—'; } else if (q.shouldTrigger) {
+        actual = hits === n ? t('trial.actTriggered') : hits === 0 ? t('trial.actNot') : t('trial.actPartial', { k: hits, n });
+        ok = hits * 2 > n ? '✓' : '✗';
+      } else {
+        actual = hits === 0 ? t('trial.actNot') : t('trial.actWrong', { k: hits, n });
+        ok = hits === 0 ? '✓' : '✗';
+      }
+      const first = [...new Set(runs.map(didWhat))].join('; ') || '—';
+      const note = [cell(q.note), q.shares ? t('trial.borrows', { words: cell(q.shares) }) : ''].filter(Boolean).join(' ');
+      L.push(`| ${q.shouldTrigger ? t('trial.should') : t('trial.shouldNot')} | ${actual} | ${ok} | ${cell(q.query)} | ${cell(first)} | ${note} |`);
+    }
+  }
+
+  const missed = records.filter((r) => r.shouldTrigger && r.verdict !== 'triggered');
+  if (missed.length) {
+    const kinds = new Map();
+    for (const r of missed) kinds.set(didWhat(r), (kinds.get(didWhat(r)) || 0) + 1);
+    L.push('', `### ${t('trial.missTitle', { n: missed.length })}`, '');
+    L.push(`| ${t('trial.hFirstTool')} | ${t('trial.hCount')} | |`, '|---|---|---|');
+    for (const [k, c] of [...kinds].sort((a, b) => b[1] - a[1])) L.push(`| ${cell(k)} | ${c} | ${bar(c / missed.length, 16)} |`);
+  }
+  if (trial.recall === 0 && trial.fpr === 0) L.push('', `> ${t('trial.allZero')}`);
+
+  // What the model said and did first, question by question. A trial stops at the first action, so this is not a
+  // complete answer, and the report says so.
+  if (answers?.size) {
+    L.push('', `### ${t('trial.answersTitle')}`, '', `> ${t('trial.stopNote')}`);
+    for (const q of trial.queries || []) {
+      const runs = (byQuery.get(q.id) || []);
+      if (!runs.length) continue;
+      L.push('', `**${q.id}** · ${q.shouldTrigger ? t('trial.should') : t('trial.shouldNot')} · ${cell(q.query)}`);
+      for (const r of runs) {
+        const a = answers.get(`${r.queryId}#${r.run}`);
+        const tag = runs.length > 1 ? `${t('trial.runN', { n: r.run + 1 })} ` : '';
+        L.push('', `- ${tag}${didWhat(r)}${a?.firstInput ? `: \`${cell(a.firstInput).replace(/`/g, "'")}\`` : ''}`);
+        if (a?.said?.trim()) L.push(`  - ${t('trial.saidFirst')}`, '', ...a.said.trim().split(/\r?\n/).map((l) => `    > ${l}`));
+        else L.push(`  - ${t('trial.saidNothing')}`);
+      }
+    }
+  }
+  return L;
+}
+
+export function trialSection(trial, records = null, answers = null) {
   if (!trial) return '';
   const L = ['', `## ${t('trial.title')}`, ''];
   L.push(`- ${t('trial.modelLine', {
@@ -54,6 +145,10 @@ export function trialSection(trial) {
   if (trial.noToolRate != null) {
     L.push(`- ${t('trial.noTool', { p: pct(trial.noToolRate) })}${trial.noToolRate >= 0.5 ? t('trial.noToolWarn') : ''}`);
   }
+  if (records?.length && trial.queries?.length) {
+    L.push(...trialDetail(trial, records, answers));
+    return L.join('\n');
+  }
   L.push('');
   L.push(`| ${t('trial.stateHeader')} | ${t('trial.queryHeader')} | ${t('trial.hitsHeader')} |`);
   L.push('|---|---|---|');