skillgauntlet 1.3.0 → 1.3.2

package/bin/skillgauntlet.js

@@ -10,13 +10,14 @@ import { findSkills, resolveSpec, skillNameOf } from '../lib/discover.js';
 import { installedSkills } from '../lib/installed.js';
 import { probeActivation } from '../lib/codex.js';
 import { fingerprint, cacheFileFor, readCachedQueries, writeCachedQueries } from '../lib/queries.js';
-import { openRun, querySetHash, readRun, validRecords } from '../lib/runs.js';
+import { openRun, querySetHash, readRun, readAnswers, validRecords } from '../lib/runs.js';
 import { DEPTHS, minutesFor } from '../lib/depth.js';
 import { ask, choose, select, confirm, say, closeUi, hasTerminal, setBack, Back } from '../lib/ui.js';
-import { hostModels, unknownModel } from '../lib/models.js';
+import { hostModels, unknownModel, codebuddyCatalog } from '../lib/models.js';
 import { browseDir } from '../lib/browse.js';
 import { withTicker } from '../lib/ticker.js';
-import { renderResult, bar } from '../lib/chart.js';
+import { renderResult } from '../lib/chart.js';
+import { makeProgress } from '../lib/progress.js';
 import { setLang, detectLang, errorText, t } from '../lib/i18n.js';
 import { resolveBin, backend, backendFor, usesOwnDeepseek, setNativeModels, HOSTS, checkModel, draftQueries, draftQuality, runTrial, requireHost, pingHost, setHostBin, binInstalled } from '../lib/trial.js';
 import { HOST_CLI, searchHost, installCommand, installHost } from '../lib/findhost.js';
@@ -127,6 +128,13 @@ function notListed(model, known, label) {
   say(`\n${t('modelUnknown', { model, host: label, list: list.slice(0, 8).join(', ') })}`);
   return true;
 }
+// Models in CodeBuddy's cached catalog that its --help does not list for this account right now.
+function catalogOnly(host) {
+  if (host !== 'codebuddy') return [];
+  const listed = new Set(modelsOf('codebuddy', 'CodeBuddy'));
+  return codebuddyCatalog().filter((m) => !listed.has(m) && !m.startsWith('custom-local:'));
+}
+
 function modelsOf(host, label) {
   if (!modelCache.has(host)) {
     const list = withTicker(waitText(t('waitModels', { host: label }), { slowAfter: 600 }), () => hostModels(host, resolveBin(host)));
@@ -205,20 +213,23 @@ async function runTriggerLayer({ skillDir, skillName, depth, model, host = 'clau
     queries: { schema: 'skillgauntlet/queries/1', queries: qs.queries },
   });
 
+  const prog = progress(trials.length);
   try {
     const outcome = await executeTrials({
       run,
       trials,
       doTrial: ({ q }) => runTrial({ bin, skillDir, skillName, query: q.query, model, maxExplore: depth.maxExplore, host }),
-      onTrial: progress(),
+      onTrial: prog,
     });
+    prog.finish();
     const result = finishRun(run, { model, backend: backendFor(host, model), host, runs: depth.runs, expected: trials.length }, outcome.stopped);
     say(`  ${t('runSaved', { path: run.dir })}`);
     if (outcome.stopped) reportStopped(outcome.stopped, run.dir, model);
     else if (result?.incomplete) reportIncomplete(result.incomplete, run.dir, model);
     else finishedRuns.push(run.dir);
-    return result && { ...result, runDir: run.dir };
+    return result && { ...result, runDir: run.dir, queries: qs.queries };
   } catch (err) {
+    prog.finish();
     run.close(null, { interrupted: true });
     throw err;
   }
@@ -250,17 +261,7 @@ function archiveProbe({ skillDir, skillName, model, probe }) {
   say(`  ${t('runSaved', { path: run.dir })}`);
 }
 
-function progress() {
-  return ({ row, done, total }) => {
-    // On a terminal one line is redrawn in place; the verdicts of single trials belong to the result, not the wait.
-    if (process.stdout.isTTY) {
-      process.stdout.write(`\r\x1b[2K  ${t('progressLine')} ${bar(done / total, 24)} ${done}/${total}${done >= total ? '\n' : ''}`);
-      return;
-    }
-    const mark = { triggered: '✓', not_triggered: '·', other_skill: '↷' }[row.verdict] || '!';
-    say(`  [${done}/${total}] ${mark} ${row.queryId}#${row.run} → ${row.verdict}`);
-  };
-}
+const progress = makeProgress;
 
 function reportIncomplete(n, dir, model) {
   say(`
@@ -286,12 +287,14 @@ async function cmdResume(args) {
   const runMeta = JSON.parse(fs.readFileSync(path.join(full, 'meta.json'), 'utf8'));
   const runHost = runMeta.host || 'claude-code';
   requireHost(runHost);
+  const resumeProgress = progress();
   const result = await resumeRun({
     dir: full,
     bin: resolveBin((HOSTS[runHost] || HOSTS['claude-code']).bin),
     want: { model: flag('--model'), runs: flag('--runs'), maxExplore: flag('--max-explore'), backend: backendFor(runHost, flag('--model') ?? runMeta.model) },
-    onTrial: progress(),
+    onTrial: resumeProgress,
   });
+  resumeProgress.finish();
   if (result.nothing) {
     say(t('resumeNothing'));
     return;
@@ -320,12 +323,14 @@ async function pickInstalled() {
 }
 
 // A model for hosts that have an account default: pick it, a known name, or type one. Returns null for the default.
-async function pickModel(known = []) {
+async function pickModel(known = [], more = []) {
   if (!hasTerminal()) return (await ask(t('modelOptionalQ'), '')).trim() || null;
   const other = '\u0000other';
+  // `known` is what the account can use now; `more` is in the host's catalog but not promised to the account
   const v = await select(t('modelQ'), [
     { label: t('modelDefault'), value: '' },
-    ...known.map((m) => ({ label: m, value: m })),
+    ...known.map((m) => ({ label: m, hint: more.length ? t('modelAccountHint') : '', value: m })),
+    ...more.map((m) => ({ label: m, hint: t('modelCatalogHint'), value: m })),
     { label: t('modelOther'), value: other },
   ], 0);
   if (v === other) return (await ask(t('modelOptionalQ'), '')).trim() || null;
@@ -540,8 +545,8 @@ async function interactive() {
         { label: t('depthStandard'), hint: t('depthStandardHint'), value: 'standard' },
       ], 1);
       S.depth = DEPTHS[depthId];
-      S.model = await pickModel(modelsOf('codebuddy', 'CodeBuddy'));
-      if (notListed(S.model, modelsOf('codebuddy', 'CodeBuddy'), 'CodeBuddy')) return AGAIN;
+      S.model = await pickModel(modelsOf('codebuddy', 'CodeBuddy'), catalogOnly('codebuddy'));
+      if (notListed(S.model, [...modelsOf('codebuddy', 'CodeBuddy'), ...catalogOnly('codebuddy')], 'CodeBuddy')) return AGAIN;
       if (usesOwnDeepseek('codebuddy', S.model) && !process.env.DEEPSEEK_API_KEY) {
         say(`\n${t('needDeepseekKey')}`);
         return END;
@@ -630,10 +635,11 @@ async function interactive() {
       archiveProbe({ skillDir: target.dir, skillName: target.name, model, probe });
     }
 
-    const md = `${staticMarkdown(result)}${trialSection(trial)}${gainSection(gain)}${probeSection(probe)}`;
-    const out = writeOut(target.dir, target.name, md, sc);
     let records = null;
-    if (trial?.runDir) { try { records = validRecords(readRun(trial.runDir).records); } catch { /* the chart then shows the totals only */ } }
+    if (trial?.runDir) { try { records = validRecords(readRun(trial.runDir).records); } catch { /* the report then shows the totals only */ } }
+    const answers = trial?.runDir ? readAnswers(trial.runDir) : null;
+    const md = `${staticMarkdown(result)}${trialSection(trial, records, answers)}${gainSection(gain)}${probeSection(probe)}`;
+    const out = writeOut(target.dir, target.name, md, sc);
     outcomes.push({ name: target.name, result, trial, records, gain, probe, reportPath: path.join(out, `${target.name}.md`) });
     // The run stopped on a failure streak (usage limit, dead token): the next skill would fail the same way.
     if ((host === 'claude' || host === 'codebuddy' || host === 'codewhale') && process.exitCode === 3) break;

package/lib/codewhale.js

@@ -65,6 +65,8 @@ function telemetryOff(bin, env) {
 
 // The verdict for one stream line, or null to keep reading.
 export function judgeLine(ev, skillName, rec) {
+  if (ev.type === 'content' && typeof ev.content === 'string') rec.said = `${rec.said || ''}${ev.content}`.slice(0, 2000);
+  if (ev.type === 'tool_use') rec.firstInput = rec.firstInput || String(ev.input?.name ?? ev.input?.query ?? ev.input?.command ?? JSON.stringify(ev.input ?? '')).slice(0, 300);
   if (ev.type === 'tool_use' && ev.name === 'load_skill') {
     rec.tools.push('load_skill');
     rec.firstTool = rec.firstTool || 'load_skill';

package/lib/execute.js

@@ -23,7 +23,7 @@ export async function executeTrials({ run, trials, doTrial, workers = 2, streakS
       if (idx >= trials.length) return;
       const { q, i } = trials[idx];
       const rec = await doTrial({ q, i });
-      const { init, ...stored } = rec;
+      const { init, said, firstInput, ...stored } = rec;
       const row = { ...stored, queryId: q.id, run: i, query: q.query, shouldTrigger: q.shouldTrigger };
       if (init && !run.meta.tools) {
         run.patch({
@@ -35,6 +35,7 @@ export async function executeTrials({ run, trials, doTrial, workers = 2, streakS
         });
       }
       run.append(row);
+      if (said || firstInput) run.appendAnswer({ queryId: q.id, run: i, said: said || '', firstInput: firstInput || '' });
       state.done += 1;
       state.streak = FAILED.has(row.verdict) ? state.streak + 1 : 0;
       if (state.streak >= streakStop && !state.stopped) {

package/lib/i18n.js

@@ -131,6 +131,8 @@ const MESSAGES = {
     'chart.otherSkill': '改用别的技能：{name}',
     'chart.more': '…还有 {n} 题，见完整报告',
     'chart.wrongTitle': '不该触发却触发的 {n} 次',
+    modelAccountHint: '你的账号现在可用',
+    modelCatalogHint: 'CodeBuddy 目录里有，账号不一定能用',
     modelUnknown: '{host} 的模型清单里没有“{model}”。它现在支持：{list}。请重新选一个，或选“默认模型”。',
     updateAvail: '有新版本 {latest}（你现在用的是 {current}）。',
     updateQ: '要更新吗？',
@@ -335,6 +337,36 @@ const MESSAGES = {
     'poll.body': '1) Claude Code　2) Codex　3) 其他　—— 回一个数字就行。',
     'poll.why': '为什么问：触发率只在支持显式技能调用的宿主上可测。Codex 的事件流里没有 skill item，那一列是空的，不是没测。所以读者用什么宿主，决定这份报告对谁有用、下一步该补哪一层。答案只用作信号，不做比例统计：回帖的人是自我选择的，算不出比例。',
     'trial.title': '触发率',
+    'trial.answersTitle': '逐题详情：模型说了什么、先做了什么',
+    'trial.stopNote': '试跑在模型做出第一个动作时就停止（那时已能判定有没有用上技能），所以这里只到“第一步”，不是完整回答；也不会让模型真的去执行命令。',
+    'trial.runN': '第 {n} 轮：',
+    'trial.saidFirst': '动作之前它先说：',
+    'trial.saidNothing': '动作之前没说任何话，直接行动。',
+    'trial.glance': '一眼看结果',
+    'trial.glanceRecall': '应触发的题：触发了  ',
+    'trial.glanceFpr': '不该触发的题：误触发',
+    'trial.legend': '读法：“期望”是出题时的设定；“实际”是模型真的做了什么；✓ 表示两者一致，✗ 表示不一致（该触发却没触发，或不该触发却触发了）。',
+    'trial.byClause': '题目与结果对照（按触发条款）',
+    'trial.clauseTitle': '条款：{clause}',
+    'trial.noClause': '未归类',
+    'trial.hExpect': '期望',
+    'trial.hActual': '实际',
+    'trial.hMatch': '一致',
+    'trial.hQuery': '题目',
+    'trial.hFirst': '模型先做了什么',
+    'trial.hNote': '出题说明',
+    'trial.actTriggered': '触发了',
+    'trial.actNot': '没触发',
+    'trial.actPartial': '部分触发 {k}/{n}',
+    'trial.actWrong': '误触发 {k}/{n}',
+    'trial.borrows': '（借用的词：{words}）',
+    'trial.didSkill': '调用了技能 {name}',
+    'trial.otherSkill': '改用别的技能 {name}',
+    'trial.noToolRun': '没调用工具',
+    'trial.missTitle': '该触发却没触发的 {n} 次：模型改做了什么',
+    'trial.hFirstTool': '先做的事',
+    'trial.hCount': '次数',
+    'trial.allZero': '一次都没触发：更可能是宿主没读到这个技能（路径、安装或权限），而不是技能写得差；请先核对技能装在宿主能读到的位置。',
     'trial.modelLine': '模型：{model}（{backend}）；宿主：{host}；每题 {runs} 轮，共 {trials} 次试跑',
     'trial.hostUnknown': '未记录',
     'trial.recall': '召回率：{p}（95% CI {lo}–{hi}）',
@@ -497,6 +529,8 @@ const MESSAGES = {
     'chart.otherSkill': 'used another skill: {name}',
     'chart.more': '…{n} more, see the full report',
     'chart.wrongTitle': '{n} times it triggered when it should not have',
+    modelAccountHint: 'usable with your account now',
+    modelCatalogHint: 'in CodeBuddy\'s catalog; your account may not have it',
     modelUnknown: '{host} does not list a model called "{model}". It currently supports: {list}. Pick another, or choose the default model.',
     updateAvail: 'A newer version {latest} is available (you are on {current}).',
     updateQ: 'Update?',
@@ -697,6 +731,36 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'poll.body': '1) Claude Code　2) Codex　3) Other — reply with a single digit.',
     'poll.why': 'Why ask: a trigger rate is only measurable on a host that exposes explicit skill invocation. Codex has no skill item in its event stream, so its column is empty rather than unmeasured. Which host a reader uses therefore decides who the report serves and which layer to build next. The answers are read as a signal, never as a proportion: whoever replies has selected themselves, so no ratio can be computed from it.',
     'trial.title': 'Trigger rate',
+    'trial.answersTitle': 'Per question: what the model said and did first',
+    'trial.stopNote': 'A trial stops at the model\'s first action (by then it is known whether the skill was used), so this goes only up to the first step. It is not a complete answer, and the model is never let run its commands.',
+    'trial.runN': 'Run {n}:',
+    'trial.saidFirst': 'Before acting it said:',
+    'trial.saidNothing': 'It said nothing before acting.',
+    'trial.glance': 'At a glance',
+    'trial.glanceRecall': 'Should trigger: triggered   ',
+    'trial.glanceFpr': 'Should not: triggered       ',
+    'trial.legend': 'How to read: "Expected" is what the question was written to do; "Actual" is what the model really did; ✓ means they agree, ✗ that they do not (a miss, or a false trigger).',
+    'trial.byClause': 'Questions and results, by trigger clause',
+    'trial.clauseTitle': 'Clause: {clause}',
+    'trial.noClause': 'Unassigned',
+    'trial.hExpect': 'Expected',
+    'trial.hActual': 'Actual',
+    'trial.hMatch': 'Agree',
+    'trial.hQuery': 'Question',
+    'trial.hFirst': 'What the model did first',
+    'trial.hNote': 'Why it was written',
+    'trial.actTriggered': 'triggered',
+    'trial.actNot': 'did not trigger',
+    'trial.actPartial': 'triggered {k}/{n}',
+    'trial.actWrong': 'false trigger {k}/{n}',
+    'trial.borrows': '(borrowed words: {words})',
+    'trial.didSkill': 'invoked skill {name}',
+    'trial.otherSkill': 'used another skill: {name}',
+    'trial.noToolRun': 'no tool',
+    'trial.missTitle': 'The {n} misses: what the model did instead',
+    'trial.hFirstTool': 'First thing it did',
+    'trial.hCount': 'Count',
+    'trial.allZero': 'Nothing triggered at all: more likely the host could not read this skill (path, install or permissions) than that the skill is badly written. Check that it sits where the host can read it.',
     'trial.modelLine': 'Model: {model} ({backend}); host: {host}; {runs} runs per query, {trials} trials',
     'trial.hostUnknown': 'not recorded',
     'trial.recall': 'Recall: {p} (95% CI {lo}–{hi})',
@@ -911,6 +975,8 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'chart.otherSkill': '別のスキルを使用：{name}',
     'chart.more': '…ほか {n} 件は詳細レポートへ',
     'chart.wrongTitle': '発動すべきでないのに発動した {n} 回',
+    modelAccountHint: 'このアカウントで今使えます',
+    modelCatalogHint: 'CodeBuddy のカタログにあります（アカウントで使えるとは限りません）',
     modelUnknown: '{host} のモデル一覧に「{model}」はありません。現在対応：{list}。別のものを選ぶか、既定のモデルを選んでください。',
     updateAvail: '新しいバージョン {latest} があります（現在は {current}）。',
     updateQ: '更新しますか？',
@@ -1058,6 +1124,36 @@ Layer 0 always runs first: it is free, and it disqualifies most problems before
     'poll.body': '1) Claude Code　2) Codex　3) その他　—— 数字 1 つで答えてください。',
     'poll.why': '質問の理由：発動率は、スキルの明示的な呼び出しを公開しているホストでしか測れません。Codex のイベントストリームには skill 項目がないため、その列は「未測定」ではなく空欄です。読者がどのホストを使っているかで、このレポートが誰の役に立つか、次にどの層を作るべきかが決まります。回答はシグナルとしてのみ読み、割合としては扱いません。回答する人は自己選択であり、そこから比率は計算できないためです。',
     'trial.title': '発動率',
+    'trial.answersTitle': '質問ごとの詳細：モデルが言ったこと・最初にしたこと',
+    'trial.stopNote': '試行はモデルの最初の動作で止まります（その時点でスキルが使われたか分かるため）。ここは最初のステップまでで、完全な回答ではなく、モデルにコマンドを実行させることもありません。',
+    'trial.runN': '{n} 回目：',
+    'trial.saidFirst': '動作の前に言ったこと：',
+    'trial.saidNothing': '動作の前には何も言わず、すぐ行動しました。',
+    'trial.glance': '結果の概要',
+    'trial.glanceRecall': '発動すべき質問：発動  ',
+    'trial.glanceFpr': '発動すべきでない質問：誤発動',
+    'trial.legend': '読み方：「期待」は質問の設計意図、「実際」はモデルが実際に行ったこと。✓ は一致、✗ は不一致（未発動、または誤発動）。',
+    'trial.byClause': '質問と結果の対照（発動条件ごと）',
+    'trial.clauseTitle': '条件：{clause}',
+    'trial.noClause': '未分類',
+    'trial.hExpect': '期待',
+    'trial.hActual': '実際',
+    'trial.hMatch': '一致',
+    'trial.hQuery': '質問',
+    'trial.hFirst': 'モデルが最初にしたこと',
+    'trial.hNote': '出題の意図',
+    'trial.actTriggered': '発動',
+    'trial.actNot': '発動せず',
+    'trial.actPartial': '一部発動 {k}/{n}',
+    'trial.actWrong': '誤発動 {k}/{n}',
+    'trial.borrows': '（借用した語：{words}）',
+    'trial.didSkill': 'スキル {name} を呼び出し',
+    'trial.otherSkill': '別のスキルを使用：{name}',
+    'trial.noToolRun': 'ツールなし',
+    'trial.missTitle': '発動すべきなのに発動しなかった {n} 回：代わりに何をしたか',
+    'trial.hFirstTool': '最初にしたこと',
+    'trial.hCount': '回数',
+    'trial.allZero': '一度も発動しませんでした。スキルの出来が悪いというより、ホストがこのスキルを読めていない（パス・インストール・権限）可能性が高いです。ホストが読める場所にあるか確認してください。',
     'trial.modelLine': 'モデル：{model}（{backend}）；ホスト：{host}；1 問あたり {runs} 回、計 {trials} 回の試行',
     'trial.hostUnknown': '記録なし',
     'trial.recall': '再現率（発動すべき質問で実際に発動した割合）：{p}（95% 信頼区間 {lo}–{hi}）',

package/lib/models.js

@@ -1,5 +1,8 @@
 // Model names a host will accept, read from the host itself so the list is the user's own (it depends on the
 // account and the custom models they set up) and never goes stale inside this package. Any failure gives [].
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
 import { execFileSync, execSync } from 'node:child_process';
 import { launchSpec } from './spawn.js';
 
@@ -44,3 +47,35 @@ export function unknownModel(model, known) {
   if (!model || !known?.length || known.includes(model) || /^deepseek/i.test(model)) return null;
   return known;
 }
+
+// `codebuddy --help` lists only what the account is signed in for right now (it can shrink to two entries), while the
+// model catalog CodeBuddy last fetched stays in its local cache. Every string array called "models" inside those cache
+// files is part of that catalog; the union with the help list is what the user remembers having been offered. A name
+// from the catalog only is not promised to work for the account, so the list marks it and a failure shows the CLI's words.
+export function catalogFromTexts(texts) {
+  const names = new Set();
+  const walk = (node, depth) => {
+    if (!node || depth > 8) return;
+    if (Array.isArray(node)) { node.forEach((n) => walk(n, depth + 1)); return; }
+    if (typeof node !== 'object') return;
+    for (const [k, v] of Object.entries(node)) {
+      if (k === 'models' && Array.isArray(v)) v.filter((x) => typeof x === 'string' && /^[\w.:-]{2,80}$/.test(x)).forEach((x) => names.add(x));
+      else walk(v, depth + 1);
+    }
+  };
+  for (const text of texts) { try { walk(JSON.parse(text), 0); } catch { /* not JSON */ } }
+  return [...names];
+}
+
+export function codebuddyCatalog(dir = path.join(os.homedir(), '.codebuddy', 'local_storage')) {
+  const texts = [];
+  try {
+    for (const f of fs.readdirSync(dir)) {
+      if (!f.endsWith('.info')) continue;
+      const file = path.join(dir, f);
+      if (fs.statSync(file).size > 2 * 1024 * 1024) continue;
+      texts.push(fs.readFileSync(file, 'utf8'));
+    }
+  } catch { /* no cache: only the help list */ }
+  return catalogFromTexts(texts);
+}

package/lib/progress.js

@@ -0,0 +1,48 @@
+// The wait for the trials. On a terminal the bar is drawn at once (0 of N) and redrawn every second with the seconds
+// elapsed, so there is something on screen from the moment the trials start rather than after the first one finishes;
+// a trial can take many seconds. `total` is known when a run starts; a resumed run learns it from its first trial.
+// Without a terminal each finished trial is one line. finish() stops the redraw and ends the line.
+import { t } from './i18n.js';
+import { bar } from './chart.js';
+
+export function makeProgress(total = 0, {
+  tty = Boolean(process.stdout.isTTY),
+  write = (s) => process.stdout.write(s),
+  line = (s) => process.stdout.write(`${s}\n`),
+  now = Date.now,
+  every = setInterval,
+  stopEvery = clearInterval,
+} = {}) {
+  const started = now();
+  let finished = 0;
+  let of = total;
+  let timer = null;
+  let open = false;
+  const draw = () => {
+    open = true;
+    write(`\r\x1b[2K  ${t('progressLine')} ${bar(of ? finished / of : 0, 24)} ${finished}/${of || '?'}  ${t('waitElapsed', { s: Math.floor((now() - started) / 1000) })}`);
+  };
+  const stop = () => {
+    if (timer) stopEvery(timer);
+    timer = null;
+    if (tty && open) { write('\n'); open = false; }
+  };
+  if (tty && total) {
+    draw();
+    timer = every(draw, 1000);
+  }
+  const onTrial = ({ row, done, total: n }) => {
+    finished = done;
+    of = n;
+    if (tty) {
+      if (!timer) timer = every(draw, 1000);
+      draw();
+      if (done >= n) stop();
+      return;
+    }
+    const mark = { triggered: '✓', not_triggered: '·', other_skill: '↷' }[row.verdict] || '!';
+    line(`  [${done}/${n}] ${mark} ${row.queryId}#${row.run} → ${row.verdict}`);
+  };
+  onTrial.finish = stop;
+  return onTrial;
+}

package/lib/report.js

@@ -39,7 +39,98 @@ export function staticMarkdown(result) {
   return L.join('\n');
 }
 
-export function trialSection(trial) {
+const bar = (v, w = 20) => {
+  const n = Math.round(Math.min(1, Math.max(0, v ?? 0)) * w);
+  return '█'.repeat(n) + '░'.repeat(w - n);
+};
+const cell = (x) => String(x ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ').trim();
+
+// What the model did in one run, in words.
+function didWhat(r) {
+  if (r.verdict === 'triggered') return t('trial.didSkill', { name: r.invokedSkill || '?' });
+  if (r.verdict === 'other_skill') return t('trial.otherSkill', { name: r.invokedSkill || '?' });
+  return r.firstTool || t('trial.noToolRun');
+}
+
+// One row per question, grouped under the trigger clause it was written for: what was expected, what happened, whether
+// they agree, what the model did instead, and why the question was written (for a near miss, the words it borrows).
+function trialDetail(trial, records, answers) {
+  const byQuery = new Map();
+  for (const r of records) {
+    if (!byQuery.has(r.queryId)) byQuery.set(r.queryId, []);
+    byQuery.get(r.queryId).push(r);
+  }
+  const L = [];
+  const pos = trial.questions.filter((q) => q.shouldTrigger);
+  const neg = trial.questions.filter((q) => !q.shouldTrigger);
+  const hitP = pos.reduce((a, q) => a + q.hits, 0);
+  const nP = pos.reduce((a, q) => a + q.n, 0);
+  const hitN = neg.reduce((a, q) => a + q.hits, 0);
+  const nN = neg.reduce((a, q) => a + q.n, 0);
+  L.push('', `### ${t('trial.glance')}`, '', '```');
+  L.push(`${t('trial.glanceRecall')}  ${bar(trial.recall)} ${pct(trial.recall)}  (${hitP}/${nP})`);
+  L.push(`${t('trial.glanceFpr')}  ${bar(trial.fpr)} ${pct(trial.fpr)}  (${hitN}/${nN})`);
+  L.push('```', '', t('trial.legend'));
+
+  // group by clause, in the order the clauses were drafted
+  const clauses = [];
+  for (const q of trial.queries || []) if (!clauses.includes(q.clause || '')) clauses.push(q.clause || '');
+  L.push('', `### ${t('trial.byClause')}`);
+  for (const clause of clauses) {
+    const qs = (trial.queries || []).filter((q) => (q.clause || '') === clause);
+    L.push('', `#### ${clause ? t('trial.clauseTitle', { clause: cell(clause) }) : t('trial.noClause')}`, '');
+    L.push(`| ${t('trial.hExpect')} | ${t('trial.hActual')} | ${t('trial.hMatch')} | ${t('trial.hQuery')} | ${t('trial.hFirst')} | ${t('trial.hNote')} |`);
+    L.push('|---|---|---|---|---|---|');
+    for (const q of [...qs.filter((x) => x.shouldTrigger), ...qs.filter((x) => !x.shouldTrigger)]) {
+      const runs = byQuery.get(q.id) || [];
+      const hits = runs.filter((r) => r.verdict === 'triggered').length;
+      const n = runs.length;
+      let actual;
+      let ok;
+      if (!n) { actual = '—'; ok = '—'; } else if (q.shouldTrigger) {
+        actual = hits === n ? t('trial.actTriggered') : hits === 0 ? t('trial.actNot') : t('trial.actPartial', { k: hits, n });
+        ok = hits * 2 > n ? '✓' : '✗';
+      } else {
+        actual = hits === 0 ? t('trial.actNot') : t('trial.actWrong', { k: hits, n });
+        ok = hits === 0 ? '✓' : '✗';
+      }
+      const first = [...new Set(runs.map(didWhat))].join('; ') || '—';
+      const note = [cell(q.note), q.shares ? t('trial.borrows', { words: cell(q.shares) }) : ''].filter(Boolean).join(' ');
+      L.push(`| ${q.shouldTrigger ? t('trial.should') : t('trial.shouldNot')} | ${actual} | ${ok} | ${cell(q.query)} | ${cell(first)} | ${note} |`);
+    }
+  }
+
+  const missed = records.filter((r) => r.shouldTrigger && r.verdict !== 'triggered');
+  if (missed.length) {
+    const kinds = new Map();
+    for (const r of missed) kinds.set(didWhat(r), (kinds.get(didWhat(r)) || 0) + 1);
+    L.push('', `### ${t('trial.missTitle', { n: missed.length })}`, '');
+    L.push(`| ${t('trial.hFirstTool')} | ${t('trial.hCount')} | |`, '|---|---|---|');
+    for (const [k, c] of [...kinds].sort((a, b) => b[1] - a[1])) L.push(`| ${cell(k)} | ${c} | ${bar(c / missed.length, 16)} |`);
+  }
+  if (trial.recall === 0 && trial.fpr === 0) L.push('', `> ${t('trial.allZero')}`);
+
+  // What the model said and did first, question by question. A trial stops at the first action, so this is not a
+  // complete answer, and the report says so.
+  if (answers?.size) {
+    L.push('', `### ${t('trial.answersTitle')}`, '', `> ${t('trial.stopNote')}`);
+    for (const q of trial.queries || []) {
+      const runs = (byQuery.get(q.id) || []);
+      if (!runs.length) continue;
+      L.push('', `**${q.id}** · ${q.shouldTrigger ? t('trial.should') : t('trial.shouldNot')} · ${cell(q.query)}`);
+      for (const r of runs) {
+        const a = answers.get(`${r.queryId}#${r.run}`);
+        const tag = runs.length > 1 ? `${t('trial.runN', { n: r.run + 1 })} ` : '';
+        L.push('', `- ${tag}${didWhat(r)}${a?.firstInput ? `: \`${cell(a.firstInput).replace(/`/g, "'")}\`` : ''}`);
+        if (a?.said?.trim()) L.push(`  - ${t('trial.saidFirst')}`, '', ...a.said.trim().split(/\r?\n/).map((l) => `    > ${l}`));
+        else L.push(`  - ${t('trial.saidNothing')}`);
+      }
+    }
+  }
+  return L;
+}
+
+export function trialSection(trial, records = null, answers = null) {
   if (!trial) return '';
   const L = ['', `## ${t('trial.title')}`, ''];
   L.push(`- ${t('trial.modelLine', {
@@ -54,6 +145,10 @@ export function trialSection(trial) {
   if (trial.noToolRate != null) {
     L.push(`- ${t('trial.noTool', { p: pct(trial.noToolRate) })}${trial.noToolRate >= 0.5 ? t('trial.noToolWarn') : ''}`);
   }
+  if (records?.length && trial.queries?.length) {
+    L.push(...trialDetail(trial, records, answers));
+    return L.join('\n');
+  }
   L.push('');
   L.push(`| ${t('trial.stateHeader')} | ${t('trial.queryHeader')} | ${t('trial.hitsHeader')} |`);
   L.push('|---|---|---|');

package/lib/runs.js

@@ -118,6 +118,11 @@ function handle(dir, meta) {
       meta.trials += 1;
       if (meta.trials % 25 === 0) flushMeta();
     },
+    // What the model said and did first, per trial. Local only: submissions copy a fixed list of files, and this
+    // is not one of them.
+    appendAnswer(row) {
+      fs.appendFileSync(path.join(dir, 'answers.jsonl'), `${JSON.stringify(row)}\n`);
+    },
     close(summary, { interrupted = false } = {}) {
       meta.finishedAt = new Date().toISOString();
       meta.interrupted = interrupted;
@@ -165,3 +170,15 @@ export function readRun(dir) {
   try { summary = JSON.parse(fs.readFileSync(path.join(dir, 'summary.json'), 'utf8')); } catch { /* unfinished */ }
   return { meta, records: raw, summary };
 }
+
+// Answers kept beside a run, keyed "queryId#run". A run made before they were kept has none: an empty map.
+export function readAnswers(dir) {
+  const out = new Map();
+  try {
+    for (const line of fs.readFileSync(path.join(dir, 'answers.jsonl'), 'utf8').split(/\r?\n/).filter(Boolean)) {
+      const row = JSON.parse(line);
+      out.set(`${row.queryId}#${row.run}`, row);
+    }
+  } catch { /* none kept */ }
+  return out;
+}

package/lib/trial.js

@@ -154,6 +154,21 @@ export function pingHost(host, { bin, model = null, timeout = 90, spawn = spawnS
   }
 }
 
+// What a trial shows of the model, kept apart from the verdict: the words it said before its first action and what that
+// action was. A trial ends at the first action (the verdict is known then), so this is "up to the first step", never a
+// complete answer. It goes to a local file, not into the run's raw records, which are what gets submitted.
+export const SAID_LIMIT = 2000;
+export function inputSummary(name, input) {
+  if (!input || typeof input !== 'object' || !Object.keys(input).length) return '';
+  const pick = input.command ?? input.file_path ?? input.path ?? input.pattern ?? input.skill ?? input.name ?? input.query;
+  const text = typeof pick === 'string' ? pick : JSON.stringify(input);
+  return `${text}`.slice(0, 300);
+}
+export function addSaid(rec, text) {
+  if (typeof text !== 'string' || !text) return;
+  rec.said = `${rec.said || ''}${text}`.slice(0, SAID_LIMIT);
+}
+
 // Variables a host (the Claude desktop app, the Agent SDK, an outer Claude Code session) sets for its own
 // child processes. A trial inherits them otherwise, and they change what the model under test sees: under
 // the desktop app the CLI adds Artifact, ArtifactComments and ArtifactData to the tool list, and the model
@@ -299,6 +314,7 @@ export function runTrial({ bin, skillDir, skillName, query, model, timeout = 180
           rec.init = { model: ev.model, apiKeySource: ev.apiKeySource, claudeCodeVersion: ev.claude_code_version,
             skills: ev.skills || [], tools: ev.tools || [] };
         } else if (type === 'stream_event') {
+          if (ev.event?.delta?.type === 'text_delta') addSaid(rec, ev.event.delta.text);
           const block = ev.event?.content_block;
           if (block?.type === 'tool_use') {
             rec.events.toolUse += 1;
@@ -311,9 +327,11 @@ export function runTrial({ bin, skillDir, skillName, query, model, timeout = 180
           }
         } else if (type === 'assistant') {
           for (const c of ev.message?.content || []) {
+            if (c.type === 'text') addSaid(rec, c.text);
             if (c.type !== 'tool_use') continue;
             rec.tools.push(c.name);
             rec.firstTool = rec.firstTool || c.name;
+            rec.firstInput = rec.firstInput || inputSummary(c.name, c.input);
             if (c.name === 'Skill') {
               const inv = String(c.input?.skill || '');
               if (inv === skillName || inv.endsWith(`:${skillName}`)) finish('triggered', { invokedSkill: inv });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skillgauntlet",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "Check an agent skill before you install it: context cost, permission surface, code risk, instruction hijacking, description quality, trigger rate and gain.",
   "type": "module",
   "bin": {