skillgauntlet 1.1.0 → 1.2.0

package/lib/browse.js

@@ -0,0 +1,58 @@
+// Picking a directory with the arrow keys: the list shows the folders under the current one, a way up, and a way
+// to type a path instead. `select` is injected so the walk can be tested without a terminal.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { t } from './i18n.js';
+
+const DOT_KEPT = new Set(['.claude', '.agents', '.codex', '.codebuddy']);
+
+function subdirs(dir) {
+  let entries = [];
+  try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
+  return entries
+    .filter((e) => e.isDirectory() && (!e.name.startsWith('.') || DOT_KEPT.has(e.name)) && e.name !== 'node_modules')
+    .map((e) => e.name)
+    .sort((a, b) => a.localeCompare(b));
+}
+
+const holdsSkill = (dir) => fs.existsSync(path.join(dir, 'SKILL.md'));
+
+function drives() {
+  const found = [];
+  for (let c = 65; c <= 90; c += 1) {
+    const root = `${String.fromCharCode(c)}:\\`;
+    if (fs.existsSync(root)) found.push(root);
+  }
+  return found;
+}
+
+// A long path would wrap and break the in-place redraw, so only its tail is shown.
+const tail = (p, n = 36) => (p.length > n ? `…${p.slice(-(n - 1))}` : p);
+
+// Returns the chosen directory, or null when the user asks to type a path instead.
+export async function browseDir(select, start = process.cwd()) {
+  let dir = path.resolve(start);
+  for (;;) {
+    const parent = path.dirname(dir);
+    const atRoot = parent === dir;
+    const options = [{ label: t('browseUse'), hint: holdsSkill(dir) ? t('browseHasSkill') : '', value: { use: dir } }];
+    if (!atRoot) options.push({ label: t('browseUp'), hint: tail(parent), value: { go: parent } });
+    else if (process.platform === 'win32') options.push({ label: t('browseDrives'), value: { drives: true } });
+    if (dir !== os.homedir()) options.push({ label: t('browseHome'), hint: tail(os.homedir()), value: { go: os.homedir() } });
+    for (const name of subdirs(dir)) {
+      const full = path.join(dir, name);
+      options.push({ label: `${name}${path.sep}`, hint: holdsSkill(full) ? t('browseHasSkill') : '', value: { go: full } });
+    }
+    options.push({ label: t('browseType'), value: { type: true } });
+
+    const pick = await select(t('browseTitle', { dir: tail(dir) }), options, 0);
+    if (pick.use) return pick.use;
+    if (pick.type) return null;
+    if (pick.go) dir = pick.go;
+    if (pick.drives) {
+      const root = await select(t('browseDrives'), drives().map((d) => ({ label: d, value: d })), 0);
+      dir = root;
+    }
+  }
+}

package/lib/chart.js

@@ -0,0 +1,157 @@
+// A result drawn for the terminal: bars for rates, one mark per question, a verdict in words. The Markdown report
+// stays the full record; this is what a person reads first, so it carries the few numbers that decide the reading.
+import { stdout } from 'node:process';
+import { gradeTrigger } from './submission.js';
+import { t, lang } from './i18n.js';
+
+const color = () => stdout.isTTY && !process.env.NO_COLOR;
+const paint = (code, s) => (color() ? `\x1b[${code}m${s}\x1b[0m` : s);
+const green = (s) => paint(32, s);
+const yellow = (s) => paint(33, s);
+const red = (s) => paint(31, s);
+const dim = (s) => paint(2, s);
+const bold = (s) => paint(1, s);
+
+const pct = (x) => (x == null ? '—' : `${Math.round(x * 100)}%`);
+
+// width cells; filled share = value (0..1)
+export function bar(value, width = 24, tone = (s) => s) {
+  const v = Math.min(1, Math.max(0, value ?? 0));
+  const n = Math.round(v * width);
+  return `${tone('█'.repeat(n))}${dim('░'.repeat(width - n))}`;
+}
+
+const good = (v) => (v >= 0.8 ? green : v >= 0.5 ? yellow : red);   // higher is better
+const bad = (v) => (v <= 0.1 ? green : v <= 0.25 ? yellow : red);    // lower is better
+
+const GRADE_TONE = { well: green, fair: yellow, poorly: red, insufficient: dim };
+
+function staticLines(result) {
+  const L = [bold(t('chart.static'))];
+  for (const g of result.groups) {
+    const mark = g.verdict === 'pass' ? green('✓') : g.verdict === 'warn' ? yellow('!') : red('✗');
+    L.push(`  ${mark} ${g.id}  ${t(g.titleKey)}`);
+    // A group that is not clean says which checks and why: a letter and a mark alone cannot be acted on.
+    for (const c of g.checks.filter((x) => x.status === 'warn' || x.status === 'fail')) {
+      L.push(`      ${c.status === 'fail' ? red('✗') : yellow('!')} ${c.id} ${t(c.titleKey)}${dim(` — ${cut(t(c.detailKey, c.detailVars), 70)}`)}`);
+      for (const e of (c.evidence || []).slice(0, 2)) L.push(`          ${dim(`${cut(`${e.file || ''}:${e.line}`, 56)}  ${cut(e.match, 40)}`)}`);
+    }
+  }
+  L.push(`  ${dim(t('chart.staticLegend'))}`);
+  const st = result.stats;
+  if (st) L.push(`  ${dim(t('chart.stats', { tokens: st.skillMdTokens, desc: st.descriptionChars, files: st.files }))}`);
+  return L;
+}
+
+function triggerLines(trial, records) {
+  const pos = trial.questions.filter((q) => q.shouldTrigger);
+  const neg = trial.questions.filter((q) => !q.shouldTrigger);
+  const nPos = pos.reduce((a, q) => a + q.n, 0);
+  const nNeg = neg.reduce((a, q) => a + q.n, 0);
+  const grade = gradeTrigger({ recall: trial.recall, fpr: trial.fpr, nPos, nNeg });
+  const ci = (c) => (c?.[0] == null ? '' : dim(`  ${t('chart.ci', { lo: pct(c[0]), hi: pct(c[1]) })}`));
+  const L = [bold(t('chart.trigger')), ''];
+  L.push(`  ${dim(t('chart.conditions', { host: trial.host || '?', model: trial.modelResolved || trial.model || '?', trials: trial.trials, failed: trial.failedTrials || 0 }))}`, '');
+  L.push(`  ${t('chart.verdict')}  ${(GRADE_TONE[grade] || ((s) => s))(bold(t(`chart.grade.${grade}`)))}`);
+  L.push('');
+  L.push(`  ${padCells(t('chart.recall'), pad(t('chart.recall'), t('chart.fpr')))}${bar(trial.recall, 24, good(trial.recall ?? 0))} ${pct(trial.recall)}${ci(trial.recallCi)}`);
+  L.push(`  ${padCells(t('chart.fpr'), pad(t('chart.recall'), t('chart.fpr')))}${bar(trial.fpr, 24, bad(trial.fpr ?? 0))} ${pct(trial.fpr)}${ci(trial.fprCi)}`);
+  L.push('');
+  L.push(`  ${dim(t('chart.legend'))}`);
+  const strip = (qs) => qs.map((q) => (q.hits * 2 > q.n ? green('●') : q.hits > 0 ? yellow('◐') : dim('○'))).join('');
+  L.push(`  ${padCells(t('chart.should'), pad(t('chart.should'), t('chart.shouldNot')))}${strip(pos)}  ${dim(`${pos.filter((q) => q.hits * 2 > q.n).length}/${pos.length}`)}`);
+  L.push(`  ${padCells(t('chart.shouldNot'), pad(t('chart.should'), t('chart.shouldNot')))}${strip(neg)}  ${dim(`${neg.filter((q) => q.hits > 0).length}/${neg.length}`)}`);
+  if (trial.incomplete) L.push(`  ${yellow(t('chart.incomplete', { n: trial.incomplete }))}`);
+  if (trial.recall === 0 && trial.fpr === 0) L.push('', `  ${yellow(t('chart.allZero'))}`);
+  if (records?.length) L.push(...detailLines(records));
+  return L;
+}
+
+// Width of the wider of two labels in terminal cells: CJK characters take two.
+function cells(s) {
+  let n = 0;
+  for (const ch of String(s)) n += /[ᄀ-ᅟ⺀-꓏가-힣豈-﫿︰-﹯＀-｠￠-￦]/.test(ch) ? 2 : 1;
+  return n;
+}
+function pad(a, b) { return Math.max(cells(a), cells(b)) + 2; }
+// String.padEnd counts code units, not cells, so columns are padded by hand.
+const padCells = (str, width) => str + ' '.repeat(Math.max(0, width - cells(str)));
+
+// What the model did instead when the skill was not used, then the questions behind the numbers.
+function detailLines(records) {
+  const L = [];
+  const missed = records.filter((r) => r.shouldTrigger && r.verdict !== 'triggered');
+  if (missed.length) {
+    const kinds = new Map();
+    for (const r of missed) {
+      const k = r.verdict === 'other_skill' ? t('chart.otherSkill', { name: r.invokedSkill || '?' }) : (r.firstTool || t('chart.noTool'));
+      kinds.set(k, (kinds.get(k) || 0) + 1);
+    }
+    const rows = [...kinds].sort((a, b) => b[1] - a[1]);
+    const w = Math.max(...rows.map(([k]) => cells(k))) + 2;
+    L.push('', `  ${bold(t('chart.missedTitle', { n: missed.length }))}`);
+    for (const [k, n] of rows) L.push(`    ${padCells(k, w)}${bar(n / missed.length, 16, yellow)} ${n}`);
+    const shown = missed.slice(0, 8);
+    L.push('', `  ${dim(t('chart.missedList'))}`);
+    for (const r of shown) L.push(`    ${dim('○')} ${cut(r.query, 46)}  ${dim(`→ ${r.firstTool || t('chart.noTool')}`)}`);
+    if (missed.length > shown.length) L.push(`    ${dim(t('chart.more', { n: missed.length - shown.length }))}`);
+  }
+  const wrong = records.filter((r) => !r.shouldTrigger && r.verdict === 'triggered');
+  if (wrong.length) {
+    L.push('', `  ${bold(t('chart.wrongTitle', { n: wrong.length }))}`);
+    for (const r of wrong.slice(0, 8)) L.push(`    ${red('●')} ${cut(r.query, 46)}`);
+  }
+  return L;
+}
+
+// Cut to n terminal cells, with an ellipsis when something was removed.
+function cut(str, n) {
+  let out = '';
+  let w = 0;
+  for (const ch of String(str).replace(/\s+/g, ' ')) {
+    const c = cells(ch);
+    if (w + c > n - 1) return `${out}…`;
+    out += ch;
+    w += c;
+  }
+  return out;
+}
+
+function probeLines(probe) {
+  const same = (b) => (b ? green(t('probeSame')) : yellow(t('probeDiffer')));
+  const L = [bold(t('chart.probe')), `  ${dim(t('chart.probeAsk'))}`, ''];
+  const w = Math.max(cells(t('probeWith')), cells(t('probeWithout'))) + 2;
+  for (const [label, side] of [[t('probeWith'), probe.withSkill], [t('probeWithout'), probe.without]]) {
+    if (!side) continue;
+    const act = side.firstAction ? cut(`${side.firstAction.kind}: ${side.firstAction.detail}`, 80) : t('chart.noAction');
+    const files = side.createdFiles?.length ? cut(side.createdFiles.join(', '), 80) : t('chart.noFiles');
+    L.push(`  ${bold(padCells(label, w))}${t('chart.firstAction')}  ${act}`);
+    L.push(`  ${padCells('', w)}${t('chart.filesMade')}  ${files}`);
+  }
+  L.push('', `  ${t('probeActionKind')}  ${same(probe.actionKindIdentical)}`, `  ${t('probeArtifacts')}  ${same(probe.artifactsIdentical)}`);
+  const reading = probe.actionKindIdentical && probe.artifactsIdentical ? 'chart.probeNoEffect'
+    : probe.artifactsIdentical === false ? 'chart.probeEffect' : 'chart.probeMixed';
+  L.push('', `  ${t(reading)}`, `  ${dim(t('chart.probeNote'))}`);
+  return L;
+}
+
+function gainLines(gain) {
+  const L = [bold(t('chart.gain')), `  ${dim(t('chart.gainLegend'))}`];
+  for (const c of gain.contracts) {
+    L.push(`  ${String(c.descI18n?.[lang()] || c.desc).slice(0, 60)}`);
+    L.push(`    ${padCells(t('gain.withHeader'), 10)}${bar(c.with, 20, green)} ${pct(c.with)}`);
+    L.push(`    ${padCells(t('gain.withoutHeader'), 10)}${bar(c.without, 20, (s) => s)} ${pct(c.without)}`);
+  }
+  return L;
+}
+
+// One block per tested skill. Any part may be absent (static only, probe only, no gain layer).
+export function renderResult({ name, result, trial, records, gain, probe, reportPath }) {
+  const L = ['', bold(`════ ${name} ════`), ''];
+  if (result) L.push(...staticLines(result), '');
+  if (trial) L.push(...triggerLines(trial, records), '');
+  if (probe) L.push(...probeLines(probe), '');
+  if (gain?.contracts?.length) L.push(...gainLines(gain), '');
+  if (reportPath) L.push(dim(t('chart.full', { path: reportPath })));
+  return L.join('\n');
+}

package/lib/depth.js

@@ -15,7 +15,7 @@ export const DEPTHS = {
     id: 'quick',
     sets: 1,
     perClause: 2,
-    negatives: 4,
+    negatives: 10, // the grade needs at least 10 on each side, so fewer would make every quick result "too few"
     runs: 1,
     maxExplore: 0,
     layers: ['static', 'trigger'],
@@ -53,3 +53,6 @@ export function estimate(depth, clauses = 3) {
     minutes: Math.max(1, Math.round((trials * 4) / 2 / 60)),
   };
 }
+
+// Minutes for a number of trials, on the same assumption as estimate(): 2 in parallel at ~4 s each.
+export const minutesFor = (trials) => Math.max(1, Math.round((trials * 4) / 2 / 60));

package/lib/discover.js

@@ -5,6 +5,9 @@ import os from 'node:os';
 import path from 'node:path';
 
 const SKIP = new Set(['.git', 'node_modules', '.cache', 'dist', 'build', 'vendor', '__pycache__']);
+// Dot directories are skipped, except the ones agent hosts install skills into: a repository's own skill
+// often lives only under .claude/skills.
+const SKILL_DOT_DIRS = new Set(['.claude', '.agents', '.codex', '.codebuddy']);
 
 export function findSkills(root, { maxDepth = 6 } = {}) {
   const found = [];
@@ -17,12 +20,25 @@ export function findSkills(root, { maxDepth = 6 } = {}) {
       return; // a skill dir does not contain nested skills
     }
     for (const e of entries) {
-      if (!e.isDirectory() || SKIP.has(e.name) || e.name.startsWith('.')) continue;
+      if (!e.isDirectory() || SKIP.has(e.name) || (e.name.startsWith('.') && !SKILL_DOT_DIRS.has(e.name))) continue;
       walk(path.join(dir, e.name), depth + 1);
     }
   };
   walk(path.resolve(root), 0);
-  return found.sort();
+  return dropCopies(found.sort());
+}
+
+// Repositories often ship a second copy of a skill (a packaged template next to the installed one). A copy with
+// the same name and byte-identical SKILL.md adds nothing to a test, so only the first path is kept.
+function dropCopies(dirs) {
+  const seen = new Set();
+  return dirs.filter((d) => {
+    let key = d;
+    try { key = [path.basename(d), fs.readFileSync(path.join(d, 'SKILL.md'), 'utf8')].join(' | '); } catch { /* keep it */ }
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
 }
 
 export function readFrontmatter(skillDir) {
@@ -81,3 +97,12 @@ export function resolveSpec(spec) {
   } catch { /* keep empty */ }
   return { kind: 'git', dir, source: { repo: `https://github.com/${owner}/${repo}`, commit, ref: ref || null } };
 }
+
+// The name a host knows a skill by is the `name` in its SKILL.md frontmatter; the folder name is only a fallback.
+// A skill at the root of a repository sits in a folder named after the repository (or its cache slug), so using
+// the folder name made every trial of such a skill look like "another skill was used".
+export function skillNameOf(skillDir) {
+  const fm = readFrontmatter(skillDir).fm;
+  const name = String(fm.name || '').trim();
+  return /^[\w.-]+$/.test(name) ? name : path.basename(skillDir);
+}

package/lib/findhost.js

@@ -0,0 +1,83 @@
+// A host CLI that is not on PATH is often still on the machine: in npm's global folder, in Downloads under the
+// name the release page gave it, or in a tools folder. Look there before telling the user to install anything,
+// and where the host is an npm package offer to install it.
+import { execSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+// The command name each host answers to, and (when there is one) the npm package that provides it.
+export const HOST_CLI = {
+  'claude-code': { bin: 'claude', pkg: '@anthropic-ai/claude-code' },
+  codebuddy: { bin: 'codebuddy', pkg: '@tencent-ai/codebuddy-code' },
+  codewhale: { bin: 'codewhale', pkg: 'codewhale' },
+};
+
+const isWin = process.platform === 'win32';
+const RUNNABLE = isWin ? /\.(exe|cmd|bat)$/i : /^[^.]*$/;
+
+function npmGlobalDir() {
+  try {
+    const prefix = execSync('npm prefix -g', { encoding: 'utf8', timeout: 15000, windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
+    return isWin ? prefix : path.join(prefix, 'bin');
+  } catch { return null; }
+}
+
+function listDir(dir) {
+  try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
+}
+
+// Files named like the host (codewhale.exe, codewhale-windows-x64.exe, codewhale.cmd) directly in dir.
+function inDir(dir, bin) {
+  const out = [];
+  for (const e of listDir(dir)) {
+    if (!e.isFile()) continue;
+    const base = e.name.toLowerCase();
+    const exact = base === bin || base.startsWith(`${bin}.`);
+    // a Windows release asset keeps its platform suffix (codewhale-windows-x64.exe); only a real exe counts for those.
+    // Elsewhere an executable has no extension and only the exact name is trusted.
+    const variant = isWin && (base.startsWith(`${bin}-`) || base.startsWith(`${bin}_`)) && base.endsWith('.exe');
+    if ((exact || variant) && RUNNABLE.test(e.name)) out.push(path.join(dir, e.name));
+  }
+  return out;
+}
+
+// Directories worth looking in, nothing recursive beyond one level of subfolders for the download-like ones.
+export function searchDirs() {
+  const home = os.homedir();
+  const dirs = [npmGlobalDir(), ...(process.env.PATH || '').split(path.delimiter)].filter(Boolean);
+  for (const sub of ['.local/bin', '.cargo/bin', 'bin', 'tools', 'Downloads', 'Desktop', 'Documents']) dirs.push(path.join(home, sub));
+  if (process.env.LOCALAPPDATA) dirs.push(path.join(process.env.LOCALAPPDATA, 'Programs'));
+  return dirs;
+}
+
+// Absolute paths of candidates for a host, best first (a real .exe before a .cmd shim). `dirs` is injectable.
+export function searchHost(host, dirs = searchDirs()) {
+  const spec = HOST_CLI[host];
+  if (!spec) return [];
+  const seen = new Set();
+  const found = [];
+  const add = (f) => { if (!seen.has(f.toLowerCase())) { seen.add(f.toLowerCase()); found.push(f); } };
+  for (const dir of dirs) {
+    inDir(dir, spec.bin).forEach(add);
+    // one level down: Downloads\codewhale\codewhale.exe, Programs\codewhale\...
+    for (const e of listDir(dir).slice(0, 400)) {
+      if (e.isDirectory() && e.name.toLowerCase().includes(spec.bin)) inDir(path.join(dir, e.name), spec.bin).forEach(add);
+    }
+  }
+  const exactName = (f) => { const b = path.basename(f).toLowerCase(); return b === spec.bin || b.startsWith(`${spec.bin}.`); };
+  const rank = (f) => (exactName(f) ? 0 : 10) + (/\.exe$/i.test(f) ? 0 : /\.(cmd|bat)$/i.test(f) ? 1 : 2);
+  return found.sort((a, b) => rank(a) - rank(b));
+}
+
+// Codewhale's package has an install script, and npm only runs install scripts it has been told to allow. The
+// allowance is given for this one command (--allow-scripts) rather than written into the user's npm config.
+const INSTALL_FLAGS = { codewhale: ' --allow-scripts=codewhale' };
+export const installCommand = (host) => (HOST_CLI[host]?.pkg ? `npm install -g ${HOST_CLI[host].pkg}${INSTALL_FLAGS[host] || ''}` : null);
+
+// Runs the npm install with its output on the user's terminal. Throws with a short reason on failure.
+export function installHost(host, run = (cmd) => execSync(cmd, { stdio: 'inherit' })) {
+  const cmd = installCommand(host);
+  if (!cmd) throw new Error('no installer for this host');
+  try { run(cmd); } catch (err) { throw new Error(String(err.message || err).split(/\r?\n/)[0]); }
+}

package/lib/gain.js

@@ -21,6 +21,8 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { launchSpec } from './spawn.js';
+import { narrowEnv } from './trial.js';
 
 const HERE = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
 const CONTRACT_DIR = path.join(HERE, 'contracts');
@@ -36,8 +38,8 @@ function copyDir(src, dst) {
   }
 }
 
-export function loadContracts(skillName) {
-  const file = path.join(CONTRACT_DIR, `${skillName}.json`);
+export function loadContracts(skillName, dir = CONTRACT_DIR) {
+  const file = path.join(dir, `${skillName}.json`);
   if (!fs.existsSync(file)) return null;
   return JSON.parse(fs.readFileSync(file, 'utf8'));
 }
@@ -111,8 +113,10 @@ function runTask({ bin, model, task, skillDir, skillName, timeout }) {
   if (model) args.push('--model', model);
 
   return new Promise((resolve) => {
-    const child = spawn(bin, args, {
-      cwd: tmp, shell: /\.(cmd|bat)$/i.test(bin), env: process.env, windowsHide: true,
+    // Not through a shell: cmd.exe would re-split the prompt on spaces (see spawn.js).
+    const launch = launchSpec(bin, args, 'claude');
+    const child = spawn(launch.cmd, launch.args, {
+      cwd: tmp, shell: launch.shell, env: narrowEnv(), windowsHide: true, stdio: 'ignore',
     });
     const timer = setTimeout(() => {
       try {
@@ -128,8 +132,8 @@ function runTask({ bin, model, task, skillDir, skillName, timeout }) {
   });
 }
 
-export async function runGain({ bin, skillDir, skillName, model, timeout = 600, onProgress }) {
-  const spec = loadContracts(skillName);
+export async function runGain({ bin, skillDir, skillName, model, timeout = 600, onProgress, contractDir }) {
+  const spec = loadContracts(skillName, contractDir);
   const hasContracts = spec?.contracts?.length || spec?.tasks?.some((t) => t.contracts?.length);
   if (!spec || !hasContracts || !spec.tasks?.length) {
     return { measurable: false, reasonKey: 'gain.noContracts' };