skillgauntlet 1.0.0 → 1.1.0

package/lib/execute.js

@@ -0,0 +1,153 @@
+// Running trials against an open run archive, and picking a run back up.
+//
+// Ported from the `run` / `--resume` path of step0/trigger_test.py (v0.4.1-v0.4.2). Two rules come
+// with it. A trial that errored or timed out is written to raw.jsonl but never counted, and is
+// retried on resume. And a run stops itself after ERROR_STREAK_STOP failed trials in a row: that is
+// a usage limit, a dead token or a network outage, and carrying on would only burn the queue.
+import fs from 'node:fs';
+import path from 'node:path';
+import { fingerprint } from './queries.js';
+import { readRun, reopenRun, validRecords, querySetHash } from './runs.js';
+import { FAILED, runTrial as realRunTrial, summarize, wilson } from './trial.js';
+
+export const ERROR_STREAK_STOP = 8;
+
+// trials: [{ q, i }]. doTrial({ q, i }) resolves to a trial record, optionally carrying `init`
+// (the CLI's init event), which is lifted into the run's meta instead of being stored on every row.
+export async function executeTrials({ run, trials, doTrial, workers = 2, streakStop = ERROR_STREAK_STOP, onTrial = () => {} }) {
+  const state = { next: 0, done: 0, streak: 0, stopped: null };
+  const worker = async () => {
+    for (;;) {
+      if (state.stopped) return;
+      const idx = state.next++;
+      if (idx >= trials.length) return;
+      const { q, i } = trials[idx];
+      const rec = await doTrial({ q, i });
+      const { init, ...stored } = rec;
+      const row = { ...stored, queryId: q.id, run: i, query: q.query, shouldTrigger: q.shouldTrigger };
+      if (init && !run.meta.tools) {
+        run.patch({
+          modelResolved: init.model,
+          apiKeySource: init.apiKeySource,
+          claudeCodeVersion: init.claudeCodeVersion,
+          loadedSkills: init.skills,
+          tools: [...(init.tools || [])].sort(),
+        });
+      }
+      run.append(row);
+      state.done += 1;
+      state.streak = FAILED.has(row.verdict) ? state.streak + 1 : 0;
+      if (state.streak >= streakStop && !state.stopped) {
+        state.stopped = (String(row.error || row.verdict).trim().split(/\r?\n/).pop() || '?').slice(0, 200);
+      }
+      onTrial({ row, done: state.done, total: trials.length });
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(workers, trials.length) }, worker));
+  return { done: state.done, stopped: state.stopped };
+}
+
+// Close a finished run, or park a stopped one. A stopped run writes no summary: it is not complete,
+// and a number computed from half the queue would read as a result.
+export function finishRun(run, { model, backend, host, runs, expected }, stopped) {
+  if (stopped) {
+    run.patch({ stoppedAt: new Date().toISOString(), stoppedReason: stopped });
+    run.close(null, { interrupted: true });
+    return null;
+  }
+  const all = readRun(run.dir).records;
+  const records = validRecords(all);
+  const s = summarize(records);
+  const posQ = s.questions.filter((q) => q.shouldTrigger);
+  const negQ = s.questions.filter((q) => !q.shouldTrigger);
+  const result = {
+    ...s,
+    failedTrials: all.length - records.length,
+    // trials still owed (errors and timeouts with no valid retry): the numbers cover fewer trials than planned
+    incomplete: expected ? Math.max(0, expected - records.length) : 0,
+    model,
+    backend,
+    host,
+    runs,
+    recallCi: wilson(posQ.filter((q) => q.hits * 2 > q.n).length, posQ.length),
+    fprCi: wilson(negQ.filter((q) => q.hits > 0).length, negQ.length),
+  };
+  if (result.incomplete) run.patch({ incompleteTrials: result.incomplete });
+  run.close(result);
+  return result;
+}
+
+export function pendingTrials(queries, runs, records) {
+  const have = new Set(validRecords(records).map((r) => `${r.queryId}#${r.run}`));
+  const out = [];
+  for (const q of queries) for (let i = 0; i < runs; i += 1) if (!have.has(`${q.id}#${i}`)) out.push({ q, i });
+  return out;
+}
+
+function mismatch(what, was, got) {
+  const err = new Error(`cannot resume: ${what} was ${was}, now ${got}`);
+  err.messageKey = 'resumeMismatch';
+  err.vars = { what, was, got };
+  return err;
+}
+
+// The conditions a resumed run must keep. Records from different conditions cannot be pooled, so a
+// change is refused rather than merged. `want` holds only what the caller supplied; anything left out
+// is taken from the run.
+export function checkResume(meta, want) {
+  for (const k of ['model', 'runs', 'maxExplore']) {
+    if (want[k] !== undefined && want[k] !== null && String(want[k]) !== String(meta[k])) {
+      throw mismatch(k, meta[k], want[k]);
+    }
+  }
+  if (want.backend !== undefined && want.backend !== meta.backend) throw mismatch('backend', meta.backend, want.backend);
+}
+
+export function loadRunQueries(dir, meta) {
+  const skillDir = path.resolve(dir, '..', '..', '..');
+  let queries = null;
+  try { queries = JSON.parse(fs.readFileSync(path.join(dir, 'queries.json'), 'utf8')).queries; } catch { /* older run */ }
+  if (!queries) {
+    // Runs written before queries.json existed: fall back to the cache file they were drawn from.
+    try { queries = JSON.parse(fs.readFileSync(path.join(skillDir, '.skillgauntlet', meta.querySets[0].file), 'utf8')).queries; } catch { /* gone */ }
+  }
+  const was = meta.querySets?.[0]?.sha256;
+  if (!queries || querySetHash(queries) !== was) {
+    const err = new Error('cannot resume: the question set no longer matches the one this run used');
+    err.messageKey = 'resumeQueriesChanged';
+    throw err;
+  }
+  if (meta.skillFingerprint && fingerprint(skillDir) !== meta.skillFingerprint) {
+    const err = new Error('cannot resume: SKILL.md changed since this run started');
+    err.messageKey = 'resumeSkillChanged';
+    throw err;
+  }
+  return { skillDir, queries };
+}
+
+// Continue a run directory: skip every valid record, retry errors and timeouts.
+// want: { model, runs, maxExplore, backend } as supplied now; mismatches throw with a messageKey.
+export async function resumeRun({ dir, want = {}, bin, runTrialFn = realRunTrial, workers = 2, timeout, onTrial }) {
+  const { meta, records } = readRun(dir);
+  checkResume(meta, want);
+  const { skillDir, queries } = loadRunQueries(dir, meta);
+  const todo = pendingTrials(queries, meta.runs, records);
+  if (!todo.length) return { nothing: true, meta };
+
+  const run = reopenRun(dir);
+  const doTrial = ({ q }) => runTrialFn({
+    bin, skillDir, skillName: meta.skill, query: q.query, model: meta.model, maxExplore: meta.maxExplore, timeout,
+    host: meta.host,
+  });
+  let outcome;
+  try {
+    outcome = await executeTrials({ run, trials: todo, doTrial, workers, onTrial });
+  } catch (err) {
+    run.close(null, { interrupted: true });
+    throw err;
+  }
+  const result = finishRun(run, {
+    model: meta.model, backend: meta.backend, host: meta.host, runs: meta.runs, expected: queries.length * meta.runs,
+  }, outcome.stopped);
+  return { nothing: false, result, stopped: outcome.stopped, dir, meta: run.meta, total: todo.length, done: outcome.done };
+}

package/lib/gain.js

@@ -0,0 +1,177 @@
+// Layer 2: gain. "Does the skill actually help" can only be answered honestly by pairing a task
+// with a verifier that code can decide -- a test that passes, a file that appears, a value that
+// matches. An LLM judge would give a general answer at the cost of being expensive, biased and
+// not reproducible, so it is deliberately not implemented.
+//
+// Contract DSL (contracts/<skill>.json):
+//   { "skill": "tdd",
+//     "tasks":    [{ "id": "t1", "prompt": "…", "files": { "calc.py": "…" } }],
+//     "contracts":[ { "desc": "a test file was added", "type": "files_added_matching", "glob": "**/*test*",
+//                     "descI18n": { "zh": "新增了测试文件", "ja": "テストファイルが追加された" } },
+//                   { "desc": "tests pass",            "type": "command_succeeds",     "cmd": "python -m pytest -q" },
+//                   { "desc": "output contains a formula", "type": "file_contains",   "glob": "out.csv", "regex": "=" } ] }
+//
+// "desc" is English and doubles as the stable id a contract is aggregated and archived under.
+// "descI18n" (optional) holds the same sentence for other UI languages; the report picks it by
+// language and falls back to "desc".
+//
+// A skill with no contracts file is reported as "not measurable", not guessed at.
+import { execFileSync, spawn } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const HERE = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const CONTRACT_DIR = path.join(HERE, 'contracts');
+
+function copyDir(src, dst) {
+  fs.mkdirSync(dst, { recursive: true });
+  for (const e of fs.readdirSync(src, { withFileTypes: true })) {
+    if (e.name === '.git') continue;
+    const s = path.join(src, e.name);
+    const d = path.join(dst, e.name);
+    if (e.isDirectory()) copyDir(s, d);
+    else if (e.isFile()) fs.copyFileSync(s, d);
+  }
+}
+
+export function loadContracts(skillName) {
+  const file = path.join(CONTRACT_DIR, `${skillName}.json`);
+  if (!fs.existsSync(file)) return null;
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function listFiles(root) {
+  const out = [];
+  const walk = (dir) => {
+    let entries = [];
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const e of entries) {
+      if (e.name === '.claude' || e.name === '.git' || e.name === 'node_modules') continue;
+      const p = path.join(dir, e.name);
+      if (e.isDirectory()) walk(p);
+      else out.push(path.relative(root, p).replace(/\\/g, '/'));
+    }
+  };
+  walk(root);
+  return out;
+}
+
+function globToRe(glob) {
+  const body = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*\*\//g, '(?:.*/)?').replace(/\*/g, '[^/]*');
+  return new RegExp(`^${body}$`);
+}
+
+export function evaluateContract(contract, dir, before) {
+  const after = listFiles(dir);
+  const added = after.filter((f) => !before.includes(f));
+  switch (contract.type) {
+    case 'files_added_matching': {
+      const re = globToRe(contract.glob);
+      return added.some((f) => re.test(f));
+    }
+    case 'no_files_matching': {
+      const re = globToRe(contract.glob);
+      return !after.some((f) => re.test(f));
+    }
+    case 'file_contains': {
+      const re = globToRe(contract.glob);
+      const target = after.find((f) => re.test(f));
+      if (!target) return false;
+      try {
+        // An .xlsx is a zip, so its entry names sit in the archive uncompressed. Reading latin1 lets
+        // the same check assert that a spreadsheet is a real one without adding a zip dependency.
+        const enc = contract.encoding === 'latin1' ? 'latin1' : 'utf8';
+        return new RegExp(contract.regex).test(fs.readFileSync(path.join(dir, target), enc));
+      } catch { return false; }
+    }
+    case 'command_succeeds': {
+      try {
+        execFileSync(contract.cmd, { cwd: dir, shell: true, stdio: 'ignore', timeout: 120000 });
+        return true;
+      } catch { return false; }
+    }
+    default:
+      return null;
+  }
+}
+
+function runTask({ bin, model, task, skillDir, skillName, timeout }) {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sg-gain-'));
+  for (const [rel, content] of Object.entries(task.files || {})) {
+    const p = path.join(tmp, rel);
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, content);
+  }
+  if (skillDir) copyDir(skillDir, path.join(tmp, '.claude', 'skills', skillName));
+  const before = listFiles(tmp);
+  const args = ['-p', task.prompt, '--permission-mode', 'dontAsk', '--setting-sources', 'project',
+    '--strict-mcp-config', '--no-session-persistence'];
+  if (model) args.push('--model', model);
+
+  return new Promise((resolve) => {
+    const child = spawn(bin, args, {
+      cwd: tmp, shell: /\.(cmd|bat)$/i.test(bin), env: process.env, windowsHide: true,
+    });
+    const timer = setTimeout(() => {
+      try {
+        if (process.platform === 'win32') execFileSync('taskkill', ['/T', '/F', '/PID', String(child.pid)], { stdio: 'ignore' });
+        else child.kill('SIGKILL');
+      } catch { /* ignore */ }
+    }, timeout * 1000);
+    child.on('close', () => {
+      clearTimeout(timer);
+      resolve({ dir: tmp, before });
+    });
+    child.on('error', () => { clearTimeout(timer); resolve({ dir: tmp, before }); });
+  });
+}
+
+export async function runGain({ bin, skillDir, skillName, model, timeout = 600, onProgress }) {
+  const spec = loadContracts(skillName);
+  const hasContracts = spec?.contracts?.length || spec?.tasks?.some((t) => t.contracts?.length);
+  if (!spec || !hasContracts || !spec.tasks?.length) {
+    return { measurable: false, reasonKey: 'gain.noContracts' };
+  }
+  const detail = [];
+  for (const task of spec.tasks) {
+    // Contracts belong to the task that can satisfy them: "a divide function was added" only makes
+    // sense for the task that asks for one. A skill-level list is still allowed for contracts that
+    // genuinely apply to every task.
+    const contracts = task.contracts?.length ? task.contracts : (spec.contracts || []);
+    const row = { id: task.id, prompt: task.prompt, with: null, without: null };
+    for (const condition of ['with', 'without']) {
+      if (onProgress) onProgress(task.id, condition);
+      const { dir, before } = await runTask({
+        bin, model, task,
+        skillDir: condition === 'with' ? skillDir : null,
+        skillName, timeout,
+      });
+      const added = listFiles(dir).filter((f) => !before.includes(f));
+      const checks = contracts.map((c) => ({ desc: c.desc, descI18n: c.descI18n, ok: Boolean(evaluateContract(c, dir, before)) }));
+      row[condition] = { added, checks };
+      try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ }
+    }
+    detail.push(row);
+  }
+  // Aggregate by description across the tasks that use it.
+  const agg = new Map();
+  for (const row of detail) {
+    for (const condition of ['with', 'without']) {
+      for (const c of row[condition].checks) {
+        const e = agg.get(c.desc) || { with: 0, without: 0, n: 0, descI18n: c.descI18n };
+        e[condition] += c.ok ? 1 : 0;
+        e.n += 1;
+        agg.set(c.desc, e);
+      }
+    }
+  }
+  return {
+    measurable: true,
+    localOnly: true,
+    tasks: detail,
+    contracts: [...agg].map(([desc, e]) => ({ desc, descI18n: e.descI18n, with: e.with / e.n, without: e.without / e.n })),
+    taskCount: detail.length,
+  };
+}