sillyspec 3.27.2 → 3.27.5

package/eval/lib/db.mjs

@@ -0,0 +1,76 @@
+// eval 结果库（node:sqlite，独立于 src/db.js 的进度库——评测数据与工具自身状态解耦）。
+// 只记录不解释：通过率/flaky/基线对比等统计全部在 report 层算，保持本层无状态可替换。
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+
+const DDL = `
+CREATE TABLE IF NOT EXISTS runs (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  arm TEXT NOT NULL,               -- A=裸 agent / B=SillySpec 流程
+  adapter TEXT NOT NULL,           -- dryrun | cli
+  model_label TEXT,
+  config_hash TEXT NOT NULL,       -- 实验分组（配置+任务指纹+版本）
+  tool_version TEXT,               -- sillyspec 版本（发版回归对比用）
+  started_at TEXT NOT NULL,
+  duration_ms INTEGER NOT NULL DEFAULT 0,
+  exit_code INTEGER,
+  status TEXT NOT NULL,            -- pass | fail | timeout | error
+  verify_output TEXT,
+  tokens_in INTEGER,
+  tokens_out INTEGER,
+  cost_usd REAL,
+  agent_log_path TEXT,
+  notes TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_runs_group ON runs(config_hash, adapter, arm, task_id);
+`;
+
+export class EvalDB {
+  constructor(dbPath) {
+    this.dbPath = dbPath;
+    this.db = null;
+  }
+
+  init() {
+    mkdirSync(dirname(this.dbPath), { recursive: true });
+    this.db = new DatabaseSync(this.dbPath);
+    this.db.exec(DDL);
+    return this;
+  }
+
+  insertRun(row) {
+    this.db.prepare(`
+      INSERT INTO runs (task_id, arm, adapter, model_label, config_hash, tool_version,
+                        started_at, duration_ms, exit_code, status, verify_output,
+                        tokens_in, tokens_out, cost_usd, agent_log_path, notes)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      row.task_id, row.arm, row.adapter, row.model_label ?? null, row.config_hash, row.tool_version ?? null,
+      row.started_at, row.duration_ms ?? 0, row.exit_code ?? null, row.status, row.verify_output ?? null,
+      row.tokens_in ?? null, row.tokens_out ?? null, row.cost_usd ?? null, row.agent_log_path ?? null, row.notes ?? null,
+    );
+    return this.db.prepare('SELECT last_insert_rowid() AS id').get().id;
+  }
+
+  /** 同 (hash, task, arm) 已有 pass/fail 终态则可跳过重跑（--force 旁路） */
+  cachedRun(configHash, taskId, arm) {
+    return this.db.prepare(`
+      SELECT * FROM runs
+      WHERE config_hash = ? AND task_id = ? AND arm = ? AND status IN ('pass','fail')
+      ORDER BY id DESC LIMIT 1
+    `).get(configHash, taskId, arm) ?? null;
+  }
+
+  runsInGroup(configHash, adapter) {
+    return this.db.prepare('SELECT * FROM runs WHERE config_hash = ? AND adapter = ? ORDER BY id').all(configHash, adapter);
+  }
+
+  groups() {
+    return this.db.prepare(`
+      SELECT config_hash, adapter, COUNT(*) AS n, MAX(id) AS last_id
+      FROM runs GROUP BY config_hash, adapter ORDER BY last_id DESC
+    `).all();
+  }
+}

package/eval/lib/report.mjs

@@ -0,0 +1,120 @@
+// 报告：分臂通过率 + Wilson 95% CI + Δ(B−A) 区间 + flaky 剔除 + 基线回归对比。
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { computeConfigHash, sillyspecVersion } from './config.mjs';
+import { EvalDB } from './db.mjs';
+import { loadTasks, tasksFingerprint } from './tasks.mjs';
+import { deltaCI, fmtPct, fmtPp, wilsonCI } from './util.mjs';
+
+export function buildReport(config, opts = {}) {
+  const dbFile = join(config.resultsDir, 'eval.db');
+  if (!existsSync(dbFile)) return { empty: true };
+  const db = new EvalDB(dbFile).init();
+
+  const adapterType = opts.dryRun ? 'dryrun' : (config.adapter.type ?? 'cli');
+  const currentHash = computeConfigHash(config, adapterType, tasksFingerprint(loadTasks(config.tasksDir)));
+  const groups = db.groups();
+  const group = groups.find((g) => g.config_hash === currentHash && g.adapter === adapterType)
+    ?? groups.find((g) => g.adapter === adapterType);
+  if (!group) return { empty: true };
+
+  const rows = db.runsInGroup(group.config_hash, adapterType);
+  const byArm = { A: new Map(), B: new Map() };
+  for (const r of rows) {
+    if (r.status !== 'pass' && r.status !== 'fail') continue; // timeout/error 不计入通过率
+    if (!byArm[r.arm]) continue;
+    if (!byArm[r.arm].has(r.task_id)) byArm[r.arm].set(r.task_id, []);
+    byArm[r.arm].get(r.task_id).push(r.status);
+  }
+
+  const summary = {
+    adapterType,
+    configHash: group.config_hash,
+    hashCurrent: group.config_hash === currentHash,
+    sillyspecVersion: sillyspecVersion(),
+    arms: {},
+  };
+  for (const arm of Object.keys(byArm)) {
+    let n = 0;
+    let pass = 0;
+    const flaky = [];
+    for (const [taskId, statuses] of byArm[arm]) {
+      if (new Set(statuses).size > 1) flaky.push(taskId); // 同任务同臂结果摇摆 → flaky，剔除
+      else { n++; if (statuses[0] === 'pass') pass++; }
+    }
+    summary.arms[arm] = {
+      label: config.arms[arm]?.label ?? arm,
+      n,
+      pass,
+      rate: n ? pass / n : null,
+      ci: wilsonCI(pass, n),
+      flaky,
+    };
+  }
+  const A = summary.arms.A;
+  const B = summary.arms.B;
+  if (A?.rate != null && B?.rate != null) {
+    summary.delta = { pp: B.rate - A.rate, ci: deltaCI(A.rate, A.n, B.rate, B.n) };
+  }
+  return summary;
+}
+
+export function printReport(config, summary) {
+  if (!summary || summary.empty) {
+    console.log('暂无可报告的评测数据（先 --dry-run 或真跑一轮）');
+    return;
+  }
+  console.log('\n==== SillySpec Eval 报告 ====');
+  console.log(`adapter=${summary.adapterType}  hash=${summary.configHash}${summary.hashCurrent ? '' : '（任务/配置已变更，以上为最近一轮）'}  sillyspec=${summary.sillyspecVersion}`);
+  console.log('\n臂            n    pass  通过率   95% CI');
+  for (const arm of ['A', 'B']) {
+    const s = summary.arms[arm];
+    if (!s || (s.n === 0 && s.flaky.length === 0)) continue;
+    console.log(
+      `${s.label.padEnd(12).slice(0, 12)}  ${String(s.n).padEnd(4)} ${String(s.pass).padEnd(5)} `
+      + `${fmtPct(s.rate).padEnd(7)} [${fmtPct(s.ci[0])}, ${fmtPct(s.ci[1])}]`,
+    );
+  }
+  if (summary.delta) {
+    console.log(`\nΔ(B−A) = ${fmtPp(summary.delta.pp)}   95% CI [${fmtPp(summary.delta.ci[0])}, ${fmtPp(summary.delta.ci[1])}]`);
+    if (Math.min(summary.arms.A.n, summary.arms.B.n) < 30) {
+      console.log('（单臂样本 < 30，区间仅供方向参考）');
+    }
+  }
+  const flakyAll = ['A', 'B'].flatMap((a) => (summary.arms[a]?.flaky ?? []).map((t) => `${t}(臂${a})`));
+  if (flakyAll.length) console.log(`flaky ${flakyAll.length} 个（已从通过率剔除）：${flakyAll.join('、')}`);
+
+  const baselinePath = join(config.resultsDir, 'baseline.json');
+  if (existsSync(baselinePath)) {
+    const baseline = JSON.parse(readFileSync(baselinePath, 'utf8'));
+    const threshold = (config.regressionThresholdPp ?? 5) / 100;
+    const parts = [];
+    for (const arm of ['A', 'B']) {
+      const cur = summary.arms[arm]?.rate;
+      const base = baseline.arms?.[arm]?.rate;
+      if (cur == null || base == null) continue;
+      const diff = cur - base;
+      const flag = diff < -threshold ? ' ⚠️超回归阈值' : '';
+      parts.push(`臂${arm} ${fmtPct(base)}→${fmtPct(cur)}（${fmtPp(diff)}${flag}）`);
+    }
+    if (parts.length) console.log(`基线对比（${baseline.createdAt}）：${parts.join('  ')}`);
+  }
+}
+
+export function setBaseline(config, opts) {
+  const summary = buildReport(config, opts);
+  if (!summary || summary.empty) {
+    console.log('暂无数据可固化为基线');
+    return;
+  }
+  const pick = (a) => (a && a.rate != null ? { n: a.n, rate: a.rate } : null);
+  const payload = {
+    createdAt: new Date().toISOString(),
+    configHash: summary.configHash,
+    adapter: summary.adapterType,
+    sillyspecVersion: summary.sillyspecVersion,
+    arms: { A: pick(summary.arms.A), B: pick(summary.arms.B) },
+  };
+  writeFileSync(join(config.resultsDir, 'baseline.json'), JSON.stringify(payload, null, 2));
+  console.log(`基线已固化 → ${join(config.resultsDir, 'baseline.json')}`);
+}

package/eval/lib/runner.mjs

@@ -0,0 +1,157 @@
+// 编排：任务 × 臂 → (环境准备 → agent → verify) → SQLite 落库。
+// 缓存：同 (config_hash, task, arm) 已有 pass/fail 终态则跳过（--force 旁路）——
+// 调提示词/流程后只重跑受影响的臂，是控制 token 成本的主要手段。
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { computeConfigHash } from './config.mjs';
+import { EvalDB } from './db.mjs';
+import { loadTasks, tasksFingerprint } from './tasks.mjs';
+import { prepareWorkdir } from './workdir.mjs';
+import { spawnCapture, tail } from './util.mjs';
+import { run as dryRunAdapter } from './adapters/dryrun.mjs';
+import { run as cliAdapter } from './adapters/cli.mjs';
+
+const FAILED_STATUSES = new Set(['fail', 'timeout', 'error']);
+
+export async function runEval(config, opts) {
+  const allTasks = loadTasks(config.tasksDir);
+  if (allTasks.length === 0) {
+    console.error(`任务目录为空: ${config.tasksDir}（先在 eval/tasks/ 下添加任务）`);
+    return { ran: 0, skipped: 0 };
+  }
+  const fingerprint = tasksFingerprint(allTasks);
+
+  let tasks = allTasks;
+  if (opts.taskIds?.length) tasks = tasks.filter((t) => opts.taskIds.includes(t.id));
+  if (opts.sample) tasks = tasks.slice(0, opts.sample);
+  if (tasks.length === 0) {
+    console.error('没有匹配的任务');
+    return { ran: 0, skipped: 0 };
+  }
+
+  const adapterType = opts.dryRun ? 'dryrun' : config.adapter.type;
+  const configHash = computeConfigHash(config, adapterType, fingerprint);
+  const db = new EvalDB(join(config.resultsDir, 'eval.db')).init();
+
+  const arms = opts.arms?.length ? opts.arms : ['A', 'B'];
+  let plan = [];
+  for (const arm of arms) {
+    for (const task of tasks) plan.push({ arm, task });
+  }
+  if (opts.onlyFailed) {
+    const latest = latestByTaskArm(db.runsInGroup(configHash, adapterType));
+    plan = plan.filter(({ arm, task }) => {
+      const prev = latest.get(`${arm}|${task.id}`);
+      return prev ? FAILED_STATUSES.has(prev.status) : true; // 无记录的新任务也跑
+    });
+  }
+
+  const workdirRoot = join(config.resultsDir, 'workdirs');
+  mkdirSync(workdirRoot, { recursive: true });
+
+  let ran = 0;
+  let skipped = 0;
+  // --rerun-failed 语义就是「重跑失败者」，必须旁路缓存（否则 fail 终态被缓存拦下，一次都跑不了）
+  const useCache = !opts.force && !opts.onlyFailed;
+  for (const { arm, task } of plan) {
+    const cached = useCache ? db.cachedRun(configHash, task.id, arm) : null;
+    if (cached) {
+      skipped++;
+      console.log(`[cache] ${task.id} 臂${arm} → ${cached.status}（--force 可重跑）`);
+      continue;
+    }
+    const result = await executeRun({ config, opts, adapterType, configHash, db, task, arm, workdirRoot });
+    ran++;
+    const icon = result.status === 'pass' ? '✓' : result.status === 'fail' ? '✗' : '!';
+    console.log(`[${icon}] ${task.id} 臂${arm} ${result.status} ${result.durationMs}ms${result.note ? `（${result.note}）` : ''}`);
+  }
+  console.log(`完成：新跑 ${ran}，缓存跳过 ${skipped}（hash=${configHash}）`);
+  return { ran, skipped, configHash, adapterType };
+}
+
+function latestByTaskArm(rows) {
+  const m = new Map(); // rows 按 id 升序，后写覆盖即最新
+  for (const r of rows) m.set(`${r.arm}|${r.task_id}`, r);
+  return m;
+}
+
+async function executeRun({ config, opts, adapterType, configHash, db, task, arm, workdirRoot }) {
+  const t0 = Date.now();
+  const armCfg = config.arms[arm] ?? {};
+  const workdir = prepareWorkdir(workdirRoot, task.id, arm, task.filesDir);
+  const row = {
+    task_id: task.id,
+    arm,
+    adapter: adapterType,
+    model_label: opts.dryRun ? 'dry-run' : (config.adapter.tool ?? adapterType),
+    config_hash: configHash,
+    tool_version: null,
+    started_at: new Date().toISOString(),
+    duration_ms: 0,
+    exit_code: null,
+    status: 'error',
+    verify_output: null,
+    agent_log_path: null,
+    notes: null,
+    tokens_in: null,
+    tokens_out: null,
+    cost_usd: null,
+  };
+
+  // 臂环境准备：B 臂执行 sillyspec init 等（dry-run 跳过，保持零副作用）
+  if (!opts.dryRun) {
+    for (const cmd of armCfg.setup ?? []) {
+      const args = render(cmd, { workdir, tool: config.adapter.tool ?? '' }).split(/\s+/).filter(Boolean);
+      const r = await spawnCapture({ args, cwd: workdir, timeoutMs: 120_000 });
+      if (r.code !== 0) {
+        row.status = 'error';
+        row.notes = `臂 setup 失败: ${cmd}`;
+        row.verify_output = tail(`${r.stdout}\n${r.stderr}`);
+        row.exit_code = r.code;
+        row.duration_ms = Date.now() - t0;
+        db.insertRun(row);
+        return { status: 'error', durationMs: row.duration_ms, note: row.notes };
+      }
+    }
+  }
+
+  const prompt = (armCfg.promptTemplate ?? '{instruction}').replaceAll('{instruction}', task.instruction);
+  const logDir = join(config.resultsDir, 'logs', `${stamp()}-${task.id}-arm${arm}`);
+  let res;
+  if (opts.dryRun) {
+    res = dryRunAdapter({ task, arm, workdir, solutionDir: task.solutionDir, mode: opts.dryMode ?? 'mixed' });
+  } else {
+    res = await cliAdapter({ adapter: config.adapter, prompt, workdir, logDir });
+  }
+  row.agent_log_path = res.logDir ?? null;
+  row.exit_code = res.code ?? null;
+  row.tokens_in = res.tokensIn ?? null;
+  row.tokens_out = res.tokensOut ?? null;
+  row.cost_usd = res.costUsd ?? null;
+
+  if (res.timedOut) {
+    row.status = 'timeout';
+  } else if ((res.code ?? -1) !== 0) {
+    row.status = 'error';
+    row.verify_output = tail(res.stderr || res.stdout);
+  } else {
+    // 判分：verify.mjs 以 workdir 为 cwd 执行，退出码 0 = pass
+    const v = await spawnCapture({ args: ['node', task.verifyPath], cwd: workdir, timeoutMs: config.verifyTimeoutMs ?? 120_000 });
+    row.status = v.code === 0 ? 'pass' : 'fail';
+    row.verify_output = tail(`${v.stderr}\n${v.stdout}`);
+  }
+
+  row.duration_ms = Date.now() - t0;
+  db.insertRun(row);
+  return { status: row.status, durationMs: row.duration_ms, note: null };
+}
+
+function render(cmd, vars) {
+  let s = cmd;
+  for (const [k, v] of Object.entries(vars)) s = s.replaceAll(`{${k}}`, v);
+  return s;
+}
+
+function stamp() {
+  return new Date().toISOString().replace(/[:.]/g, '-');
+}

package/eval/lib/tasks.mjs

@@ -0,0 +1,73 @@
+// 任务加载与内容指纹。任务 = eval/tasks/<id>/ 下四件套：
+//   task.json（id/instruction）· verify.mjs（cwd=被测工作目录，退出码 0=过）· files/（工作副本脚手架）· solution/（dry-run 参考解）
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+export function loadTasks(tasksDir) {
+  if (!existsSync(tasksDir)) return [];
+  const out = [];
+  for (const entry of readdirSync(tasksDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const dir = join(tasksDir, entry.name);
+    const taskFile = join(dir, 'task.json');
+    if (!existsSync(taskFile)) continue;
+    const meta = JSON.parse(readFileSync(taskFile, 'utf8'));
+    const task = {
+      id: meta.id ?? entry.name,
+      title: meta.title ?? entry.name,
+      instruction: meta.instruction,
+      tags: meta.tags ?? [],
+      dir,
+      verifyPath: join(dir, 'verify.mjs'),
+      filesDir: join(dir, 'files'),
+      solutionDir: join(dir, 'solution'),
+    };
+    if (typeof task.instruction !== 'string' || !task.instruction.trim()) {
+      throw new Error(`任务 ${task.id} 缺少 instruction`);
+    }
+    if (!existsSync(task.verifyPath)) {
+      throw new Error(`任务 ${task.id} 缺少 verify.mjs`);
+    }
+    out.push(task);
+  }
+  out.sort((a, b) => a.id.localeCompare(b.id));
+  const seen = new Set();
+  for (const t of out) {
+    if (seen.has(t.id)) throw new Error(`任务 id 重复: ${t.id}`);
+    seen.add(t.id);
+  }
+  return out;
+}
+
+/**
+ * 全量任务内容指纹（task.json + verify.mjs + files/ 递归内容；solution/ 不参与——
+ * 它只服务 dry-run，改动不应让真实评测换组）。始终对目录内全部任务计算，
+ * 与实际抽样跑哪些无关——否则 --task 子集会产生另一套分组，报告对不上。
+ */
+export function tasksFingerprint(tasks) {
+  const h = createHash('sha256');
+  for (const t of tasks) {
+    h.update(t.id); h.update('\0');
+    h.update(readFileSync(join(t.dir, 'task.json')));
+    h.update('\0');
+    h.update(readFileSync(t.verifyPath));
+    h.update('\0');
+    for (const rel of walkFiles(t.filesDir)) {
+      h.update(rel); h.update('\0');
+      h.update(readFileSync(join(t.filesDir, rel))); h.update('\0');
+    }
+  }
+  return h.digest('hex').slice(0, 16);
+}
+
+function walkFiles(dir, prefix = '') {
+  if (!existsSync(dir)) return [];
+  const acc = [];
+  for (const e of readdirSync(dir, { withFileTypes: true })) {
+    const rel = prefix ? `${prefix}/${e.name}` : e.name;
+    if (e.isDirectory()) acc.push(...walkFiles(join(dir, e.name), rel));
+    else acc.push(rel);
+  }
+  return acc.sort();
+}

package/eval/lib/util.mjs

@@ -0,0 +1,107 @@
+// eval 通用工具：稳定哈希、跨平台 shell 执行、统计区间。
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+
+export function sha256(text) {
+  return createHash('sha256').update(text).digest('hex');
+}
+
+// 键排序的稳定序列化：同一配置内容（键序无关）必须得到同一哈希，缓存/报告分组才可靠
+export function stableStringify(value) {
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  if (value && typeof value === 'object') {
+    const keys = Object.keys(value).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+// 平台感知参数引用：Windows(cmd) 双引号、POSIX 单引号。
+// 评测命令/路径不含引号；prompt 一律走 stdin 注入，绕开跨平台转义深坑。
+export function shQuote(arg) {
+  const s = String(arg);
+  if (process.platform === 'win32') return `"${s.replace(/"/g, '""')}"`;
+  return `'${s.replace(/'/g, `'\\''`)}'`;
+}
+
+/**
+ * 执行命令并捕获输出（shell 模式，Windows 下解析 .cmd 入口如 npx/claude 需要）。
+ * 输出超过 maxOutputChars 停止累积（防 agent 刷屏撑爆内存），标记 truncated。
+ * timeoutMs 到点杀进程树（win32 用 taskkill /T，POSIX kill 进程组语义下杀子进程）。
+ */
+export function spawnCapture({ args, cwd, timeoutMs = 0, input = null, maxOutputChars = 2_000_000 }) {
+  return new Promise((resolve) => {
+    const started = Date.now();
+    const command = args.map(shQuote).join(' ');
+    let child;
+    try {
+      child = spawn(command, { cwd, shell: true, windowsHide: true });
+    } catch (err) {
+      resolve({ code: -1, stdout: '', stderr: String(err), timedOut: false, truncated: false, durationMs: 0 });
+      return;
+    }
+    const state = { stdout: '', stderr: '' };
+    let truncated = false;
+    let timedOut = false;
+    let settled = false;
+    const onChunk = (key) => (buf) => {
+      if (state[key].length >= maxOutputChars) { truncated = true; return; }
+      state[key] += buf.toString('utf8');
+    };
+    child.stdout?.on('data', onChunk('stdout'));
+    child.stderr?.on('data', onChunk('stderr'));
+    let timer = null;
+    if (timeoutMs > 0) {
+      timer = setTimeout(() => {
+        timedOut = true;
+        if (process.platform === 'win32' && child.pid) {
+          try { spawn('taskkill /pid ' + child.pid + ' /T /F', { shell: true, windowsHide: true }); } catch { /* 尽力杀 */ }
+        }
+        child.kill('SIGKILL');
+      }, timeoutMs);
+    }
+    if (input != null) {
+      child.stdin?.on('error', () => { /* stdin 中途关闭（进程先退出）不视为致命 */ });
+      child.stdin?.end(input);
+    }
+    const settle = (patch) => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      resolve({ durationMs: Date.now() - started, truncated, timedOut, ...state, ...patch });
+    };
+    child.on('error', (err) => settle({ code: -1, stderr: state.stderr + String(err) }));
+    child.on('close', (code) => settle({ code: code === null ? -1 : code }));
+  });
+}
+
+/** Wilson 95% 置信区间（小样本比正态近似稳） */
+export function wilsonCI(pass, n, z = 1.96) {
+  if (!n) return [null, null];
+  const p = pass / n;
+  const den = 1 + (z * z) / n;
+  const center = (p + (z * z) / (2 * n)) / den;
+  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / den;
+  return [Math.max(0, center - half), Math.min(1, center + half)];
+}
+
+/** 两比例之差（B−A）的正态近似区间 */
+export function deltaCI(pA, nA, pB, nB, z = 1.96) {
+  if (!nA || !nB) return [null, null];
+  const se = Math.sqrt((pA * (1 - pA)) / nA + (pB * (1 - pB)) / nB);
+  const d = pB - pA;
+  return [d - z * se, d + z * se];
+}
+
+export function fmtPct(x) {
+  return x == null ? '—' : `${(x * 100).toFixed(1)}%`;
+}
+
+export function fmtPp(x) {
+  return x == null ? '—' : `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}pp`;
+}
+
+export function tail(text, n = 600) {
+  const s = (text ?? '').trim();
+  return s.length <= n ? s : `…${s.slice(-n)}`;
+}

package/eval/lib/workdir.mjs

@@ -0,0 +1,30 @@
+// 被测工作副本：每个 (task, arm) 一个独立目录，从任务 files/ 复制脚手架。
+// 每次重跑前清空重建，保证 agent 面对的初始状态确定。目录保留不删（排查用），在 .gitignore 里。
+import { cpSync, existsSync, mkdirSync, renameSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+
+// Windows 下刚退出的子进程句柄可能滞后释放，直接 rmSync 会 EPERM：
+// 先带重试删；仍失败则改名隔离（.stale），新一轮照常进行，遗留目录不影响判分。
+function removeDir(p) {
+  try {
+    rmSync(p, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+  } catch {
+    const stale = `${p}.stale`;
+    try {
+      rmSync(stale, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+      renameSync(p, stale);
+    } catch { /* 双重失败只能上抛，fail-loud */ }
+  }
+}
+
+export function prepareWorkdir(workdirRoot, taskId, arm, filesDir) {
+  const workdir = join(workdirRoot, `${taskId}__arm${arm}`);
+  removeDir(workdir);
+  mkdirSync(workdir, { recursive: true });
+  if (existsSync(filesDir)) cpSync(filesDir, workdir, { recursive: true });
+  return workdir;
+}
+
+export function copyInto(workdir, sourceDir) {
+  if (existsSync(sourceDir)) cpSync(sourceDir, workdir, { recursive: true });
+}

package/eval/run.mjs

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+// SillySpec A/B 评测入口。
+// A 臂 = 裸 agent 直接做任务；B 臂 = 同 agent + SillySpec 流程。同任务、同模型、同判分脚本（verify.mjs），
+// 差值 Δ 即 SillySpec 的真实增量。详细文档见 eval/README.md。
+import { loadConfig } from './lib/config.mjs';
+import { runEval } from './lib/runner.mjs';
+import { buildReport, printReport, setBaseline } from './lib/report.mjs';
+import { spawnCapture, tail } from './lib/util.mjs';
+
+function parseArgs(argv) {
+  const opts = {
+    dryRun: false, dryMode: 'mixed', taskIds: [], arms: [], sample: 0,
+    force: false, onlyFailed: false, report: false, setBaseline: false, preflight: false, help: false,
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--dry-run') opts.dryRun = true;
+    else if (a.startsWith('--dry-run=')) { opts.dryRun = true; opts.dryMode = a.slice('--dry-run='.length); }
+    else if (a === '--task') opts.taskIds.push(argv[++i]);
+    else if (a === '--arm') opts.arms.push(...(argv[++i] ?? '').split(',').map((s) => s.trim().toUpperCase()).filter(Boolean));
+    else if (a === '--sample') opts.sample = Number(argv[++i]);
+    else if (a === '--pilot') opts.sample = 20; // pilot = 前 20 个任务双臂
+    else if (a === '--force') opts.force = true;
+    else if (a === '--rerun-failed') opts.onlyFailed = true;
+    else if (a === '--report') opts.report = true;
+    else if (a === '--set-baseline') opts.setBaseline = true;
+    else if (a === '--preflight') opts.preflight = true;
+    else if (a === '--help' || a === '-h') opts.help = true;
+    else { console.error(`未知参数: ${a}`); opts.help = true; }
+  }
+  return opts;
+}
+
+function help() {
+  console.log(`SillySpec A/B 评测（A=裸 agent，B=SillySpec 流程，verify.mjs 客观判分）
+
+用法：node eval/run.mjs <选项>
+
+  --dry-run[=mixed|pass|fail]  零 token 验证管道（假 agent，不调模型不装依赖）
+  --preflight                  检查 Node 版本与 agent CLI 可用性（零 token）
+  --pilot                      真跑 pilot：前 20 个任务 × 双臂（消耗真实 token）
+  --task <id>                  只跑指定任务（可重复）
+  --arm A|B|A,B                只跑指定臂
+  --sample <N>                 只跑前 N 个任务
+  --rerun-failed               只重跑上一轮 fail/timeout/error（省 token 的定向重跑）
+  --force                      忽略缓存强制重跑
+  --report [--dry-run]         查看报告（Wilson CI / Δ / flaky / 基线对比）
+  --set-baseline               把当前成绩固化为回归基线（发版前对比用）`);
+}
+
+async function preflight(config) {
+  const major = Number(process.versions.node.split('.')[0]);
+  console.log(`Node ${process.version} ${major >= 22 ? '✓' : '✗（eval 需 >= 22，node:sqlite）'}`);
+  const bin = config.adapter.command[0];
+  const r = await spawnCapture({ args: [bin, '--version'], timeoutMs: 20_000 });
+  if (r.code === 0) {
+    console.log(`${bin} --version ✓ ${tail(r.stdout, 120)}`);
+  } else {
+    console.log(`${bin} --version ✗ 不可用（${tail(r.stderr || `exit ${r.code}`, 120)}）`);
+    console.log('  提示：先安装该 agent CLI 并在终端手动登录一次；认证沿用其登录态，本评测不接管密钥。');
+  }
+}
+
+const opts = parseArgs(process.argv.slice(2));
+if (opts.help) {
+  help();
+  process.exit(0);
+}
+const config = loadConfig();
+if (opts.preflight) {
+  await preflight(config);
+  process.exit(0);
+}
+if (opts.report) {
+  printReport(config, buildReport(config, opts));
+  process.exit(0);
+}
+if (opts.setBaseline) {
+  setBaseline(config, opts);
+  process.exit(0);
+}
+await runEval(config, opts);
+printReport(config, buildReport(config, opts)); // 跑完顺手出报告

package/eval/tasks/001-slugify/files/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "task-slugify",
+  "version": "1.0.0",
+  "type": "module",
+  "private": true
+}

package/eval/tasks/001-slugify/solution/src/slugify.mjs

@@ -0,0 +1,26 @@
+// 参考解（dry-run 假 agent 使用，也供人工对照）。
+// 思路：空白先归一为 \u0000 哨兵；字符过滤同时保留 [a-z0-9]、哨兵和字面分隔符；
+// 再把「哨兵/分隔符的连续串」折叠为单个分隔符并去首尾。
+// 注意不能只折叠哨兵——输入里本来就有的连续分隔符（如 'a--b'）也要折叠。
+const SENTINEL = '\u0000';
+
+function escapeRe(ch) {
+  return ch.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&');
+}
+
+export function slugify(value, options = {}) {
+  if (value === null || value === undefined) return '';
+  const sep = options.hyphen ?? '-';
+  if (typeof sep !== 'string' || sep.length !== 1) {
+    throw new TypeError('hyphen must be a single character');
+  }
+  const esc = escapeRe(sep);
+  return String(value)
+    .toLowerCase()
+    .replace(/\s+/g, SENTINEL)
+    .replace(new RegExp(`[^a-z0-9\\u0000${esc}]`, 'g'), '')
+    .replace(new RegExp(`[\\u0000${esc}]+`, 'g'), sep)
+    .replace(new RegExp(`^${esc}+|${esc}+$`, 'g'), '');
+}
+
+export default slugify;