promptwheel 0.8.1 → 0.8.2

package/README.md

@@ -69,7 +69,7 @@ Gaming detection is on by default in `run`; `--no-detect-gaming` gives you the b
     config, or grader, not the source, so revert and earn it in the source. Exit 1 means a guarded
     metric regressed, revert that too. Never weaken, skip, or delete a test to go green.
 
-**Two callers, by design.** Your *agent* consumes the verdict as a per-turn signal (`run --working --json`). The *harness* (a Stop-hook, or CI) runs the same check as an audit the agent cannot self-clear, because the check re-proves from the source alone and never asks the code to grade itself. See [docs/ENFORCEMENT.md](docs/ENFORCEMENT.md) for the three places to enforce it.
+**Two callers, by design.** Your *agent* consumes the verdict as a per-turn signal (`run --working --json`). The *harness* (a Stop-hook, or CI) runs the same check as an audit the agent cannot self-clear, because the check re-proves from the source alone and never asks the code to grade itself. See [docs/ENFORCEMENT.md](docs/ENFORCEMENT.md) for the two places to enforce it.
 
 ## Config, `promptwheel.config.json`
 
@@ -98,7 +98,7 @@ Gaming detection is on by default in `run`; `--no-detect-gaming` gives you the b
 
 Dependency and environment keys (config-level; `init` writes stack defaults):
 
-- **linkDirs**, dirs to symlink from your checkout into each measuring worktree, for deps that live outside git. Default `["node_modules"]`; `init` writes `[".venv"]` for Python and `[]` for Rust (`cargo test` writes to `target/`, so it is built fresh in the worktree rather than shared across refs). `"linkNodeModules": false` still works as shorthand for "link nothing".
+- **linkDirs**, dirs to symlink from your checkout into each measuring worktree, for deps that live outside git. With nothing set, a Node repo links its root `node_modules` and the `node_modules` of every workspace (any folder with a tracked `package.json`), so npm, yarn and pnpm workspaces are measured with no config. A `node_modules` that pnpm laid out, or that holds workspace links, is rebuilt in the worktree entry by entry, because pnpm refuses a symlinked one, and a tool's own state or cache folder inside it is left out. A link from `node_modules` back into the repo, which is how those tools wire a sibling package, is pointed at the worktree, so a test that imports a sibling package reads the ref being measured. `init` writes `[".venv"]` for Python and `[]` for Rust (`cargo test` writes to `target/`, so it is built fresh in the worktree rather than shared across refs). `"linkNodeModules": false` still works as shorthand for "link nothing".
 - **env**, environment variables for every metric command; `{wt}` is substituted with the worktree path. Python configs get `"PYTHONPATH": "{wt}/src:{wt}"` so the *measured ref* is imported, not an editable install of your original checkout.
 - **setup**, an optional command run once per ref before metrics (for example `npm run build` or `pip install -e .`), after any source patch, so a build-gated suite measures the ref it is actually on.
 

package/bin/promptwheel.d.ts

@@ -146,6 +146,10 @@ export interface MetricResult extends Evaluation {
   guard: boolean;
   /** With `inert`: where the command ran and failed (`both` refs, or only `base` or `head`). Absent when it ran at neither: exit 126 or 127, a timeout, or the placeholder command `init` writes. */
   ran?: 'both' | 'base' | 'head';
+  /** With `inert`: a hint read from the command's output. `missing-dependency` when it names a module, package or command the clean copy does not have, else `failing`. Never part of a verdict. */
+  cause?: 'missing-dependency' | 'failing' | 'too-much-output';
+  /** With `cause: 'missing-dependency'`: the line that names what is missing, at most 200 characters. */
+  detail?: string;
   /** `true` when the win did not survive the source-only re-run; `null` when the source-only patch could not be applied (then `unprovable` is set and the verdict is `inconclusive`, never `pass`). Absent without gaming detection or when the metric did not improve. */
   gamed?: boolean | null;
   /** `true`: the source-only patch could not be applied, so this win could not be re-proven and the verdict is `inconclusive`. `false` only after a declared override cleared it (then `allowed` is `true`). */
@@ -234,9 +238,17 @@ export interface Report {
   mandates?: MandateViolation[];
   gate?: GateRecord;
   allowed?: AllowedRecord;
+  /** Present only when the `setup` command failed: at which ref, with what exit code, and its last output line. */
+  setup?: { base?: SetupFailure; head?: SetupFailure };
   provenance: Provenance;
 }
 
+export interface SetupFailure {
+  /** The setup command's exit code, or `null` when it was killed (a timeout). */
+  exit: number | null;
+  lastLine: string;
+}
+
 // ---------------------------------------------------------------------------
 // Public library API (stable across 0.x minors)
 // ---------------------------------------------------------------------------

package/bin/promptwheel.mjs

@@ -16,7 +16,7 @@ import { execSync, execFileSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, realpathSync, readdirSync, lstatSync, copyFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { join, dirname, isAbsolute, relative } from 'node:path';
+import { join, dirname, basename, isAbsolute, relative, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 // ---------------------------------------------------------------------------
@@ -147,15 +147,26 @@ function resolveBase(repo, base) {
 // test that ran and failed
 const exitCodes = new WeakMap();
 const neverRan = (samples) => { const c = exitCodes.get(samples) || []; return c.length > 0 && c.every((x) => x == null || x === 126 || x === 127); };
-function runMetric(cwd, m, env, codes) {
+// Why a metric's command failed, read from its output: a dependency the clean copy does not have reads differently
+// from tests that simply fail. A hint for the person, never part of a verdict.
+const MISSING_DEP = /Cannot find (?:module|package) ['"][^'"\n]+['"]|ERR_MODULE_NOT_FOUND[^\n]*|ModuleNotFoundError: [^\n]+|No module named [^\n]+|error TS2307: [^\n]+|[^\n:]*: (?:command )?not found[^\n]*|can't find crate[^\n]*|cannot find package [^\n]+/i;
+function whyFailed(text) {
+  const m = String(text || '').replace(/\x1b\[[0-9;]*m/g, '').match(MISSING_DEP);
+  return m ? { cause: 'missing-dependency', detail: m[0].trim().slice(0, 200) } : { cause: 'failing' };
+}
+const failNotes = new WeakMap();   // samples -> why the last failing run failed
+function runMetric(cwd, m, env, codes, notes) {
   let stdout = '', code = 0, status = 0;
   // Never leak the harness's own node:test context into the measured gate: a metric like
   // `node --test` would see NODE_TEST_CONTEXT, act as a child reporter, and report a bogus
   // exit code (so the gate can't be trusted when PromptWheel itself runs under a test runner).
   const childEnv = { ...(env || process.env) }; delete childEnv.NODE_TEST_CONTEXT;
   try {
-    stdout = execSync(m.cmd, { cwd, env: childEnv, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: (m.timeoutSec ?? 300) * 1000 });
-  } catch (e) { status = e.status ?? null; code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; }
+    // maxBuffer: the default is 1 MB, and a suite that prints more than that was killed and read as failing. A real
+    // repo found it (tap prints every assertion when its output is a pipe): the check said "your tests fail" about
+    // a suite that passes.
+    stdout = execSync(m.cmd, { cwd, env: childEnv, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: (m.timeoutSec ?? 300) * 1000, maxBuffer: 1024 * 1024 * 1024 });
+  } catch (e) { status = e.status ?? null; code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; if (notes) notes.push(e.code === 'ENOBUFS' ? { cause: 'too-much-output', detail: 'the command printed more than 1 GB, so it was stopped before it finished' } : whyFailed(`${e.stderr || ''}\n${e.stdout || ''}`)); }
   if (codes) codes.push(status);
   return extract(stdout, code, m.extract, m.allowNonzero);
 }
@@ -186,11 +197,62 @@ function extract(stdout, code, mode, allowNonzero) {
 // Python run can put the measured ref FIRST on PYTHONPATH). Ecosystem knowledge lives in config,
 // not here, the engine stays ecosystem-agnostic.
 const linkDirsOf = (cfg) => (Array.isArray(cfg.linkDirs) ? cfg.linkDirs : (cfg.linkNodeModules === false ? [] : ['node_modules']));
-function bridgeEnv(repo, wt, cfg) {
-  for (const d of linkDirsOf(cfg)) {
-    if (existsSync(join(repo, d)) && !existsSync(join(wt, d))) {
-      try { symlinkSync(join(repo, d), join(wt, d)); } catch { /* best effort */ }
+// Every folder that has a tracked package.json and a node_modules of its own: the workspaces of a monorepo, whichever
+// tool laid them out. Asked of git, so nothing walks the tree.
+function workspaceModuleDirs(repo) {
+  let files = '';
+  try { files = execFileSync('git', ['ls-files', '-z', '--', '*/package.json'], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 64 * 1024 * 1024 }); } catch { return []; }
+  const out = [];
+  for (const f of files.split('\0')) {
+    if (!f || f.split('/').includes('node_modules')) continue;
+    const d = join(dirname(f), 'node_modules');
+    if (existsSync(join(repo, d))) out.push(d);
+    if (out.length >= 500) break;
+  }
+  return out;
+}
+// A node_modules folder in a workspaces repo holds links back into the repo: npm and yarn link each workspace package
+// from the root, pnpm from the workspace that depends on it. One symlink to the real folder would make a test in the
+// worktree import a sibling package from the working tree, at both refs, so a regression there would never show. Such
+// a folder is rebuilt in the worktree instead: each entry linked on its own, and an entry that points back into the
+// repo pointed at the same path in the worktree. A folder with no such entry, in a repo pnpm did not lay out, keeps
+// the single link it always had.
+function linkModules(repo, wt, d) {
+  const src = join(repo, d), dst = join(wt, d);
+  const entries = [];
+  try {
+    for (const e of readdirSync(src, { withFileTypes: true })) {
+      if (e.name.startsWith('@') && e.isDirectory()) { for (const s of readdirSync(join(src, e.name), { withFileTypes: true })) entries.push([join(e.name, s.name), s]); }
+      else entries.push([e.name, e]);
     }
+  } catch { return; }
+  const back = (name, e) => {
+    if (!e.isSymbolicLink()) return null;
+    try { const r = relative(repo, realpathSync(join(src, name))); return r && !r.startsWith('..') && !isAbsolute(r) && !r.split(sep).includes('node_modules') ? r : null; } catch { return null; }
+  };
+  const links = entries.map(([name, e]) => [name, back(name, e), e]);
+  // pnpm refuses a node_modules, or its own state folder inside it, that is a symlink: its scripts stop at once
+  const pnpm = entries.some(([name]) => name === '.pnpm' || name === '.modules.yaml');
+  try {
+    if (!pnpm && !links.some(([, rel]) => rel)) { symlinkSync(src, dst); return; }
+    mkdirSync(dst, { recursive: true });
+    for (const [name, rel, e] of links) {
+      // a tool's own state or cache folder (pnpm's task-run state, .cache, .vite) is written to, so each worktree gets
+      // its own, made by the tool. The store (.pnpm) and the executables (.bin) are read, so they are linked.
+      if (name.startsWith('.') && name !== '.bin' && name !== '.pnpm' && e.isDirectory()) continue;
+      if (name.includes(sep)) mkdirSync(dirname(join(dst, name)), { recursive: true });
+      symlinkSync(rel ? join(wt, rel) : join(src, name), join(dst, name));
+    }
+  } catch { /* best effort */ }
+}
+function bridgeEnv(repo, wt, cfg) {
+  let root = repo; try { root = realpathSync(repo); } catch { /* keep the path as given */ }
+  // with nothing declared, a Node repo links its root node_modules and each workspace's own
+  const dirs = (Array.isArray(cfg.linkDirs) || cfg.linkNodeModules === false) ? linkDirsOf(cfg) : ['node_modules', ...workspaceModuleDirs(root)];
+  for (const d of dirs) {
+    if (!existsSync(join(root, d)) || existsSync(join(wt, d)) || !existsSync(dirname(join(wt, d)))) continue;
+    if (basename(d) === 'node_modules') linkModules(root, wt, d);
+    else { try { symlinkSync(join(root, d), join(wt, d)); } catch { /* best effort */ } }
   }
   if (!cfg.env) return process.env;
   return { ...process.env, ...Object.fromEntries(Object.entries(cfg.env).map(([k, v]) => [k, String(v).replaceAll('{wt}', () => wt)])) }; // fn replacement: a literal wt avoids $-pattern interpretation
@@ -198,10 +260,16 @@ function bridgeEnv(repo, wt, cfg) {
 // an optional per-ref build/install (e.g. `npm run build`, `pip install -e .`). Best-effort: a
 // failed setup just leaves the metric inert → INCONCLUSIVE (never a false green). Run by the
 // caller AFTER any source patch, so it builds the tree actually being measured.
+// Returns nothing when setup worked, or { exit, lastLine } when it did not, so the report can say so.
 function runSetup(wt, cfg, env) {
-  if (!cfg.setup) return;
-  try { execSync(cfg.setup, { cwd: wt, env, stdio: 'ignore', timeout: 300_000 }); } catch { /* inert → inconclusive */ }
+  if (!cfg.setup) return undefined;
+  try { execSync(cfg.setup, { cwd: wt, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 300_000, maxBuffer: 1024 * 1024 * 1024 }); return undefined; }
+  catch (e) {
+    const lines = `${e.stdout || ''}\n${e.stderr || ''}`.replace(/\x1b\[[0-9;]*m/g, '').split('\n').map((l) => l.trim()).filter(Boolean);
+    return { exit: e.status ?? null, lastLine: (lines[lines.length - 1] || '').slice(0, 200) };   // the metric goes inert → inconclusive
+  }
 }
+const setupFails = new WeakMap();   // a measureAt result -> how its setup failed
 
 // measure every metric `repeat` times at a ref, in a throwaway worktree (never touches your tree)
 function measureAt(repo, ref, cfg, repeat) {
@@ -209,12 +277,14 @@ function measureAt(repo, ref, cfg, repeat) {
   git(['worktree', 'add', '--quiet', '--detach', wt, ref], repo);
   try {
     const env = bridgeEnv(repo, wt, cfg);
-    runSetup(wt, cfg, env);
+    const setupFailed = runSetup(wt, cfg, env);
     const out = {};
+    if (setupFailed) setupFails.set(out, setupFailed);
     for (const m of cfg.metrics) {
-      const samples = [], codes = [];
-      for (let i = 0; i < repeat; i++) samples.push(runMetric(wt, m, env, codes));
+      const samples = [], codes = [], notes = [];
+      for (let i = 0; i < repeat; i++) samples.push(runMetric(wt, m, env, codes, notes));
       exitCodes.set(samples, codes);
+      if (notes.length) failNotes.set(samples, notes[notes.length - 1]);
       out[m.name] = samples;
     }
     return out;
@@ -883,6 +953,8 @@ function gate(repo, opts) {
     if (ev.inert && m.cmd !== PLACEHOLDER_TEST) {
       const nb = neverRan(before[m.name]), na = neverRan(after[m.name]);
       if (!nb || !na) ev.ran = !nb && !na ? 'both' : nb ? 'head' : 'base';
+      const note = failNotes.get(after[m.name]) || failNotes.get(before[m.name]);   // a hint: a missing dependency or tests that fail
+      if (note) Object.assign(ev, note);
     }
     return { name: m.name, direction: m.direction || 'up', guard: !!m.guard, ...(gamingCheckOff(m, baseGate) ? { tripwire: true } : {}), ...ev };
   });
@@ -954,6 +1026,9 @@ function gate(repo, opts) {
     ...(gateInfo ? { gate: gateInfo } : {}),
     ...(allowed ? { allowed } : {}),
   };
+  // a setup command that failed, at the ref it failed at: the usual reason a metric could not be measured
+  const sb = setupFails.get(before), sa = setupFails.get(after);
+  if (sb || sa) report.setup = { ...(sb ? { base: sb } : {}), ...(sa ? { head: sa } : {}) };
   // provenance receipt: makes the verdict tamper-evident + reproducible ("re-run and get the same
   // verdict is checkable"). Additive to the record; never changes the verdict.
   report.provenance = {
@@ -988,7 +1063,7 @@ const PRESETS = {
   'antihack': { desc: 'catch reward-hacking: a target + tripwires; pairs with `run --detect-gaming` (source-only re-run)',
     metrics: [
       { name: 'tests_pass', cmd: '__TESTCMD__', extract: 'exit', direction: 'pass', guard: true },
-      { name: 'test_count', cmd: 'grep -rIoE "\\b(it|test|describe) ?\\(|def test_" --exclude-dir=node_modules --exclude-dir=.git --exclude-dir=dist --exclude-dir=build --exclude-dir=coverage --exclude-dir=.next --exclude-dir=out --exclude-dir=lib-cov --exclude-dir=__pycache__ --exclude-dir=.pytest_cache --exclude-dir=target --exclude-dir=.venv --exclude-dir=venv --exclude=promptwheel.config.json . 2>/dev/null | wc -l | tr -d " "', extract: 'number', direction: 'up', guard: true, gamingCheck: false },
+      { name: 'test_count', cmd: 'grep -rIoE "\\b(it|test|describe) ?\\(|def test_" --exclude-dir=node_modules --exclude-dir=.promptwheel --exclude-dir=.git --exclude-dir=dist --exclude-dir=build --exclude-dir=coverage --exclude-dir=.next --exclude-dir=out --exclude-dir=lib-cov --exclude-dir=__pycache__ --exclude-dir=.pytest_cache --exclude-dir=target --exclude-dir=.venv --exclude-dir=venv --exclude=promptwheel.config.json . 2>/dev/null | wc -l | tr -d " "', extract: 'number', direction: 'up', guard: true, gamingCheck: false },
       { name: 'skipped_tests', cmd: 'grep -rIoE "\\.(skip|only) ?\\(|xit ?\\(|@pytest\\.mark\\.skip" --exclude-dir=node_modules --exclude-dir=.promptwheel --exclude-dir=.git --exclude-dir=dist --exclude-dir=build --exclude-dir=coverage --exclude-dir=.next --exclude-dir=out --exclude-dir=lib-cov --exclude-dir=__pycache__ --exclude-dir=.pytest_cache --exclude-dir=target --exclude-dir=.venv --exclude-dir=venv --exclude=promptwheel.config.json . 2>/dev/null | wc -l | tr -d " "', extract: 'number', direction: 'down', guard: true, gamingCheck: false },
       { name: 'suppressions', cmd: 'grep -rIoE "eslint-disable|@ts-(ignore|nocheck)|# ?type: ?ignore|# ?noqa|# ?nosec|//nolint|#!?\\[allow\\(" --exclude-dir=node_modules --exclude-dir=.promptwheel --exclude-dir=.git --exclude-dir=dist --exclude-dir=build --exclude-dir=coverage --exclude-dir=.next --exclude-dir=out --exclude-dir=lib-cov --exclude-dir=__pycache__ --exclude-dir=.pytest_cache --exclude-dir=target --exclude-dir=.venv --exclude-dir=venv --exclude=promptwheel.config.json . 2>/dev/null | wc -l | tr -d " "', extract: 'number', direction: 'down', guard: true, gamingCheck: false },
       { name: 'assertions', cmd: 'grep -rIoE "expect ?\\(|\\bassert|\\bt\\.(is|deepEqual|throws|truthy|falsy|not|ok) ?\\(" --exclude-dir=node_modules --exclude-dir=.promptwheel --exclude-dir=.git --exclude-dir=dist --exclude-dir=build --exclude-dir=coverage --exclude-dir=.next --exclude-dir=out --exclude-dir=lib-cov --exclude-dir=__pycache__ --exclude-dir=.pytest_cache --exclude-dir=target --exclude-dir=.venv --exclude-dir=venv --exclude=promptwheel.config.json . 2>/dev/null | wc -l | tr -d " "', extract: 'number', direction: 'up', guard: true, gamingCheck: false },
@@ -1179,10 +1254,12 @@ function printHuman(r) {
     const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
     const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
     console.log(`  ${arrowFor(m)} ${m.name.padEnd(18)} ${String(m.before).padStart(8)} → ${String(m.after).padStart(8)}  (${d}, ${m.status}) [${tag}, ${m.confidence}]`);
-    if (m.inert && m.ran === 'both') console.log(`      ⚠ "${m.name}" ran and failed before and after this change, so it guards nothing here. Fix the failing tests, or the command, before trusting a pass`);
+    if (m.inert && m.ran && m.cause === 'missing-dependency') console.log(`      ⚠ "${m.name}" could not run in the clean copy at either ref: ${m.detail}. Link or install what it needs (linkDirs or setup in promptwheel.config.json), then check again`);
+    else if (m.inert && m.ran === 'both') console.log(`      ⚠ "${m.name}" ran and failed before and after this change, so it guards nothing here. Fix the failing tests, or the command, before trusting a pass`);
     else if (m.inert) console.log(`      ⚠ "${m.name}" ${m.ran ? `ran and failed at the ${m.ran} ref and did not run at the other` : 'never ran (same value at both refs)'}: its command needs a build/deps first, or is not your real test command. Fix: promptwheel init, or set setup/linkDirs in promptwheel.config.json`);
     if (m.gamed === true) console.log(`      🚩 GAMED, ${m.gamingReason}`);
   }
+  if (r.setup) { const at = Object.keys(r.setup), f = r.setup[at[0]]; console.log(`  ⚠ setup failed (exit ${f.exit}) at the ${at.join(' and ')} ref${at.length > 1 ? 's' : ''}${f.lastLine ? ': ' + f.lastLine : ''}`); }
   if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(w.change === 'measurement-edited'
     ? `  🚩 GATE WEAKENED, how the guard "${w.name}" is measured (setup, env or linked dirs) was edited in this change, and measured the way it was before it would have ${wouldHave(w.status)} at head (goalpost moved, not earned)`
     : `  🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : w.change === 'guard-edited' ? 'edited' : 'disabled'} in this change and it would have ${wouldHave(w.status)} at head (goalpost moved, not earned)`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptwheel",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "description": "A deterministic, zero-LLM check for your coding loop. A win must survive a re-run from the agent's source edits alone, so a test that went green by editing the test reads GAMED. Runs in CI or a Claude Code plugin.",
   "type": "module",
   "bin": {