promptwheel 0.5.1 → 0.7.0

package/bin/promptwheel.mjs

@@ -75,6 +75,18 @@ function loadConfig(repo) {
   return cfg;
 }
 
+// does the repo have a committed gate config?
+const hasConfigFile = (repo) => ['promptwheel.config.json', 'outcome-gate.config.json'].some((f) => existsSync(join(repo, f)));
+
+// the zero-config default: the gate `init` would write, built in memory from stack detection.
+// It gates the detected test command (gaming detection on) with the stack's env/linkDirs so the
+// measurement actually runs in a throwaway worktree. Lets `run` and `check()` work with no config
+// file; `init` is for customizing. A repo with no test command gets a sentinel that reads inert ->
+// INCONCLUSIVE (never a fake green), with a message pointing at the fix.
+function autoConfig(repo) {
+  return { ...detectEnv(repo), metrics: [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }] };
+}
+
 function resolveBase(repo, base) {
   if (base) return base;
   for (const b of ['origin/main', 'origin/master', 'main', 'master']) {
@@ -460,6 +472,20 @@ function gateWeakenings(baseGate, headMetrics) {
   return weak;
 }
 
+// A declared, auditable override (CONTRACT Section 3). The head commit can acknowledge an
+// intentional test/config/grader change - an edit, a delete, or a new file - or an intentional
+// gate change, with a trailer:   Sawbuck-Allow: <reason>
+// so a change that would read GAMED reads PASS instead, with the acknowledgment RECORDED in
+// report.allowed. The check does not judge intent; it forces intent to be explicit. An UNdeclared
+// test-only win is still gamed. `--allow "<reason>"` (CLI) / `allow` (check) supplies the same for
+// a working-tree or CI run that has no commit message to carry the trailer.
+function readAllowTrailer(repo, ref) {
+  let msg = '';
+  try { msg = execFileSync('git', ['log', '-1', '--format=%B', ref], { cwd: repo, encoding: 'utf8' }); } catch { return null; }
+  const m = msg.match(/^[ \t]*(?:Sawbuck-Allow|Allow-Gamed|Promptwheel-Allow)[ \t]*:[ \t]*(\S.*?)[ \t]*$/im);
+  return m && m[1].trim() ? { reason: m[1].trim().slice(0, 300) } : null;
+}
+
 function gate(repo, opts) {
   selfHeal(repo);
   warnEditableInstall(repo);
@@ -521,8 +547,21 @@ function gate(repo, opts) {
       ...(weakened.length ? { weakened } : {}),
     };
   }
+  // declared override: a Sawbuck-Allow trailer (or --allow) acknowledges an intentional
+  // goalpost/gate change, so a would-be GAMED becomes PASS, recorded in report.allowed for review.
+  const allow = opts.allow ? { reason: String(opts.allow).slice(0, 300) } : (opts.working ? null : readAllowTrailer(repo, head));
+  let allowed = null;
+  if (opts.detectGaming && allow) {
+    const gm = metrics.filter((m) => m.gamed === true);
+    const gw = !!(gateInfo && gateInfo.weakened);
+    if (gm.length || gw) {
+      allowed = { reason: allow.reason, cleared: [...gm.map((m) => m.name), ...(gw ? ['gate'] : [])] };
+      for (const m of gm) { m.gamed = false; m.allowed = true; }
+      if (gw) gateInfo.allowed = true;
+    }
+  }
   const failed = metrics.some((m) => m.guard && !m.ok);
-  const gamed = metrics.some((m) => m.gamed === true) || !!(gateInfo && gateInfo.weakened);
+  const gamed = metrics.some((m) => m.gamed === true) || !!(gateInfo && gateInfo.weakened && !gateInfo.allowed);
   // a guard that never runs (inert: a pass/fail metric stuck at 0 across both refs) verifies
   // nothing, don't launder that into a green PASS. The honest verdict is "couldn't measure".
   // BUT scope it to a bare no-op run: an inert guard must NOT bury a genuine, measured win
@@ -538,6 +577,7 @@ function gate(repo, opts) {
     subsystems: subsystemsOf(repo, base, head, cfg),
     verdict, metrics,
     ...(gateInfo ? { gate: gateInfo } : {}),
+    ...(allowed ? { allowed } : {}),
   };
   if (!opts.noRecord && cfg.record !== false) recordOutcome(repo, report);
   return report;
@@ -546,7 +586,12 @@ function gate(repo, opts) {
 function run(argv) {
   const args = parseArgs(argv);
   const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
-  const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, label: args.label });
+  let config;
+  if (!hasConfigFile(repo) && !args.noAuto) {
+    config = autoConfig(repo);
+    console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd}  (promptwheel init to customize; --no-auto to require a config)`);
+  }
+  const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, label: args.label, allow: args.allow, config });
   if (args.json) console.log(JSON.stringify(report, null, 2));
   else if (args.markdown) console.log(renderMarkdown(report));
   else printHuman(report);
@@ -841,6 +886,20 @@ function detectTestCmd(repo) {
 
 // deps that live outside git + the env a test needs, defaulted per stack. init writes these into
 // config; the measurement bridge (bridgeEnv/runSetup) consumes them, keeps the engine agnostic.
+// a per-ref build/install so the MEASURED worktree is actually runnable. Closes the "inert" class
+// where the detected test command cannot run without a build first (a 10-repo replay put this at
+// ~2/3 of gates). Written by `init` into `setup`; a failed setup stays honest (inert -> inconclusive,
+// never a fake green). Kept out of the zero-config default so the fast per-turn path stays fast.
+function detectSetup(repo) {
+  const has = (f) => existsSync(join(repo, f));
+  try {
+    const pkg = JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8'));
+    if (pkg.scripts && pkg.scripts.build) return 'npm run build --if-present';   // TS/bundler tests that run against a built dir
+  } catch { /* no or unparsable package.json */ }
+  if (has('pyproject.toml') || has('setup.py')) return 'pip install -e . --quiet 2>/dev/null || true';   // editable install so imports resolve the measured ref
+  return undefined;
+}
+
 function detectEnv(repo) {
   const has = (f) => existsSync(join(repo, f));
   // Python: link the venv (3rd-party deps) and put the worktree src FIRST on PYTHONPATH, so imports
@@ -906,7 +965,8 @@ function init(argv) {
     if (lint) metrics = [...metrics, { name: 'lint_errors', cmd: LINT_CMD, extract: 'number', direction: 'down', guard: false }];
     note = `tests + antihack tripwires${lint ? ' + lint' : ''} (detected: ${testCmd})`;
   }
-  const envCfg = presetName ? {} : detectEnv(repo);
+  const setup = presetName ? undefined : detectSetup(repo);
+  const envCfg = presetName ? {} : { ...detectEnv(repo), ...(setup ? { setup } : {}) };
   if (envCfg.linkDirs) note += ` · linking ${envCfg.linkDirs.join(', ')}`;
   writeFileSync(out, JSON.stringify({ repeat: 1, ...envCfg, ...(evaluatorPaths ? { evaluatorPaths } : {}), metrics }, null, 2) + '\n');
   console.log(`✓ wrote promptwheel.config.json, ${note}`);
@@ -951,12 +1011,13 @@ function printHuman(r) {
     const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
     const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
     console.log(`  ${arrowFor(m)} ${m.name.padEnd(18)} ${String(m.before).padStart(8)} → ${String(m.after).padStart(8)}  (${d}, ${m.status}) [${tag}, ${m.confidence}]`);
-    if (m.inert) console.log('      ⚠ never passed at either ref, this guard is protecting nothing (check its command)');
+    if (m.inert) console.log(`      ⚠ "${m.name}" never ran (same value at both refs): its command needs a build/deps first, or is not your real test command. Fix: promptwheel init, or set setup/linkDirs in promptwheel.config.json`);
     if (m.gamed === true) console.log(`      🚩 GAMED, ${m.gamingReason}`);
   }
   if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(`  🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : 'disabled'} in this change and it would have ${w.status} at head (goalpost moved, not earned)`);
   console.log(`\n  VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? ', a guarded metric regressed (beyond noise)' : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
   if (r.verdict === 'fail') console.log('  intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name), `promptwheel guards` shows the effective set');
+  if (r.allowed) console.log(`  ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Sawbuck-Allow: ${r.allowed.reason})`);
   console.log('');
 }
 
@@ -993,6 +1054,8 @@ function parseArgs(argv) {
     else if (argv[i] === '--repeat') a.repeat = parseInt(argv[++i], 10);
     else if (argv[i] === '--working') a.working = true;
     else if (argv[i] === '--no-record') a.noRecord = true;
+    else if (argv[i] === '--no-auto') a.noAuto = true;
+    else if (argv[i] === '--allow') a.allow = argv[++i];
     else if (argv[i] === '--detect-gaming' || argv[i] === '--antihack') { a.detectGaming = true; a.dgExplicit = true; }
     else if (argv[i] === '--no-detect-gaming' || argv[i] === '--no-antihack') { a.detectGaming = false; a.dgExplicit = true; }
     else if (argv[i] === '--since') a.since = argv[++i];
@@ -1046,7 +1109,8 @@ export function verdictExitCode(verdict) {
 //   head          string   head ref (default: HEAD)
 //   working       boolean  gate uncommitted changes instead of base..head
 //   config        object   a resolved config { metrics:[...], ... }; omit to load
-//                          promptwheel.config.json from the repo
+//                          promptwheel.config.json, or auto-detect a default when none exists
+//   auto          boolean  allow the zero-config auto-detect fallback (default true; false throws if no config)
 //   repeat        number   measurement repeats for the noise band (default: config.repeat ?? 1)
 //   detectGaming  boolean  run the Section 3 source-only re-proof (default: true)
 //   cohort        string   cohort tag for the record (default: CI ? 'ci' : 'local')
@@ -1059,23 +1123,25 @@ export function check(options = {}) {
                         : git(['rev-parse', '--show-toplevel'], process.cwd());
   } catch { throw new Error(`promptwheel check: not inside a git repository (${options.repo || process.cwd()})`); }
 
-  if (options.config !== undefined) {
-    const c = options.config;
-    if (!c || typeof c !== 'object' || !Array.isArray(c.metrics) || c.metrics.length === 0)
+  let config = options.config;
+  if (config !== undefined) {
+    if (!config || typeof config !== 'object' || !Array.isArray(config.metrics) || config.metrics.length === 0)
       throw new Error('promptwheel check: options.config.metrics must be a non-empty array');
-  } else if (!['promptwheel.config.json', 'outcome-gate.config.json'].some((f) => existsSync(join(repo, f)))) {
-    throw new Error(`promptwheel check: no promptwheel.config.json in ${repo} (pass options.config to supply one in process)`);
+  } else if (!hasConfigFile(repo)) {
+    if (options.auto === false) throw new Error(`promptwheel check: no promptwheel.config.json in ${repo} (pass options.config, or allow auto-detect)`);
+    config = autoConfig(repo);   // zero-config: gate the detected test command in process
   }
 
   return gate(repo, {
     base: options.base,
     head: options.head,
     working: options.working,
-    config: options.config,
+    config,
     repeat: options.repeat,
     detectGaming: options.detectGaming ?? true,
     cohort: options.cohort,
     label: options.label,
+    allow: options.allow,
     noRecord: options.record !== true,
   });
 }
@@ -1117,6 +1183,7 @@ export function conformsToContract(report) {
     if (m.noise !== undefined && !(isNum(m.noise) && m.noise >= 0)) E(`${at}.noise must be a number >= 0 (the measured jitter band; repeat count is the top-level report.repeat)`);
     if (m.confidence !== undefined && typeof m.confidence !== 'string') E(`${at}.confidence must be a string`);
     if (m.gamed !== undefined && typeof m.gamed !== 'boolean') E(`${at}.gamed must be a boolean`);
+    if (m.allowed !== undefined && typeof m.allowed !== 'boolean') E(`${at}.allowed must be a boolean`);
     if (m.sourceOnly !== undefined && !numOrNull(m.sourceOnly)) E(`${at}.sourceOnly must be a number or null`);
     if (m.retained !== undefined && !numOrNull(m.retained)) E(`${at}.retained must be a number or null`);
     if (m.gamingReason !== undefined && typeof m.gamingReason !== 'string') E(`${at}.gamingReason must be a string`);
@@ -1132,12 +1199,21 @@ export function conformsToContract(report) {
     }
   }
 
+  // the optional declared-override record (report.allowed): a reason and the names it cleared.
+  if (report.allowed !== undefined) {
+    if (!report.allowed || typeof report.allowed !== 'object' || Array.isArray(report.allowed)) E('allowed must be an object { reason, cleared }');
+    else {
+      if (typeof report.allowed.reason !== 'string' || !report.allowed.reason) E('allowed.reason must be a non-empty string');
+      if (report.allowed.cleared !== undefined && !Array.isArray(report.allowed.cleared)) E('allowed.cleared must be an array');
+    }
+  }
+
   // Honest-verdict rules that hold regardless of how a verdict is derived:
   // a clean pass cannot hide a failing guard, a gamed win, or a weakened gate; and a gamed
   // verdict must point at a gamed metric or a weakened gate. (Fail-vs-gamed precedence is
   // left to the implementation.)
   if (VERDICTS.includes(report.verdict)) {
-    const gateWeakened = !!(report.gate && Array.isArray(report.gate.weakened) && report.gate.weakened.length);
+    const gateWeakened = !!(report.gate && Array.isArray(report.gate.weakened) && report.gate.weakened.length && !report.gate.allowed);
     if (report.verdict === 'pass' && report.metrics.some((m) => m.guard === true && m.ok === false))
       E("verdict 'pass' but a guarded metric regressed (guard:true, ok:false)");
     if (report.verdict === 'pass' && report.metrics.some((m) => m.gamed === true))
@@ -1212,11 +1288,12 @@ function main() {
     console.log([
       'PromptWheel, catch your agent cheating. Prove a change moved a real metric (and that the agent earned it, not gamed it). The per-turn reward + source-only audit for AI coding loops.',
       '',
-      '  promptwheel init [--preset <name> | --list]  write a starter config for your stack',
+      '  promptwheel init [--preset <name> | --list]  write a starter config (OPTIONAL: run works zero-config, auto-detecting your test command)',
       '  promptwheel run [--base R] [--head R] [--repeat N] [--json|--markdown]',
       '       catches reward-hacking BY DEFAULT, re-proves each win from SOURCE edits alone (verdict GAMED,',
       '       exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
       '  promptwheel run --working                    gate uncommitted changes (incl. newly added files)',
+      '  promptwheel run --allow "<reason>"           acknowledge an intentional test/config change (or a `Sawbuck-Allow: <reason>` commit trailer): recorded, not flagged GAMED',
       '  promptwheel improve --attempt "<cmd>"        run an agent/script; keep only if a metric improved',
       '                                               exit 0=kept · 1=regression · 3=plateau · add --json',
       '  promptwheel insights                         which metrics actually respond (raw counts)',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptwheel",
-  "version": "0.5.1",
+  "version": "0.7.0",
   "description": "A deterministic, zero-LLM check for your coding loop. It proves every change earned its win by re-running the result from the agent's source edits alone, so a green that came from editing the test reads GAMED. Runs in CI or a Claude Code hook, or import { check } in process.",
   "type": "module",
   "bin": {