promptwheel 0.6.0 → 0.7.1

package/README.md

@@ -75,6 +75,8 @@ npx promptwheel run                       # detection ON by default · exit 0 wi
 
 Deterministic, zero-LLM, zero-network: an LLM judge asking "did you cheat?" is itself gameable; this is a diff partition plus a re-run, so a flag is trustworthy without a human in the loop. The 50%-of-gain-survives threshold is the default and is tunable via `gamingThreshold` (config-level, or per-metric).
 
+**An intentional goalpost change has a sanctioned path.** When you *do* mean to change a test, config, or grader, declare it: a `Sawbuck-Allow: <reason>` trailer on the commit (or `--allow "<reason>"` on the CLI, or the `allow` option on `check()`) turns what would read `GAMED` into a `pass` with the reason recorded in `report.allowed`. An *undeclared* test-only win is still `gamed`, and a reasonless declaration does not override; the check never judges intent, it just forces intent to be explicit and auditable (the reason lives in the commit, so a reviewer sees it instead of a silent green).
+
 This is **one layer**, not a silver bullet: it catches the evaluator-tampering class (test/grader/golden/config edits) deterministically and for free, so the expensive layers, held-out tests for semantically-weak wins, an LLM judge or a human for intent and leakage, are reserved for the calls only they can make. See **[docs/DETECTION-LAYERS.md](docs/DETECTION-LAYERS.md)** for the coverage matrix and honest scope, and **[bench/RESULTS.md](bench/RESULTS.md)** for the measured numbers (`node bench/gaming-bench.mjs` to reproduce, it includes a cross-stack table with a real pytest run + a numeric eval-pass-rate metric). **Gate your own stack**, pytest · tsc · coverage · bundle · llm-eval, copy a config from **[examples/](examples/)**.
 
 ## Use
@@ -250,5 +252,8 @@ The engine is one importable file; pure helpers are exported for unit tests, the
 - [x] `--detect-gaming`, reward-hack detection: re-prove the win from source edits alone + `antihack` preset
 - [x] npm publish, `promptwheel@0.4.2` (the lead magnet)
 - [x] outcome-curated learning + UCB work-discovery, `playbook` + `suggest` + the compounding A/B harness (**experimental**; compounding *claims* stay gated on real-data proof, see [docs/LEARNING.md](docs/LEARNING.md))
+- [x] gate integrity: a win that disables a guard or deletes a guarded metric in the same diff reads `GAMED` (SHA-256 gate fingerprint; 0.5.0)
+- [x] zero-config: `run` and `check()` auto-detect the gate with no committed config, `init` becomes optional (0.6.0)
+- [x] declared override (`Sawbuck-Allow:` trailer / `--allow`) + `init` build-setup detection, so a build-gated test measures instead of reading inert (0.7.0)
 
-> Status: **published** (npm `promptwheel`, v0.4.2), all core phases built. Lineage: CommandLayer → BlockSpool → PromptWheel (orchestrator, archived) → **PromptWheel (outcome gate)**.
+> Status: **published** (npm `promptwheel`, v0.7.0), all core phases built. Lineage: CommandLayer → BlockSpool → PromptWheel (orchestrator, archived) → **PromptWheel (outcome gate)**.

package/bin/promptwheel.mjs

@@ -472,6 +472,20 @@ function gateWeakenings(baseGate, headMetrics) {
   return weak;
 }
 
+// A declared, auditable override (CONTRACT Section 3). The head commit can acknowledge an
+// intentional test/config/grader change - an edit, a delete, or a new file - or an intentional
+// gate change, with a trailer:   Sawbuck-Allow: <reason>
+// so a change that would read GAMED reads PASS instead, with the acknowledgment RECORDED in
+// report.allowed. The check does not judge intent; it forces intent to be explicit. An UNdeclared
+// test-only win is still gamed. `--allow "<reason>"` (CLI) / `allow` (check) supplies the same for
+// a working-tree or CI run that has no commit message to carry the trailer.
+function readAllowTrailer(repo, ref) {
+  let msg = '';
+  try { msg = execFileSync('git', ['log', '-1', '--format=%B', ref], { cwd: repo, encoding: 'utf8' }); } catch { return null; }
+  const m = msg.match(/^[ \t]*(?:Sawbuck-Allow|Allow-Gamed|Promptwheel-Allow)[ \t]*:[ \t]*(\S.*?)[ \t]*$/im);
+  return m && m[1].trim() ? { reason: m[1].trim().slice(0, 300) } : null;
+}
+
 function gate(repo, opts) {
   selfHeal(repo);
   warnEditableInstall(repo);
@@ -533,8 +547,21 @@ function gate(repo, opts) {
       ...(weakened.length ? { weakened } : {}),
     };
   }
+  // declared override: a Sawbuck-Allow trailer (or --allow) acknowledges an intentional
+  // goalpost/gate change, so a would-be GAMED becomes PASS, recorded in report.allowed for review.
+  const allow = opts.allow ? { reason: String(opts.allow).slice(0, 300) } : (opts.working ? null : readAllowTrailer(repo, head));
+  let allowed = null;
+  if (opts.detectGaming && allow) {
+    const gm = metrics.filter((m) => m.gamed === true);
+    const gw = !!(gateInfo && gateInfo.weakened);
+    if (gm.length || gw) {
+      allowed = { reason: allow.reason, cleared: [...gm.map((m) => m.name), ...(gw ? ['gate'] : [])] };
+      for (const m of gm) { m.gamed = false; m.allowed = true; }
+      if (gw) gateInfo.allowed = true;
+    }
+  }
   const failed = metrics.some((m) => m.guard && !m.ok);
-  const gamed = metrics.some((m) => m.gamed === true) || !!(gateInfo && gateInfo.weakened);
+  const gamed = metrics.some((m) => m.gamed === true) || !!(gateInfo && gateInfo.weakened && !gateInfo.allowed);
   // a guard that never runs (inert: a pass/fail metric stuck at 0 across both refs) verifies
   // nothing, don't launder that into a green PASS. The honest verdict is "couldn't measure".
   // BUT scope it to a bare no-op run: an inert guard must NOT bury a genuine, measured win
@@ -550,6 +577,7 @@ function gate(repo, opts) {
     subsystems: subsystemsOf(repo, base, head, cfg),
     verdict, metrics,
     ...(gateInfo ? { gate: gateInfo } : {}),
+    ...(allowed ? { allowed } : {}),
   };
   if (!opts.noRecord && cfg.record !== false) recordOutcome(repo, report);
   return report;
@@ -563,7 +591,7 @@ function run(argv) {
     config = autoConfig(repo);
     console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd}  (promptwheel init to customize; --no-auto to require a config)`);
   }
-  const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, label: args.label, config });
+  const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, label: args.label, allow: args.allow, config });
   if (args.json) console.log(JSON.stringify(report, null, 2));
   else if (args.markdown) console.log(renderMarkdown(report));
   else printHuman(report);
@@ -858,6 +886,20 @@ function detectTestCmd(repo) {
 
 // deps that live outside git + the env a test needs, defaulted per stack. init writes these into
 // config; the measurement bridge (bridgeEnv/runSetup) consumes them, keeps the engine agnostic.
+// a per-ref build/install so the MEASURED worktree is actually runnable. Closes the "inert" class
+// where the detected test command cannot run without a build first (a 10-repo replay put this at
+// ~2/3 of gates). Written by `init` into `setup`; a failed setup stays honest (inert -> inconclusive,
+// never a fake green). Kept out of the zero-config default so the fast per-turn path stays fast.
+function detectSetup(repo) {
+  const has = (f) => existsSync(join(repo, f));
+  try {
+    const pkg = JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8'));
+    if (pkg.scripts && pkg.scripts.build) return 'npm run build --if-present';   // TS/bundler tests that run against a built dir
+  } catch { /* no or unparsable package.json */ }
+  if (has('pyproject.toml') || has('setup.py')) return 'pip install -e . --quiet 2>/dev/null || true';   // editable install so imports resolve the measured ref
+  return undefined;
+}
+
 function detectEnv(repo) {
   const has = (f) => existsSync(join(repo, f));
   // Python: link the venv (3rd-party deps) and put the worktree src FIRST on PYTHONPATH, so imports
@@ -923,7 +965,8 @@ function init(argv) {
     if (lint) metrics = [...metrics, { name: 'lint_errors', cmd: LINT_CMD, extract: 'number', direction: 'down', guard: false }];
     note = `tests + antihack tripwires${lint ? ' + lint' : ''} (detected: ${testCmd})`;
   }
-  const envCfg = presetName ? {} : detectEnv(repo);
+  const setup = presetName ? undefined : detectSetup(repo);
+  const envCfg = presetName ? {} : { ...detectEnv(repo), ...(setup ? { setup } : {}) };
   if (envCfg.linkDirs) note += ` · linking ${envCfg.linkDirs.join(', ')}`;
   writeFileSync(out, JSON.stringify({ repeat: 1, ...envCfg, ...(evaluatorPaths ? { evaluatorPaths } : {}), metrics }, null, 2) + '\n');
   console.log(`✓ wrote promptwheel.config.json, ${note}`);
@@ -974,6 +1017,7 @@ function printHuman(r) {
   if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(`  🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : 'disabled'} in this change and it would have ${w.status} at head (goalpost moved, not earned)`);
   console.log(`\n  VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? ', a guarded metric regressed (beyond noise)' : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
   if (r.verdict === 'fail') console.log('  intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name), `promptwheel guards` shows the effective set');
+  if (r.allowed) console.log(`  ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Sawbuck-Allow: ${r.allowed.reason})`);
   console.log('');
 }
 
@@ -1011,6 +1055,7 @@ function parseArgs(argv) {
     else if (argv[i] === '--working') a.working = true;
     else if (argv[i] === '--no-record') a.noRecord = true;
     else if (argv[i] === '--no-auto') a.noAuto = true;
+    else if (argv[i] === '--allow') a.allow = argv[++i];
     else if (argv[i] === '--detect-gaming' || argv[i] === '--antihack') { a.detectGaming = true; a.dgExplicit = true; }
     else if (argv[i] === '--no-detect-gaming' || argv[i] === '--no-antihack') { a.detectGaming = false; a.dgExplicit = true; }
     else if (argv[i] === '--since') a.since = argv[++i];
@@ -1096,6 +1141,7 @@ export function check(options = {}) {
     detectGaming: options.detectGaming ?? true,
     cohort: options.cohort,
     label: options.label,
+    allow: options.allow,
     noRecord: options.record !== true,
   });
 }
@@ -1137,6 +1183,7 @@ export function conformsToContract(report) {
     if (m.noise !== undefined && !(isNum(m.noise) && m.noise >= 0)) E(`${at}.noise must be a number >= 0 (the measured jitter band; repeat count is the top-level report.repeat)`);
     if (m.confidence !== undefined && typeof m.confidence !== 'string') E(`${at}.confidence must be a string`);
     if (m.gamed !== undefined && typeof m.gamed !== 'boolean') E(`${at}.gamed must be a boolean`);
+    if (m.allowed !== undefined && typeof m.allowed !== 'boolean') E(`${at}.allowed must be a boolean`);
     if (m.sourceOnly !== undefined && !numOrNull(m.sourceOnly)) E(`${at}.sourceOnly must be a number or null`);
     if (m.retained !== undefined && !numOrNull(m.retained)) E(`${at}.retained must be a number or null`);
     if (m.gamingReason !== undefined && typeof m.gamingReason !== 'string') E(`${at}.gamingReason must be a string`);
@@ -1152,12 +1199,21 @@ export function conformsToContract(report) {
     }
   }
 
+  // the optional declared-override record (report.allowed): a reason and the names it cleared.
+  if (report.allowed !== undefined) {
+    if (!report.allowed || typeof report.allowed !== 'object' || Array.isArray(report.allowed)) E('allowed must be an object { reason, cleared }');
+    else {
+      if (typeof report.allowed.reason !== 'string' || !report.allowed.reason) E('allowed.reason must be a non-empty string');
+      if (report.allowed.cleared !== undefined && !Array.isArray(report.allowed.cleared)) E('allowed.cleared must be an array');
+    }
+  }
+
   // Honest-verdict rules that hold regardless of how a verdict is derived:
   // a clean pass cannot hide a failing guard, a gamed win, or a weakened gate; and a gamed
   // verdict must point at a gamed metric or a weakened gate. (Fail-vs-gamed precedence is
   // left to the implementation.)
   if (VERDICTS.includes(report.verdict)) {
-    const gateWeakened = !!(report.gate && Array.isArray(report.gate.weakened) && report.gate.weakened.length);
+    const gateWeakened = !!(report.gate && Array.isArray(report.gate.weakened) && report.gate.weakened.length && !report.gate.allowed);
     if (report.verdict === 'pass' && report.metrics.some((m) => m.guard === true && m.ok === false))
       E("verdict 'pass' but a guarded metric regressed (guard:true, ok:false)");
     if (report.verdict === 'pass' && report.metrics.some((m) => m.gamed === true))
@@ -1237,6 +1293,7 @@ function main() {
       '       catches reward-hacking BY DEFAULT, re-proves each win from SOURCE edits alone (verdict GAMED,',
       '       exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
       '  promptwheel run --working                    gate uncommitted changes (incl. newly added files)',
+      '  promptwheel run --allow "<reason>"           acknowledge an intentional test/config change (or a `Sawbuck-Allow: <reason>` commit trailer): recorded, not flagged GAMED',
       '  promptwheel improve --attempt "<cmd>"        run an agent/script; keep only if a metric improved',
       '                                               exit 0=kept · 1=regression · 3=plateau · add --json',
       '  promptwheel insights                         which metrics actually respond (raw counts)',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptwheel",
-  "version": "0.6.0",
+  "version": "0.7.1",
   "description": "A deterministic, zero-LLM check for your coding loop. It proves every change earned its win by re-running the result from the agent's source edits alone, so a green that came from editing the test reads GAMED. Runs in CI or a Claude Code hook, or import { check } in process.",
   "type": "module",
   "bin": {
@@ -50,4 +50,4 @@
     "detect-gaming",
     "reward-hack-detection"
   ]
-}
+}

package/packs/security/scan.mjs

@@ -5,7 +5,7 @@
 // The full AST+dataflow engine (the other 75 recommended patterns) is the follow-up.
 //
 // Usage: node scan.mjs [path]            → prints the finding count (for `promptwheel` metrics)
-//        node scan.mjs [path] --list     → prints each finding (file:line — id — message)
+//        node scan.mjs [path] --list     → prints each finding (file:line · id · message)
 import { readFileSync, readdirSync, statSync } from 'node:fs';
 import { join, extname } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -50,7 +50,7 @@ for (const file of walk(ROOT)) {
 }
 
 if (LIST) {
-  for (const f of findings) console.error(`  [${f.sev}] ${f.file}:${f.line} — ${f.id} — ${f.msg}`);
+  for (const f of findings) console.error(`  [${f.sev}] ${f.file}:${f.line} · ${f.id} · ${f.msg}`);
   console.error(`\n${findings.length} security-invariant violation(s)`);
 }
 // the metric value: lower is better, 0 is clean