promptwheel 0.7.16 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,7 +14,7 @@
14
14
 
15
15
  import { execSync, execFileSync } from 'node:child_process';
16
16
  import { createHash } from 'node:crypto';
17
- import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, appendFileSync, realpathSync, readdirSync, statSync, copyFileSync } from 'node:fs';
17
+ import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, realpathSync, readdirSync, lstatSync, copyFileSync } from 'node:fs';
18
18
  import { tmpdir } from 'node:os';
19
19
  import { join, dirname, isAbsolute, relative } from 'node:path';
20
20
  import { fileURLToPath } from 'node:url';
@@ -36,6 +36,9 @@ const spread = (xs) => { const a = xs.filter((x) => x != null); return a.length
36
36
  // repo is still measuring in (a long gate elsewhere must not lose its worktree mid-run)
37
37
  function tmpOwnerTag(repo) { let r = repo; try { r = realpathSync(repo); } catch { /* keep as given */ } return createHash('sha1').update(String(r)).digest('hex').slice(0, 8); }
38
38
  const tmpPrefix = (repo, kind) => `${kind}-${tmpOwnerTag(repo)}-`;
39
+ // exactly the names mkdtemp makes from tmpPrefix: the kind, the 8-hex owner tag, then mkdtemp's 6 random
40
+ // characters. Self-heal removes nothing else, so a folder that only starts with "promptwheel-" is never ours.
41
+ const ENGINE_TMP = /^(promptwheel-src|promptwheel|pw-idx)-([0-9a-f]{8})-[A-Za-z0-9]{6}$/;
39
42
 
40
43
  // usage / config errors exit 64 (EX_USAGE), so they can never be confused with a verdict
41
44
  // (0 pass, 1 fail, 2 gamed, 3 inconclusive): in CI a config typo must not read as GAMED.
@@ -68,7 +71,7 @@ const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { retu
68
71
  // resolve a config + its `extends` chain into a flat, provenance-tagged metric list.
69
72
  // `extends` is a path (or array of paths) to base configs, relative to the config file:
70
73
  // a repo INHERITS their guardrails, and a local metric of the same name overrides the inherited one.
71
- function resolveConfig(p, repo, label, seen) {
74
+ function resolveConfig(p, repo, seen) {
72
75
  const abs = realpathSync(p);
73
76
  if (seen.has(abs)) bad(`config inheritance cycle at ${relOf(repo, p)}`);
74
77
  seen.add(abs);
@@ -80,16 +83,16 @@ function resolveConfig(p, repo, label, seen) {
80
83
  for (const ref of [].concat(cfg.extends || [])) {
81
84
  const bp = isAbsolute(ref) ? ref : join(dirname(p), ref);
82
85
  if (!existsSync(bp)) bad(`extends target not found: ${ref} (from ${relOf(repo, p)})`);
83
- const base = resolveConfig(bp, repo, relOf(repo, bp), seen);
86
+ const base = resolveConfig(bp, repo, seen);
84
87
  for (const m of base.metrics) byName.set(m.name, m);
85
88
  for (const md of (base.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
86
- for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'record', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (base[k] !== undefined) scalars[k] = base[k];
89
+ for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (base[k] !== undefined) scalars[k] = base[k];
87
90
  }
88
91
  for (const m of (cfg.metrics || [])) {
89
- byName.set(m.name, { ...m, __src: label, __override: byName.has(m.name) });
92
+ byName.set(m.name, m);
90
93
  }
91
- for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, { ...md, __src: label });
92
- for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'record', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (cfg[k] !== undefined) scalars[k] = cfg[k];
94
+ for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
95
+ for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (cfg[k] !== undefined) scalars[k] = cfg[k];
93
96
  return { ...scalars, metrics: [...byName.values()], mandates: [...mandatesByName.values()] };
94
97
  } finally {
95
98
  seen.delete(abs); // only ACTIVE ancestors are a cycle, a diamond (a base reached two ways) is fine
@@ -99,7 +102,7 @@ function resolveConfig(p, repo, label, seen) {
99
102
  function loadConfig(repo) {
100
103
  const p = join(repo, 'promptwheel.config.json');
101
104
  if (!existsSync(p)) bad('no promptwheel.config.json, run: promptwheel init (writes one for your stack)');
102
- const cfg = resolveConfig(p, repo, 'local', new Set());
105
+ const cfg = resolveConfig(p, repo, new Set());
103
106
  if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0) bad('config.metrics must be a non-empty array');
104
107
  return cfg;
105
108
  }
@@ -112,8 +115,14 @@ const hasConfigFile = (repo) => existsSync(join(repo, 'promptwheel.config.json')
112
115
  // measurement actually runs in a throwaway worktree. Lets `run` and `check()` work with no config
113
116
  // file; `init` is for customizing. A repo with no test command gets a sentinel that reads inert ->
114
117
  // INCONCLUSIVE (never a fake green), with a message pointing at the fix.
115
- function autoConfig(repo) {
116
- return { ...detectEnv(repo), metrics: [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }] };
118
+ function autoConfig(repo, at = stackAt(repo)) {
119
+ return { ...detectEnv(repo, at), metrics: [{ name: 'tests_pass', cmd: detectTestCmd(repo, at), extract: 'exit', direction: 'pass', guard: true }] };
120
+ }
121
+ // where stack detection looks: the working tree, or the tracked files as of `ref` (the repo as it was before a change)
122
+ function stackAt(repo, ref) {
123
+ if (!ref) return { has: (f) => existsSync(join(repo, f)), read: (f) => readFileSync(join(repo, f), 'utf8') };
124
+ const read = (f) => execFileSync('git', ['show', `${ref}:${f}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
125
+ return { has: (f) => { try { read(f); return true; } catch { return false; } }, read };
117
126
  }
118
127
 
119
128
  function resolveBase(repo, base) {
@@ -123,7 +132,8 @@ function resolveBase(repo, base) {
123
132
  catch { bad('promptwheel: no commits yet, make an initial commit before gating'); }
124
133
  for (const b of ['origin/main', 'origin/master', 'main', 'master']) {
125
134
  let mb = '';
126
- try { mb = git(['merge-base', 'HEAD', b], repo); } catch { continue; }
135
+ // stderr ignored: a ref that is not there (no origin/main in this clone) is expected, not an error to print
136
+ try { mb = execFileSync('git', ['merge-base', 'HEAD', b], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { continue; }
127
137
  // on the trunk itself the merge-base IS HEAD; comparing a commit to itself would be a
128
138
  // trivial PASS that verified nothing, so fall through to the last commit instead.
129
139
  if (mb && mb !== head) return mb;
@@ -132,15 +142,21 @@ function resolveBase(repo, base) {
132
142
  catch { bad('promptwheel: no commit to compare against (one commit, or a shallow clone): pass --base, or --working'); }
133
143
  }
134
144
 
135
- function runMetric(cwd, m, env) {
136
- let stdout = '', code = 0;
145
+ // exit statuses of a sample list, kept beside it: 127 (the shell found no such command), 126 (it could not
146
+ // execute it) or no status (killed: a timeout) mean the metric never ran, which needs a different fix from a
147
+ // test that ran and failed
148
+ const exitCodes = new WeakMap();
149
+ const neverRan = (samples) => { const c = exitCodes.get(samples) || []; return c.length > 0 && c.every((x) => x == null || x === 126 || x === 127); };
150
+ function runMetric(cwd, m, env, codes) {
151
+ let stdout = '', code = 0, status = 0;
137
152
  // Never leak the harness's own node:test context into the measured gate: a metric like
138
153
  // `node --test` would see NODE_TEST_CONTEXT, act as a child reporter, and report a bogus
139
154
  // exit code (so the gate can't be trusted when PromptWheel itself runs under a test runner).
140
155
  const childEnv = { ...(env || process.env) }; delete childEnv.NODE_TEST_CONTEXT;
141
156
  try {
142
157
  stdout = execSync(m.cmd, { cwd, env: childEnv, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: (m.timeoutSec ?? 300) * 1000 });
143
- } catch (e) { code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; }
158
+ } catch (e) { status = e.status ?? null; code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; }
159
+ if (codes) codes.push(status);
144
160
  return extract(stdout, code, m.extract, m.allowNonzero);
145
161
  }
146
162
 
@@ -169,9 +185,9 @@ function extract(stdout, code, mode, allowNonzero) {
169
185
  // with linkNodeModules:false → link nothing); `env` sets vars ({wt} → the worktree path, so a
170
186
  // Python run can put the measured ref FIRST on PYTHONPATH). Ecosystem knowledge lives in config,
171
187
  // not here, the engine stays ecosystem-agnostic.
188
+ const linkDirsOf = (cfg) => (Array.isArray(cfg.linkDirs) ? cfg.linkDirs : (cfg.linkNodeModules === false ? [] : ['node_modules']));
172
189
  function bridgeEnv(repo, wt, cfg) {
173
- const linkDirs = Array.isArray(cfg.linkDirs) ? cfg.linkDirs : (cfg.linkNodeModules === false ? [] : ['node_modules']);
174
- for (const d of linkDirs) {
190
+ for (const d of linkDirsOf(cfg)) {
175
191
  if (existsSync(join(repo, d)) && !existsSync(join(wt, d))) {
176
192
  try { symlinkSync(join(repo, d), join(wt, d)); } catch { /* best effort */ }
177
193
  }
@@ -196,8 +212,9 @@ function measureAt(repo, ref, cfg, repeat) {
196
212
  runSetup(wt, cfg, env);
197
213
  const out = {};
198
214
  for (const m of cfg.metrics) {
199
- const samples = [];
200
- for (let i = 0; i < repeat; i++) samples.push(runMetric(wt, m, env));
215
+ const samples = [], codes = [];
216
+ for (let i = 0; i < repeat; i++) samples.push(runMetric(wt, m, env, codes));
217
+ exitCodes.set(samples, codes);
201
218
  out[m.name] = samples;
202
219
  }
203
220
  return out;
@@ -266,9 +283,12 @@ const NON_SOURCE = [
266
283
  /_spec\.rb$/i,
267
284
  /[a-z0-9](Test|Tests)\.(java|kt|kts|scala)$/,
268
285
  /\.snap$/i, /(^|\/)golden[^/]*$/i, /\.golden$/i,
269
- // a grader/eval/score FILE by name (eval.ts, grader-x.js, score.test.mjs, scoring.ts), never a prefix:
270
- // evaluate.ts, evaluation.js, scoreboard.js are production source and must stay in the slice
271
- /(^|\/)(eval|evals|grader|graders|score|scorer|scoring)([._-][^/]*)?\.[cm]?[jt]s$/i,
286
+ // a grader/eval/score FILE by name in a scripting language (eval.py, grader-x.js, grade.sh, scoring.ts), never a
287
+ // prefix: evaluate.ts, evaluation.py, scoreboard.js stay source. Compiled names stay source too (eval.c, eval.go
288
+ // are interpreters; Score.java is a type), and so do .tsx/.vue files (a Score component, a /score page).
289
+ /(^|\/)(eval|evals|grade|grader|graders|score|scorer|scoring)([._-][^/]*)?\.([cm]?[jt]s|py|sh|bash|zsh|rb|pl|php|lua|r|jl)$/i,
290
+ // anything under an evals/ or grader(s)/ directory. Not eval/ (an interpreter's package) or scoring/ (a product module).
291
+ /(^|\/)(evals|graders?)\//i,
272
292
  // config files only, require a .config/.conf/.setup segment so PRODUCTION source that
273
293
  // happens to be named after a tool (e.g. src/installers/eslint.ts) is NOT swept out of
274
294
  // the source slice (a false sweep here can flag an honest win as GAMED).
@@ -279,7 +299,7 @@ const NON_SOURCE = [
279
299
  // promptwheel's OWN evaluator surface: the config that defines the gate
280
300
  // and any pack copied into the repo by `init --preset`. Editing these is
281
301
  // moving the goalposts, never a production improvement, an agent that
282
- // "improved" security_findings by deleting pattern rules must show 0%
302
+ // "improved" a scanner count by deleting its rules must show 0%
283
303
  // retained in the source-only re-run, not 100%.
284
304
  /(^|\/)promptwheel\.config\.json$/i,
285
305
  /(^|\/)\.promptwheel\//i,
@@ -311,9 +331,9 @@ function evaluatorMatcher(cfg) {
311
331
  // forbids ANY change to its paths (same glob semantics as evaluatorPaths). A hit fails the gate
312
332
  // regardless of metrics and rides at the top of the report; mandates inherit through `extends`,
313
333
  // so a parent scope's rule holds for every repo beneath it.
314
- // The mandates in force are the UNION of the head config's and the base ref's (`baseMandates`,
315
- // resolved by gateDefAtRef): deleting a mandate in the same change that violates it does not
316
- // clear it. A mandate removed at head but violated in the diff is reported with `removed: true`; one
334
+ // The mandates in force are the base ref's (`baseMandates`, resolved by gateDefAtRef), in their head
335
+ // version when both have them: deleting a mandate in the same change that violates it does not
336
+ // clear it, and one the change adds does not apply to it (see below). A mandate removed at head but violated in the diff is reported with `removed: true`; one
317
337
  // whose paths were edited at head (same name, different paths) is tested in BOTH versions and reported
318
338
  // with `edited: true`, so narrowing a rule in the change that breaks it does not clear it either.
319
339
  // The honest way to retire one is a separate change that only edits the config.
@@ -457,7 +477,7 @@ function judgeGaming(m, srcResult, threshold = 0.5) {
457
477
  // main
458
478
  // ---------------------------------------------------------------------------
459
479
  // snapshot the working tree (tracked + untracked) into a dangling commit WITHOUT
460
- // touching the real index or files, so a gate/improve sees a file an agent just added.
480
+ // touching the real index or files, so a gate sees a file an agent just added.
461
481
  // Returns 'HEAD' when nothing changed.
462
482
  function workingSnapshot(repo) {
463
483
  const idxDir = mkdtempSync(join(tmpdir(), tmpPrefix(repo, 'pw-idx')));
@@ -482,12 +502,16 @@ function workingSnapshot(repo) {
482
502
  }
483
503
  }
484
504
 
485
- // self-heal: a hard-killed run (SIGKILL / power loss) can leave an orphaned worktree,
505
+ // self-heal: a hard-killed run (SIGKILL / power loss) can leave an orphaned worktree,
486
506
  // its `finally` never ran. On the next run, drop stale worktree registry entries and any
487
- // abandoned /tmp checkout, so nothing accumulates in your repo or temp dir. Never touches a
488
- // live worktree (a still-registered dir, or this repo's checkout younger than 1h). A dir tagged
489
- // for ANOTHER repo may be mid-measurement, so it gets a full day; after that it is an orphan of a
490
- // repo that was deleted or moved (a new real path is a new tag nobody will ever claim).
507
+ // abandoned /tmp checkout, so nothing accumulates in your repo or temp dir. It only ever removes a
508
+ // directory whose name is exactly what this engine generates (ENGINE_TMP): a folder named
509
+ // promptwheel-notes, or another repo's worktree named promptwheel-site-wt, is not ours at any age.
510
+ // Ages, by dir mtime: this repo's unregistered checkout after 1h; this repo's still-registered worktree
511
+ // after a full day (a killed run leaves it registered, and a younger one may be a concurrent gate still
512
+ // measuring); a dir tagged for ANOTHER repo after a full day (an orphan of a repo that was deleted or
513
+ // moved: a new real path is a new tag nobody will ever claim). A registered worktree that is not
514
+ // tagged for this repo is never touched: it may be yours.
491
515
  function selfHeal(repo) {
492
516
  try {
493
517
  git(['worktree', 'prune'], repo); // remove registry entries whose dir is already gone (always safe)
@@ -496,16 +520,18 @@ function selfHeal(repo) {
496
520
  .split('\n').filter((l) => l.startsWith('worktree ')).map((l) => l.slice(9).trim()),
497
521
  );
498
522
  const tmp = tmpdir(), now = Date.now(), mine = tmpOwnerTag(repo);
499
- const TAGGED = /^(promptwheel-src|promptwheel|pw-idx)-([0-9a-f]{8})-/;
500
523
  for (const name of readdirSync(tmp)) {
501
- if (!name.startsWith('promptwheel-') && !name.startsWith('pw-idx-')) continue; // worktrees + temp-index dirs
502
- const tag = (name.match(TAGGED) || [])[2] || null;
503
- // ours after an hour; another repo's, or an untagged dir (an older engine's), only after a full day
504
- const maxAge = tag === mine ? 3600_000 : 86_400_000;
524
+ const made = ENGINE_TMP.exec(name);
525
+ if (!made) continue; // not a name this engine generates (worktrees + temp-index dirs): never removed
526
+ const tag = made[2];
527
+ // ours after an hour; another repo's only after a full day
505
528
  const p = join(tmp, name);
506
- if (tracked.has(p)) continue; // a live/registered worktree, leave it
507
- try { if (statSync(p).mtimeMs < now - maxAge) rmSync(p, { recursive: true, force: true }); } catch { /* */ }
529
+ if (tracked.has(p) && tag !== mine) continue; // a registered worktree this repo's gate did not make
530
+ const maxAge = tag === mine && !tracked.has(p) ? 3600_000 : 86_400_000;
531
+ // lstat: the engine makes directories, so a file or a symlink under one of its names is left alone
532
+ try { const st = lstatSync(p); if (st.isDirectory() && st.mtimeMs < now - maxAge) rmSync(p, { recursive: true, force: true }); } catch { /* */ }
508
533
  }
534
+ git(['worktree', 'prune'], repo); // drop the registry entries of anything removed above
509
535
  } catch { /* cleanup must never block the gate */ }
510
536
  }
511
537
 
@@ -537,18 +563,25 @@ function warnEditableInstall(repo) {
537
563
  // guarded metric in the very diff it "passes". We resolve the gate as of BASE too and, if a
538
564
  // guard was dropped, re-measure it under its BASE definition; the pass only reads GAMED when
539
565
  // that guard WOULD HAVE FAILED at head (a real regression hidden by moving the goalposts).
540
- const GATE_SCALARS = ['gamingThreshold', 'evaluatorPaths', 'allowTrailer'];
566
+ const GATE_SCALARS = ['gamingThreshold', 'evaluatorPaths', 'allowTrailer', 'setup', 'env', 'linkDirs', 'linkNodeModules'];
567
+ // what runs or shapes every measurement besides the metric's own command. A setup can rewrite a test and an
568
+ // env var can skip one, so these are part of the gate: fingerprinted, and held at their base values for the
569
+ // change that edits them (see detectGamingPass).
570
+ const measureOf = (s) => ({ setup: s.setup, env: s.env, linkDirs: s.linkDirs, linkNodeModules: s.linkNodeModules });
571
+ const measuredWith = (s) => JSON.stringify([s.setup ?? null, Object.entries(s.env || {}).map(([k, v]) => `${k}=${v}`).sort(), [...linkDirsOf(s)].sort()]);
541
572
 
542
573
  // Resolve the gate definition as of `ref` (metrics + mandates + scalars), following `extends` by
543
574
  // reading every file from that ref via `git show`, mirroring resolveConfig (a parent's mandates
544
575
  // hold beneath it; a local mandate of the same name overrides). Returns { metrics:Map, scalars }
545
- // with scalars.mandates, or null when there is no config at `ref` (a gate ADDED in this range is
546
- // not a weakening). Never exits: any missing/unreadable file or parse error yields null (that
547
- // branch simply contributes nothing).
576
+ // with scalars.mandates, or null when there is no config at `ref` (gate() then falls back to the
577
+ // zero-config gate the repo ran under, see zeroConfigGate). Never exits: any missing/unreadable file or
578
+ // parse error yields null (that branch simply contributes nothing), and `partial` says an extends target
579
+ // could not be read at `ref` (it lives outside the repo), so how the base measured is not fully known.
548
580
  function gateDefAtRef(repo, ref) {
549
581
  // quiet: a missing path makes `git show` exit nonzero and print "fatal:" to stderr; swallow it
550
582
  const readAt = (p) => { try { return execFileSync('git', ['show', `${ref}:${p}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return null; } };
551
583
  const seen = new Set();
584
+ let partial = false;
552
585
  const resolve = (p) => {
553
586
  if (seen.has(p)) return null;
554
587
  seen.add(p);
@@ -557,9 +590,11 @@ function gateDefAtRef(repo, ref) {
557
590
  let cfg; try { cfg = JSON.parse(raw); } catch { return null; }
558
591
  const byName = new Map(); const mandatesByName = new Map(); const scalars = {};
559
592
  for (const ext of [].concat(cfg.extends || [])) {
560
- if (isAbsolute(ext)) continue; // outside the repo, cannot git-show; unchanged anyway
561
- const base = resolve(join(dirname(p), ext).split('\\').join('/'));
562
- if (!base) continue;
593
+ if (isAbsolute(ext)) { partial = true; continue; } // outside the repo, cannot git-show; unchanged anyway
594
+ const bp = join(dirname(p), ext).split('\\').join('/');
595
+ const again = seen.has(bp); // a diamond: already merged, not a missing piece
596
+ const base = resolve(bp);
597
+ if (!base) { if (!again) partial = true; continue; }
563
598
  for (const m of base.metrics.values()) byName.set(m.name, m);
564
599
  for (const md of base.scalars.mandates) mandatesByName.set(md.name, md);
565
600
  for (const k of GATE_SCALARS) if (base.scalars[k] !== undefined) scalars[k] = base.scalars[k];
@@ -570,18 +605,35 @@ function gateDefAtRef(repo, ref) {
570
605
  scalars.mandates = [...mandatesByName.values()];
571
606
  return { metrics: byName, scalars };
572
607
  };
573
- return resolve('promptwheel.config.json');
608
+ const def = resolve('promptwheel.config.json');
609
+ return def && { ...def, partial };
574
610
  }
575
611
 
612
+ // The gate a repo with no config ran under: the detected test command (source-only re-run on) with the
613
+ // stack's env and linkDirs, detected from the tracked files as of `ref`. A change that adds the first
614
+ // config is judged against this, so `tests_pass.cmd: "true"` in a new config is an edited guard, not a
615
+ // fresh start. `initSetup` is the setup `init` would add: a first config that carries exactly it has not
616
+ // changed how the gate measures. With no test command to detect (the placeholder) the old gate measured
617
+ // nothing, so its setup is no baseline (`partial`).
618
+ function zeroConfigGate(repo, ref) {
619
+ const at = stackAt(repo, ref), auto = autoConfig(repo, at);
620
+ return { metrics: new Map(auto.metrics.map((m) => [m.name, m])), scalars: { mandates: [], ...measureOf(auto) }, zeroConfig: true, initSetup: detectSetup(repo, at), partial: auto.metrics[0].cmd === PLACEHOLDER_TEST };
621
+ }
622
+
623
+ // how a metric is measured and judged: change any of these on a guard and it is a different guard
624
+ // what init writes when it finds no test command. It guards nothing, so replacing it is setup, not a weakened gate.
625
+ const PLACEHOLDER_TEST = 'echo "set your test command in promptwheel.config.json" && false';
626
+ const measuredBy = (m) => JSON.stringify([m.cmd ?? null, m.extract ?? 'number', m.direction || 'up', m.exact === true, m.report === true]);
576
627
  // an order-independent SHA-256 (truncated) fingerprint of a gate definition, for the audit record
577
628
  function gateFingerprint(metricsList, scalars) {
578
629
  const norm = {
579
630
  metrics: metricsList.map((m) => ({
580
- name: m.name, guard: m.guard === true, direction: m.direction || 'up',
631
+ name: m.name, guard: m.guard === true, measuredBy: measuredBy(m),
581
632
  gamingThreshold: m.gamingThreshold ?? null, gamingCheck: m.gamingCheck !== false,
582
633
  })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
583
634
  gamingThreshold: scalars.gamingThreshold ?? null,
584
635
  evaluatorPaths: Array.isArray(scalars.evaluatorPaths) ? [...scalars.evaluatorPaths].sort() : [],
636
+ measuredWith: measuredWith(scalars),
585
637
  mandates: Array.isArray(scalars.mandates) ? scalars.mandates.map((m) => ({ name: String(m.name || ''), paths: [...(Array.isArray(m.paths) ? m.paths : [])].sort() })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) : [],
586
638
  };
587
639
  return createHash('sha256').update(JSON.stringify(norm)).digest('hex').slice(0, 16);
@@ -626,7 +678,8 @@ function verdictFingerprint(report) {
626
678
  return createHash('sha256').update(JSON.stringify(norm)).digest('hex').slice(0, 16);
627
679
  }
628
680
 
629
- // guarded metrics that HEAD dropped relative to BASE (removed entirely, or guard turned off)
681
+ // guarded metrics that HEAD dropped relative to BASE (removed entirely, or guard turned off), or
682
+ // measures another way (a new cmd, extract, direction, exact or report flag): `cmd: "true"` is a dropped guard too
630
683
  function gateWeakenings(baseGate, headMetrics) {
631
684
  if (!baseGate) return [];
632
685
  const head = new Map(headMetrics.map((m) => [m.name, m]));
@@ -636,6 +689,7 @@ function gateWeakenings(baseGate, headMetrics) {
636
689
  const hm = head.get(bm.name);
637
690
  if (!hm) weak.push({ name: bm.name, change: 'guard-removed', def: bm });
638
691
  else if (hm.guard !== true) weak.push({ name: bm.name, change: 'guard-disabled', def: bm });
692
+ else if (!bm.report && bm.cmd !== PLACEHOLDER_TEST && measuredBy(hm) !== measuredBy(bm)) weak.push({ name: bm.name, change: 'guard-edited', def: bm });
639
693
  }
640
694
  return weak;
641
695
  }
@@ -654,29 +708,51 @@ function readAllowTrailer(repo, ref) {
654
708
  return m && m[1].trim() ? { reason: m[1].trim().slice(0, 300) } : null;
655
709
  }
656
710
 
711
+ // A change is judged by the gate that was in force before it (`baseGate`: the config at base, or the
712
+ // zero-config gate when this change adds the first config). So a change cannot switch off the source-only
713
+ // re-run for a metric that had it, loosen gamingThreshold, or swap the setup, env or linked dirs it is
714
+ // measured with: the base values hold for the change that edits them, and apply from the next one.
715
+ const gamingCheckOff = (cm, baseGate) => {
716
+ if (!cm || cm.gamingCheck !== false) return false;
717
+ const bm = baseGate && baseGate.metrics.get(cm.name);
718
+ return !bm || bm.gamingCheck === false; // off before this change too, or a metric this change adds
719
+ };
720
+ function gamingThresholdFor(cm, cfg, baseGate) {
721
+ const now = cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5;
722
+ const bm = baseGate && baseGate.metrics.get(cm.name);
723
+ const before = bm ? (bm.gamingThreshold ?? baseGate.scalars.gamingThreshold ?? (baseGate.partial ? now : 0.5)) : now;
724
+ return Math.max(now, before); // the stricter of before and after
725
+ }
726
+
657
727
  // The three Section 3 checks, run only with detectGaming, in order: (1) re-prove every actual
658
728
  // win with the agent's source edits alone; (2) content-hash auto-fail: a declared evaluatorPath
659
729
  // gate file edited in this diff is a moved goalpost, so a metric that improved alongside it is
660
730
  // gamed by declaration, deterministic even if the source-only re-run read inert; (3) gate
661
- // integrity: resolve the gate as of BASE, re-measure any guard HEAD dropped under its base
662
- // definition (it only counts as tampering if it WOULD HAVE FAILED at head), and fingerprint the
663
- // gate definition at both refs (metrics, evaluatorPaths, gamingThreshold, mandates) so a quietly
664
- // deleted guard or mandate is tamper-evident: base != head is recorded as gate.changed and
731
+ // integrity: take the gate as of BASE, re-measure any guard HEAD dropped, edited or measures with
732
+ // another setup, env or linked dirs, under its base definition (it only counts as tampering if it
733
+ // WOULD HAVE FAILED at head), and fingerprint the gate definition at both refs (metrics,
734
+ // evaluatorPaths, gamingThreshold, mandates, setup, env, linked dirs) so a quietly deleted guard
735
+ // or mandate is tamper-evident: base != head is recorded as gate.changed and
665
736
  // printed, whatever the verdict. Mutates `metrics` in place (gamed/sourceOnly/retained/
666
737
  // gamingReason) and returns the report.gate record. An allow trailer can still clear a gamed
667
738
  // metric afterwards, in gate().
668
- function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
739
+ function detectGamingPass(repo, base, head, cfg, metrics, repeat, baseGate) {
740
+ // how the base measured. Unknown (`partial`) reads as unchanged; so does a first config that adds only the setup `init` detects
741
+ const bs = baseGate ? (baseGate.partial ? { ...baseGate.scalars, ...measureOf(cfg) } : baseGate.scalars) : null;
742
+ const asInit = !!bs && baseGate.zeroConfig && measuredWith({ ...bs, setup: baseGate.initSetup }) === measuredWith(cfg);
743
+ const measureChanged = !!bs && !asInit && measuredWith(bs) !== measuredWith(cfg);
744
+ const baseRun = measureChanged ? { ...cfg, ...measureOf(bs) } : cfg; // the head gate, run with the base setup, env and linked dirs
669
745
  const toProve = [];
670
746
  for (const m of metrics) {
671
747
  if (m.status !== 'improved') continue;
672
748
  const cm = cfg.metrics.find((c) => c.name === m.name);
673
- if (cm.gamingCheck === false) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
749
+ if (gamingCheckOff(cm, baseGate)) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
674
750
  toProve.push(cm);
675
751
  }
676
- const srcResults = measureSourceOnlyAll(repo, base, head, toProve, cfg, repeat); // one patched checkout for all of them
752
+ const srcResults = measureSourceOnlyAll(repo, base, head, toProve, baseRun, repeat); // one patched checkout for all of them
677
753
  for (const cm of toProve) {
678
754
  const m = metrics.find((x) => x.name === cm.name);
679
- const j = judgeGaming(m, srcResults[cm.name], cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5);
755
+ const j = judgeGaming(m, srcResults[cm.name], gamingThresholdFor(cm, cfg, baseGate));
680
756
  m.gamed = j.gamed; m.sourceOnly = j.sourceOnly; m.retained = j.retained; m.gamingReason = j.reason;
681
757
  if (j.unprovable) m.unprovable = true;
682
758
  }
@@ -687,32 +763,55 @@ function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
687
763
  if (split.source.length && split.nonSource.length) {
688
764
  const held = metrics.filter((m) => m.status === 'unchanged' && m.direction === 'pass' && m.after === 1 && !m.tripwire);
689
765
  const defs = held.map((m) => cfg.metrics.find((c) => c.name === m.name)).filter(Boolean);
690
- const res = measureSourceOnlyAll(repo, base, head, defs, cfg, repeat);
766
+ const res = measureSourceOnlyAll(repo, base, head, defs, baseRun, repeat);
691
767
  for (const m of held) { const r = res[m.name]; if (r && r.samples && r.samples.length && median(r.samples) !== 1) m.heldByTestEdit = true; }
692
768
  }
693
- const evalChanged = changedEvaluatorPaths(repo, base, head, cfg);
769
+ // a file declared as a goalpost before this change still is one for this change: dropping the declaration in the
770
+ // same diff that edits the file does not make the edit count as source
771
+ const declared = [...new Set([...(cfg.evaluatorPaths || []), ...((baseGate && baseGate.scalars.evaluatorPaths) || [])])];
772
+ const evalChanged = changedEvaluatorPaths(repo, base, head, { ...cfg, evaluatorPaths: declared });
694
773
  if (evalChanged.length) {
695
774
  for (const m of metrics) {
696
775
  if (m.status !== 'improved' || m.gamed === true) continue;
697
776
  const cm = cfg.metrics.find((c) => c.name === m.name);
698
- if (cm && cm.gamingCheck === false) continue; // tripwire/test-side guards legitimately live in gate files
777
+ if (gamingCheckOff(cm, baseGate)) continue; // tripwire/test-side guards legitimately live in gate files
699
778
  m.gamed = true;
700
779
  m.gamingReason = `a declared evaluatorPath gate file changed in this diff (${evalChanged.map((e) => e.path).join(', ')}); a win alongside a goalpost edit is gamed by declaration`;
701
780
  }
702
781
  }
703
- const baseGate = gateDefAtRef(repo, base);
704
782
  const weak = gateWeakenings(baseGate, cfg.metrics);
783
+ // the setup, env or linked dirs changed: every guard the base had is measured again the way it was measured before
784
+ if (measureChanged) for (const bm of baseGate.metrics.values()) {
785
+ if (bm.guard === true && !bm.report && bm.cmd !== PLACEHOLDER_TEST && !weak.some((w) => w.name === bm.name)) weak.push({ name: bm.name, change: 'measurement-edited', def: bm });
786
+ }
705
787
  const weakened = [];
706
788
  if (weak.length) {
707
789
  const defs = weak.map((w) => w.def);
708
- const b = measureAt(repo, base, { ...cfg, metrics: defs }, repeat);
709
- const a = measureAt(repo, head, { ...cfg, metrics: defs }, repeat);
790
+ const b = measureAt(repo, base, { ...baseRun, metrics: defs }, repeat);
791
+ const a = measureAt(repo, head, { ...baseRun, metrics: defs }, repeat);
792
+ const evs = new Map();
710
793
  for (const w of weak) {
711
794
  const ev = evaluate(w.def, b[w.name], a[w.name], repeat);
712
- if (!ev.ok) weakened.push({ name: w.name, change: w.change, status: ev.status });
795
+ evs.set(w.name, ev);
796
+ // an edited guard that still fails under its old definition was turned green by the edit, not the code
797
+ let hidden = !ev.ok || (w.change === 'guard-edited' && ev.inert);
798
+ // a guard measured another way counts only when the new measurement hides it: it fails the old way and reads fine the new way
799
+ if (w.change === 'measurement-edited') { const hm = metrics.find((x) => x.name === w.name); hidden = (!ev.ok || !!ev.inert) && !!hm && hm.ok && !hm.inert; }
800
+ if (hidden) weakened.push({ name: w.name, change: w.change, status: ev.inert ? 'failed' : ev.status });
801
+ }
802
+ // a win under the old definition still has to be earned in the source: swapping or renaming the guard in the same
803
+ // change does not skip the source-only re-run
804
+ const won = weak.filter((w) => w.def.gamingCheck !== false && !weakened.some((x) => x.name === w.name) && evs.get(w.name).status === 'improved');
805
+ if (won.length) {
806
+ const src = measureSourceOnlyAll(repo, base, head, won.map((w) => w.def), baseRun, repeat);
807
+ for (const w of won) {
808
+ const j = judgeGaming({ name: w.name, direction: w.def.direction || 'up', guard: true, ...evs.get(w.name) }, src[w.name], w.def.gamingThreshold ?? (bs && bs.gamingThreshold) ?? 0.5);
809
+ if (j.gamed) weakened.push({ name: w.name, change: w.change, status: 'gamed' });
810
+ }
713
811
  }
714
812
  }
715
- const fpBase = baseGate ? gateFingerprint([...baseGate.metrics.values()], baseGate.scalars) : null;
813
+ // the fingerprint is of a config file: `base` stays null when there was none there (the zero-config gate is not a file)
814
+ const fpBase = baseGate && !baseGate.zeroConfig ? gateFingerprint([...baseGate.metrics.values()], bs) : null;
716
815
  const fpHead = gateFingerprint(cfg.metrics, cfg);
717
816
  const info = {
718
817
  fingerprint: { base: fpBase, head: fpHead },
@@ -731,7 +830,7 @@ function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
731
830
  // (or a weakened gate) is GAMED; INCONCLUSIVE when nothing could be certified. A guard that never
732
831
  // runs (inert: a pass/fail metric stuck at 0 across both refs) verifies nothing, so a bare no-op
733
832
  // run is not laundered into a green PASS, BUT an inert guard must not bury a genuine, measured win
734
- // elsewhere (that would make `improve` revert good changes and misread them as a plateau). And a
833
+ // elsewhere (a loop reading the verdict would throw away a good change). And a
735
834
  // win whose source-only re-run could not be applied (`unprovable`) is never a PASS either: a green
736
835
  // the engine could not re-prove is exactly the green it exists to doubt.
737
836
  export function deriveVerdict(metrics, gateInfo = null, mandated = false) {
@@ -769,17 +868,26 @@ function gate(repo, opts) {
769
868
  // a guard this change adds has no value at base, because its scanner did not exist there. That is adoption,
770
869
  // not a failure: it starts counting from head. Removing a guard is still caught (gateWeakenings), and a guard the
771
870
  // base contract already had still fails when it cannot be measured. A zero-config run has no contract to adopt.
772
- const baseDefs = opts.config ? null : gateDefAtRef(repo, base);
871
+ const fileGate = gateDefAtRef(repo, base);
872
+ const baseDefs = opts.config ? null : fileGate;
773
873
  const adopted = (name) => !opts.config && !(baseDefs && baseDefs.metrics.has(name));
874
+ // the gate in force before this change: the config committed at base, or, when this change adds the
875
+ // first config, the zero-config gate the repo ran under until now
876
+ const baseGate = opts.detectGaming ? (fileGate || (opts.config ? null : zeroConfigGate(repo, base))) : null;
774
877
  const metrics = cfg.metrics.map((m) => {
775
878
  const ev = evaluate(m, before[m.name], after[m.name], repeat);
776
879
  if (ev.status === 'unmeasurable' && ev.before == null && ev.after != null && adopted(m.name)) { ev.ok = true; ev.adopted = true; }
777
880
  // a `report` guard tells you and does not fail: a contract can start by watching and tighten later
778
881
  if (m.report && !ev.ok && ev.status === 'regressed') { ev.ok = true; ev.noted = true; }
779
- return { name: m.name, direction: m.direction || 'up', guard: !!m.guard, ...(m.gamingCheck === false ? { tripwire: true } : {}), ...ev };
882
+ // an inert guard: did its command run and fail, or never run? `ran` says where it ran (absent: at neither ref)
883
+ if (ev.inert && m.cmd !== PLACEHOLDER_TEST) {
884
+ const nb = neverRan(before[m.name]), na = neverRan(after[m.name]);
885
+ if (!nb || !na) ev.ran = !nb && !na ? 'both' : nb ? 'head' : 'base';
886
+ }
887
+ return { name: m.name, direction: m.direction || 'up', guard: !!m.guard, ...(gamingCheckOff(m, baseGate) ? { tripwire: true } : {}), ...ev };
780
888
  });
781
889
  // Section 3 (gaming detection), on by default: mutates `metrics` and yields the gate record
782
- const gateInfo = opts.detectGaming ? detectGamingPass(repo, base, head, cfg, metrics, repeat) : null;
890
+ const gateInfo = opts.detectGaming ? detectGamingPass(repo, base, head, cfg, metrics, repeat, baseGate) : null;
783
891
  // declared override: a Promptwheel-Allow trailer (or --allow) acknowledges an intentional
784
892
  // goalpost/gate change, so a would-be GAMED becomes PASS, recorded in report.allowed for review.
785
893
  // It covers an `unprovable` win too (the source-only re-run could not be applied): the declared
@@ -820,7 +928,7 @@ function gate(repo, opts) {
820
928
  for (const m of ga) { m.ok = true; m.allowed = true; allowed.cleared.push(m.name); }
821
929
  } else for (const m of ga) m.allowable = true; // failed, and a declared reason would clear it: say so in the report
822
930
  }
823
- // mandates run first: a protected-path hard rule fails the gate no matter what the metrics say.
931
+ // mandates: a protected-path hard rule fails the gate no matter what the metrics say.
824
932
  // The base ref's mandates count too (resolved through `extends` by gateDefAtRef), so a change
825
933
  // cannot clear a mandate by deleting it in the same diff that violates it.
826
934
  const baseMandates = (gateInfo && gateInfo.baseMandates) || (!opts.detectGaming ? ((gateDefAtRef(repo, base) || {}).scalars || {}).mandates : null);
@@ -854,7 +962,6 @@ function gate(repo, opts) {
854
962
  configHash: gateFingerprint(cfg.metrics, cfg),
855
963
  verdictHash: verdictFingerprint(report),
856
964
  };
857
- if (!opts.noRecord && cfg.record !== false) recordOutcome(repo, report);
858
965
  return report;
859
966
  }
860
967
 
@@ -866,115 +973,18 @@ function run(argv) {
866
973
  config = autoConfig(repo);
867
974
  console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd} (promptwheel init to customize; --no-auto to require a config)`);
868
975
  }
869
- const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, allow: args.allow, config });
976
+ const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, detectGaming: args.detectGaming, allow: args.allow, config });
870
977
  if (args.json) console.log(JSON.stringify(report, null, 2));
871
978
  else if (args.markdown) console.log(renderMarkdown(report));
872
979
  else printHuman(report);
873
980
  process.exit(verdictExitCode(report.verdict));
874
981
  }
875
982
 
876
- // persisted record: append every gated run to a per-repo outcome record (best-effort, never fails the gate)
877
- function recordOutcome(repo, report) {
878
- try {
879
- const dir = join(repo, '.promptwheel');
880
- mkdirSync(dir, { recursive: true });
881
- appendFileSync(join(dir, 'outcomes.jsonl'), JSON.stringify({ ts: new Date().toISOString(), ...report }) + '\n');
882
- } catch { /* recording must never break the gate */ }
883
- }
884
-
885
- // keep-if-improved: run any agent/script, gate the result, keep the change ONLY if a metric improved
886
- function improve(argv) {
887
- const args = parseArgs(argv);
888
- if (!args.attempt) die('improve requires --attempt "<command that changes the repo>"');
889
- const repo = repoRoot(process.cwd());
890
- const dirty = git(['status', '--porcelain'], repo).split('\n').filter((l) => l.trim() && !l.includes('.promptwheel'));
891
- if (dirty.length) die('working tree not clean, commit or stash first (improve needs a clean base to revert to)');
892
-
893
- console.error(`▶ attempt: ${args.attempt}`);
894
- try { execSync(args.attempt, { cwd: repo, stdio: 'inherit' }); }
895
- catch (e) { console.error(` (attempt exited ${e.status ?? 1}, gating whatever it changed)`); }
896
-
897
- const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
898
- const noChange = report.metrics.every((m) => m.delta === 0 || m.delta == null);
899
- const improvedNames = report.metrics.filter((m) => m.status === 'improved').map((m) => m.name);
900
-
901
- // result + exit code express loop progress so `while improve; do :; done` converges, with the
902
- // same codes as `run`: 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed
903
- // (reverted) · 3 = plateau/no-op OR inconclusive (reverted)
904
- let result, exit, note;
905
- if (report.verdict === 'fail') { result = 'regression'; exit = 1; revert(repo); note = '✗ guarded regression, reverted'; }
906
- else if (report.verdict === 'gamed') { result = 'gamed'; exit = 2; revert(repo); note = '🚩 gamed, a metric "improved" by editing tests/config, not source, reverted'; }
907
- else if (report.verdict === 'inconclusive') { result = 'inconclusive'; exit = 3; revert(repo); note = '≈ inconclusive, a guard measured nothing (inert); reverted'; }
908
- else if (noChange || improvedNames.length === 0) {
909
- result = 'plateau'; exit = 3; revert(repo);
910
- note = noChange ? '= no metric moved, reverted' : '= nothing improved beyond noise, reverted';
911
- } else {
912
- git(['add', '-A'], repo);
913
- execFileSync('git', ['commit', '-qm', `promptwheel: ${args.attempt} [improved ${improvedNames.join(', ')}]`], { cwd: repo });
914
- result = 'kept'; exit = 0;
915
- note = `✓ kept, committed ${git(['rev-parse', '--short', 'HEAD'], repo)} (improved ${improvedNames.join(', ')})`;
916
- }
917
-
918
- // stdout carries the value (JSON report or the human table); the decision line goes to
919
- // stderr so a loop driver can consume clean stdout.
920
- if (args.json) console.log(JSON.stringify({ result, ...report }, null, 2));
921
- else printHuman(report);
922
- console.error(` ${note}`);
923
- process.exit(exit);
924
- }
925
-
926
- // discard the attempt's changes; keep the .promptwheel outcome record
927
- function revert(repo) {
928
- git(['reset', '--hard', 'HEAD'], repo);
929
- try { git(['clean', '-fd', '-e', '.promptwheel'], repo); } catch { /* best effort */ }
930
- }
931
-
932
- // insights: honest aggregation of the per-repo outcome record (.promptwheel/outcomes.jsonl)
933
- // into per-metric counts. No model, no decay, no claims: just which metrics actually respond.
934
- // The reader is tolerant of older records that carry extra fields (cohort, label, subsystems).
935
- function insights(argv) {
936
- const args = parseArgs(argv);
937
- const repo = repoRoot(process.cwd());
938
- const f = join(repo, '.promptwheel', 'outcomes.jsonl');
939
- if (!existsSync(f)) die('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)');
940
- const runs = readFileSync(f, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
941
- const agg = {};
942
- for (const r of runs) for (const m of (r.metrics || [])) {
943
- const a = agg[m.name] ??= { runs: 0, improved: 0, regressed: 0, inconclusive: 0, unchanged: 0, net: 0, last: null };
944
- a.runs++; if (a[m.status] != null) a[m.status]++;
945
- if (typeof m.delta === 'number') a.net = +(a.net + m.delta).toFixed(6);
946
- a.last = m.after;
947
- }
948
- // "lever score" = how reliably acting on this metric yields a real improvement
949
- const rows = Object.entries(agg).map(([name, a]) => ({ name, ...a, lever: a.runs ? a.improved / a.runs : 0 }))
950
- .sort((x, y) => y.lever - x.lever);
951
- if (args.json) { console.log(JSON.stringify({ runs: runs.length, metrics: rows }, null, 2)); return; }
952
- console.log(`\nPromptWheel insights, ${runs.length} gated runs recorded\n`);
953
- console.log(` ${'metric'.padEnd(18)} ${'runs'.padStart(5)} ${'impr'.padStart(5)} ${'regr'.padStart(5)} ${'inconc'.padStart(6)} ${'net Δ'.padStart(9)} lever`);
954
- for (const r of rows) {
955
- console.log(` ${r.name.padEnd(18)} ${String(r.runs).padStart(5)} ${String(r.improved).padStart(5)} ${String(r.regressed).padStart(5)} ${String(r.inconclusive).padStart(6)} ${String(r.net).padStart(9)} ${(r.lever * 100).toFixed(0)}%`);
956
- }
957
- console.log('\n lever = improved/runs, how reliably this metric actually responds. The');
958
- console.log(' highest-lever metrics are where an agent loop should spend its attempts.\n');
959
- }
960
-
961
983
  // ---------------------------------------------------------------------------
962
984
  // init, write a starter config so a newcomer isn't staring at a blank page
963
985
  // ---------------------------------------------------------------------------
964
986
  const LINT_CMD = 'npx eslint . -f unix 2>/dev/null | grep -c " error " || true';
965
987
  const PRESETS = {
966
- 'tests-pass': { desc: 'gate: your test suite still passes (command auto-detected)', metrics: null },
967
- 'lint': { desc: 'track: lint error count does not climb',
968
- metrics: [{ name: 'lint_errors', cmd: LINT_CMD, extract: 'number', direction: 'down', guard: false }] },
969
- 'bundle-size': { desc: 'track: build output size in kB',
970
- metrics: [{ name: 'bundle_kb', cmd: 'du -sk dist 2>/dev/null | cut -f1 || echo 0', extract: 'number', direction: 'down', guard: false }] },
971
- 'security': { desc: 'gate: security-invariant violations must not climb (CORS, secrets, cookies, payment; see packs/security)',
972
- metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }] },
973
- 'llm-eval': { desc: 'gate: AI-feature eval pass-rate + est $/run (see examples/reliability-sprint)',
974
- metrics: [
975
- { name: 'eval_pass_rate', cmd: 'node eval.mjs', extract: 'number', direction: 'up', guard: true },
976
- { name: 'cost_per_run_usd', cmd: 'node estimate-cost.mjs', extract: 'number', direction: 'down', guard: true },
977
- ] },
978
988
  'antihack': { desc: 'catch reward-hacking: a target + tripwires; pairs with `run --detect-gaming` (source-only re-run)',
979
989
  metrics: [
980
990
  { name: 'tests_pass', cmd: '__TESTCMD__', extract: 'exit', direction: 'pass', guard: true },
@@ -1004,7 +1014,7 @@ const PRESETS = {
1004
1014
  { name: 'codeowners-frozen', paths: ['CODEOWNERS'], reason: 'review ownership is set by people' },
1005
1015
  { name: 'scanner-frozen', paths: ['.promptwheel/packs/**'], reason: 'the scanners that read the contract are not edited by the change they read' },
1006
1016
  { name: 'lock-frozen', paths: ['promptwheel.lock.json'], allowable: 'flag', reason: 'the architecture lock changes with a reason from a person (--allow), never as a side effect of a change' },
1007
- { name: 'hooks-frozen', paths: ['.claude/settings.json', '.claude/settings.local.json', '.sawbuck/**'], reason: 'the hooks that enforce the contract are wired by a person, not detached by the agent' },
1017
+ { name: 'hooks-frozen', paths: ['.claude/settings.json', '.claude/settings.local.json', '.sawbuck/**'], allowable: 'flag', reason: 'the hooks that enforce the contract change with a reason from a person (--allow), and are never detached by the agent' },
1008
1018
  ],
1009
1019
  metrics: [
1010
1020
  { name: 'lifecycle_scripts', cmd: 'node packs/supply-chain/scan.mjs lifecycle .', extract: 'number', direction: 'down', guard: true, gamingCheck: false, allowable: 'flag' },
@@ -1017,32 +1027,17 @@ const PRESETS = {
1017
1027
  ] },
1018
1028
  };
1019
1029
 
1020
- // security-antihack = the security-invariant findings gate + the SAME antihack tripwires
1021
- // (so an agent that suppresses a security rule, # noqa / # nosec / eslint-disable, or
1022
- // skips a security test is caught), PLUS a starter `evaluatorPaths` that protects common
1023
- // security-gate config files (a ruleset or a budget the agent could otherwise loosen to
1024
- // pass). The scanner pack installs into `.promptwheel/` (already evaluator-protected). It
1025
- // is the "guard the guards" preset: pairs with --detect-gaming so a win that only moved by
1026
- // loosening the security gate, not by fixing the code, is GAMED. Blind spot (by design,
1027
- // not this tool's job): a source-resident backdoor that behaves safely only under test;
1028
- // that needs held-out/behavioral analysis, not a source-only re-run.
1029
- PRESETS['security-antihack'] = {
1030
- desc: 'gate security-invariant findings + antihack tripwires + evaluatorPaths so an agent cannot loosen the security gate to pass it (pairs with --detect-gaming)',
1031
- evaluatorPaths: ['.semgrep.yml', '.semgrep.yaml', '.bandit', 'bandit.yaml', '.ruff.toml', 'ruff.toml', '.snyk', 'security-budget.json'],
1032
- metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }, ...PRESETS.antihack.metrics.slice(1)],
1033
- };
1034
-
1035
- function detectTestCmd(repo) {
1036
- const has = (f) => existsSync(join(repo, f));
1030
+ function detectTestCmd(repo, { has, read } = stackAt(repo)) {
1037
1031
  if (has('go.mod')) return 'go test ./...';
1038
1032
  if (has('Cargo.toml')) return 'cargo test';
1039
- if (has('pyproject.toml') || has('setup.py') || has('pytest.ini')) return has('.venv/bin/pytest') ? '.venv/bin/pytest -q' : 'pytest -q';
1033
+ // the venv is not tracked: it is looked for on disk whichever ref the stack is read from
1034
+ if (has('pyproject.toml') || has('setup.py') || has('pytest.ini')) return existsSync(join(repo, '.venv/bin/pytest')) ? '.venv/bin/pytest -q' : 'pytest -q';
1040
1035
  // only trust `npm test` when a test script actually exists, otherwise the metric can
1041
1036
  // never pass and the gate would "protect" nothing (very common in Next.js app repos)
1042
1037
  try {
1043
- if (JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8')).scripts?.test) return 'npm test --silent';
1038
+ if (JSON.parse(read('package.json')).scripts?.test) return 'npm test --silent';
1044
1039
  } catch { /* no or unparsable package.json, fall through */ }
1045
- return 'echo "set your test command in promptwheel.config.json" && false';
1040
+ return PLACEHOLDER_TEST;
1046
1041
  }
1047
1042
 
1048
1043
  // deps that live outside git + the env a test needs, defaulted per stack. init writes these into
@@ -1051,18 +1046,16 @@ function detectTestCmd(repo) {
1051
1046
  // where the detected test command cannot run without a build first (a 10-repo replay put this at
1052
1047
  // ~2/3 of gates). Written by `init` into `setup`; a failed setup stays honest (inert -> inconclusive,
1053
1048
  // never a fake green). Kept out of the zero-config default so the fast per-turn path stays fast.
1054
- function detectSetup(repo) {
1055
- const has = (f) => existsSync(join(repo, f));
1049
+ function detectSetup(repo, { has, read } = stackAt(repo)) {
1056
1050
  try {
1057
- const pkg = JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8'));
1051
+ const pkg = JSON.parse(read('package.json'));
1058
1052
  if (pkg.scripts && pkg.scripts.build) return 'npm run build --if-present'; // TS/bundler tests that run against a built dir
1059
1053
  } catch { /* no or unparsable package.json */ }
1060
1054
  if (has('pyproject.toml') || has('setup.py')) return 'pip install -e . --quiet 2>/dev/null || true'; // editable install so imports resolve the measured ref
1061
1055
  return undefined;
1062
1056
  }
1063
1057
 
1064
- function detectEnv(repo) {
1065
- const has = (f) => existsSync(join(repo, f));
1058
+ function detectEnv(repo, { has } = stackAt(repo)) {
1066
1059
  // Python: link the venv (3rd-party deps) and put the worktree src FIRST on PYTHONPATH, so imports
1067
1060
  // resolve the MEASURED ref instead of an editable install pointing back at your original checkout.
1068
1061
  if (has('pyproject.toml') || has('setup.py') || has('pytest.ini')) return { linkDirs: ['.venv'], env: { PYTHONPATH: '{wt}/src:{wt}' } }; // src/ AND flat layouts; worktree src wins over an editable install
@@ -1104,10 +1097,9 @@ function init(argv) {
1104
1097
  if (presetName) {
1105
1098
  const p = PRESETS[presetName];
1106
1099
  if (!p) die(`unknown preset "${presetName}", try: promptwheel init --list`);
1107
- evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (e.g. security-antihack)
1100
+ evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (supply-chain: the contract)
1108
1101
  mandates = p.mandates; allowTrailer = p.allowTrailer; // a preset may ship hard rules (supply-chain: workflows, registries, publish scripts, hooks)
1109
- metrics = p.metrics || [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }];
1110
- metrics = metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
1102
+ metrics = p.metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
1111
1103
  // pack-backed presets: the referenced scripts live in THIS package, not
1112
1104
  // the user's repo, copy them in so the generated config is
1113
1105
  // self-contained and committable (a bare reference silently 404'd)
@@ -1123,8 +1115,7 @@ function init(argv) {
1123
1115
  const files = readdirSync(src);
1124
1116
  for (const f of files) copyFileSync(join(src, f), join(dst, f));
1125
1117
  console.log(` ✓ installed ${pk[2]} → .promptwheel/${pk[2]} (commit it with the config)`);
1126
- // the gate measures in clean worktrees, so a pack that git ignores (common: the outcome
1127
- // record lives in .promptwheel/ too) is absent at both refs and the metric reads
1118
+ // the gate measures in clean worktrees, so a pack that git ignores is absent at both refs and the metric reads
1128
1119
  // unmeasurable, failing the guard with no hint why. Say so now, while it is one edit.
1129
1120
  if (files.length && gitIgnores(repo, `.promptwheel/${pk[2]}/${files[0]}`))
1130
1121
  console.log(` ⚠ .promptwheel/${pk[2]} is gitignored: the gate measures in clean worktrees, so an ignored pack is absent at both refs and ${m.name} reads unmeasurable (the guard fails). Replace ".promptwheel/" in .gitignore with ".promptwheel/*" and "!.promptwheel/packs/", then commit the pack`);
@@ -1154,7 +1145,7 @@ function init(argv) {
1154
1145
  if (wrote.length) console.log(`✓ wrote the agent guide (when to run the gate) to ${wrote.join(', ')}`);
1155
1146
  }
1156
1147
  console.log('\n next: promptwheel run --working # gate your uncommitted changes');
1157
- console.log(' promptwheel init --list # other presets (llm-eval, bundle-size, …)');
1148
+ console.log(' promptwheel init --list # other presets (antihack, no-ai-tells, supply-chain)');
1158
1149
  }
1159
1150
 
1160
1151
  // Write (or refresh) a short agent-orienting block so the coding AGENT runs the gate itself, not just
@@ -1177,36 +1168,9 @@ function writeAgentBlock(repo) {
1177
1168
  return done;
1178
1169
  }
1179
1170
 
1180
- // observability: list the EFFECTIVE guardrails (incl. inherited) + each one's flag record
1181
- function guards(argv) {
1182
- const args = parseArgs(argv);
1183
- const repo = repoRoot(process.cwd());
1184
- const cfg = loadConfig(repo);
1185
- const hist = {};
1186
- const f = join(repo, '.promptwheel', 'outcomes.jsonl');
1187
- if (existsSync(f)) for (const line of readFileSync(f, 'utf8').split('\n').filter(Boolean)) {
1188
- let r; try { r = JSON.parse(line); } catch { continue; }
1189
- for (const m of (r.metrics || [])) {
1190
- const h = hist[m.name] ??= { runs: 0, flagged: 0, last: null };
1191
- h.runs++; if (m.guard && m.ok === false) h.flagged++; h.last = m.status;
1192
- }
1193
- }
1194
- const rows = cfg.metrics.map((m) => ({
1195
- name: m.name, direction: m.direction || 'up', guard: !!m.guard,
1196
- source: m.__src, override: !!m.__override, ...(hist[m.name] || { runs: 0, flagged: 0, last: null }),
1197
- }));
1198
- if (args.json) { console.log(JSON.stringify({ guards: rows }, null, 2)); return; }
1199
- console.log('\nPromptWheel guardrails, effective set for this repo\n');
1200
- for (const r of rows) {
1201
- const prov = r.override ? 'local override' : (r.source === 'local' ? 'local' : `inherited ← ${r.source}`);
1202
- const rec = r.guard ? (r.runs ? `· ${r.flagged} flagged / ${r.runs} runs` : '· no runs yet') : '';
1203
- console.log(` ${r.guard ? '🛡️ GUARD' : '· info '} ${r.name.padEnd(18)} better=${r.direction.padEnd(5)} [${prov}] ${rec}`);
1204
- }
1205
- console.log('\n 🛡️ = enforced (a trusted regression fails the gate) · info = tracked only\n');
1206
- }
1207
-
1208
1171
  const short = (repo, ref) => { try { return git(['rev-parse', '--short', ref], repo); } catch { return ref; } };
1209
1172
 
1173
+ const wouldHave = (status) => (status === 'unmeasurable' ? 'been unmeasurable' : status);
1210
1174
  function printHuman(r) {
1211
1175
  const arrowFor = (m) => (m.delta == null ? '?' : m.delta > 0 ? '▲' : m.delta < 0 ? '▼' : '=');
1212
1176
  console.log(`\nPromptWheel ${r.base} → ${r.head}${r.repeat > 1 ? ` (×${r.repeat})` : ''}\n`);
@@ -1215,16 +1179,22 @@ function printHuman(r) {
1215
1179
  const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
1216
1180
  const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
1217
1181
  console.log(` ${arrowFor(m)} ${m.name.padEnd(18)} ${String(m.before).padStart(8)} → ${String(m.after).padStart(8)} (${d}, ${m.status}) [${tag}, ${m.confidence}]`);
1218
- if (m.inert) console.log(` ⚠ "${m.name}" never ran (same value at both refs): its command needs a build/deps first, or is not your real test command. Fix: promptwheel init, or set setup/linkDirs in promptwheel.config.json`);
1182
+ if (m.inert && m.ran === 'both') console.log(` ⚠ "${m.name}" ran and failed before and after this change, so it guards nothing here. Fix the failing tests, or the command, before trusting a pass`);
1183
+ else if (m.inert) console.log(` ⚠ "${m.name}" ${m.ran ? `ran and failed at the ${m.ran} ref and did not run at the other` : 'never ran (same value at both refs)'}: its command needs a build/deps first, or is not your real test command. Fix: promptwheel init, or set setup/linkDirs in promptwheel.config.json`);
1219
1184
  if (m.gamed === true) console.log(` 🚩 GAMED, ${m.gamingReason}`);
1220
1185
  }
1221
- if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(` 🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : 'disabled'} in this change and it would have ${w.status} at head (goalpost moved, not earned)`);
1222
- if (r.gate && r.gate.changed) console.log(` ⚠ gate definition changed in this change (fingerprint ${r.gate.fingerprint.base} -> ${r.gate.fingerprint.head}): a metric, guard, evaluatorPath or mandate was added, removed or edited; review promptwheel.config.json as part of the diff`);
1186
+ if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(w.change === 'measurement-edited'
1187
+ ? ` 🚩 GATE WEAKENED, how the guard "${w.name}" is measured (setup, env or linked dirs) was edited in this change, and measured the way it was before it would have ${wouldHave(w.status)} at head (goalpost moved, not earned)`
1188
+ : ` 🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : w.change === 'guard-edited' ? 'edited' : 'disabled'} in this change and it would have ${wouldHave(w.status)} at head (goalpost moved, not earned)`);
1189
+ if (r.gate && r.gate.changed) console.log(` ⚠ gate definition changed in this change (fingerprint ${r.gate.fingerprint.base} -> ${r.gate.fingerprint.head}): a metric, guard, evaluatorPath, mandate, or the setup, env or linked dirs they are measured with was added, removed or edited; review promptwheel.config.json as part of the diff`);
1223
1190
  const failReason = r.mandates ? ', a mandate was violated (a protected path changed)' : ', a guarded metric regressed (beyond noise)';
1224
1191
  console.log(`\n VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? failReason : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
1225
- if (r.verdict === 'fail' && r.mandates) console.log(' a mandate is a hard rule; if the change is truly intended, edit the mandate in promptwheel.config.json (a tracked, tamper-evident change), do not work around it');
1226
- else if (r.verdict === 'fail' && r.metrics.some((m) => m.allowable)) console.log(` intentional? this guard accepts a person's declared reason: promptwheel run --allow "<reason>" passes with the reason on the record (${r.metrics.filter((m) => m.allowable).map((m) => m.name).join(', ')})`);
1227
- else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name), `promptwheel guards` shows the effective set');
1192
+ if (r.verdict === 'fail' && r.mandates) {
1193
+ const takes = r.mandates.filter((md) => md.allowable).map((md) => `"${md.name}"`);
1194
+ if (takes.length) console.log(` intentional? ${takes.join(', ')} ${takes.length > 1 ? 'accept' : 'accepts'} a person's declared reason: promptwheel run --allow "<reason>" passes with the reason on the record`);
1195
+ if (takes.length < r.mandates.length) console.log(` ${takes.length ? 'any other' : 'a'} mandate is a hard rule; if the change is truly intended, retire the mandate in promptwheel.config.json in a change of its own first (a tracked, tamper-evident change), do not work around it`);
1196
+ } else if (r.verdict === 'fail' && r.metrics.some((m) => m.allowable)) console.log(` intentional? this guard accepts a person's declared reason: promptwheel run --allow "<reason>" passes with the reason on the record (${r.metrics.filter((m) => m.allowable).map((m) => m.name).join(', ')})`);
1197
+ else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name)');
1228
1198
  if (r.allowed) console.log(` ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Promptwheel-Allow: ${r.allowed.reason})`);
1229
1199
  console.log('');
1230
1200
  }
@@ -1277,9 +1247,10 @@ function renderMarkdown(r) {
1277
1247
  '|---|--:|--:|--:|---|---|',
1278
1248
  rows,
1279
1249
  '',
1280
- r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A protected path changed. This is a hard rule, enforced before any metric.' : '> ❌ A 🛡️ guarded metric regressed beyond the noise band.') : r.verdict === 'inconclusive' ? '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.' : '> ✅ No guarded metric regressed.',
1281
- ...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : 'disabled'} (would have ${w.status} at head)`).join('; ')}. The pass was bought by dropping a guard, not by earning it.`] : []),
1282
- ...(r.gate && r.gate.changed ? ['', `> ⚠ **Gate definition changed** in this diff (fingerprint \`${r.gate.fingerprint.base}\` -> \`${r.gate.fingerprint.head}\`): a metric, guard, evaluatorPath or mandate was added, removed or edited. Review the config change with the code.`] : []),
1250
+ r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A protected path changed. This is a hard rule; no metric result clears it.' : '> ❌ A 🛡️ guarded metric regressed beyond the noise band.') : r.verdict === 'inconclusive' ? (r.metrics.some((m) => m.inert) && r.metrics.every((m) => !m.inert || m.ran === 'both') ? '> 🟡 A 🛡️ guarded check is **inert**, it fails before and after this change, so nothing was actually verified. Fix it before trusting a pass.' : '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.') : '> ✅ No guarded metric regressed.',
1251
+ ...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : w.change === 'guard-edited' ? 'edited' : w.change === 'measurement-edited' ? 'measured another way (setup, env or linked dirs)' : 'disabled'} (would have ${wouldHave(w.status)} at head)`).join('; ')}. The pass was bought by changing the gate, not by earning it.`] : []),
1252
+ ...(r.allowed ? ['', `> ✓ **Allowed:** \`${r.allowed.cleared.join('`, `')}\` cleared by a declared reason: ${r.allowed.reason}`] : []),
1253
+ ...(r.gate && r.gate.changed ? ['', `> ⚠ **Gate definition changed** in this diff (fingerprint \`${r.gate.fingerprint.base}\` -> \`${r.gate.fingerprint.head}\`): a metric, guard, evaluatorPath, mandate, or the setup, env or linked dirs they are measured with was added, removed or edited. Review the config change with the code.`] : []),
1283
1254
  '',
1284
1255
  '<sub>🛡️ = guard · _prove every change moved a metric_ · [PromptWheel](https://github.com/promptwheel-ai/promptwheel)</sub>',
1285
1256
  ].join('\n');
@@ -1292,12 +1263,10 @@ function parseArgs(argv) {
1292
1263
  else if (argv[i] === '--head') a.head = argv[++i];
1293
1264
  else if (argv[i] === '--repeat') a.repeat = parseInt(argv[++i], 10);
1294
1265
  else if (argv[i] === '--working') a.working = true;
1295
- else if (argv[i] === '--no-record') a.noRecord = true;
1296
1266
  else if (argv[i] === '--no-auto') a.noAuto = true;
1297
1267
  else if (argv[i] === '--allow') a.allow = argv[++i];
1298
1268
  else if (argv[i] === '--detect-gaming') a.detectGaming = true;
1299
1269
  else if (argv[i] === '--no-detect-gaming') a.detectGaming = false;
1300
- else if (argv[i] === '--attempt') a.attempt = argv[++i];
1301
1270
  else if (argv[i] === '--json') a.json = true;
1302
1271
  else if (argv[i] === '--markdown') a.markdown = true;
1303
1272
  }
@@ -1314,7 +1283,7 @@ export { extract, evaluate, median, spread, renderMarkdown, isNonSource, evaluat
1314
1283
  // Importing this file never runs the CLI (see the isMain guard at the bottom).
1315
1284
  // Stable across 0.x minor versions. The schema check() returns and the exit codes
1316
1285
  // verdictExitCode() maps are specified in CONTRACT.md; integration paths (conform,
1317
- // shell, import, MCP) are in INTEGRATE.md.
1286
+ // shell, import) are in INTEGRATE.md.
1318
1287
  // ─────────────────────────────────────────────────────────────────────────────
1319
1288
 
1320
1289
  // The contract revision this build implements (CONTRACT.md "Versioning"). Bumps only
@@ -1351,10 +1320,9 @@ export function verdictExitCode(verdict) {
1351
1320
  // repeat number measurement repeats for the noise band (default: config.repeat ?? 1)
1352
1321
  // detectGaming boolean run the Section 3 source-only re-proof (default: true)
1353
1322
  // allow string a declared override reason (same as --allow / a Promptwheel-Allow: trailer)
1354
- // record boolean append the run to .promptwheel/outcomes.jsonl (default: false)
1355
1323
  export function check(options = {}) {
1356
- // `repo` may name a subdirectory: the gate runs at the toplevel (that is where the config lives
1357
- // and where .promptwheel/ is written), exactly as the CLI does from the same directory
1324
+ // `repo` may name a subdirectory: the gate runs at the toplevel (that is where the config
1325
+ // lives), exactly as the CLI does from the same directory
1358
1326
  let start = process.cwd();
1359
1327
  if (options.repo) { try { start = realpathSync(options.repo); } catch { bad(`promptwheel: not inside a git repository (${options.repo})`); } }
1360
1328
  const repo = repoRoot(start);
@@ -1376,7 +1344,6 @@ export function check(options = {}) {
1376
1344
  repeat: options.repeat,
1377
1345
  detectGaming: options.detectGaming ?? true,
1378
1346
  allow: options.allow,
1379
- noRecord: options.record !== true,
1380
1347
  });
1381
1348
  }
1382
1349
 
@@ -1466,72 +1433,44 @@ export function conformsToContract(report) {
1466
1433
  return { ok: errors.length === 0, errors };
1467
1434
  }
1468
1435
 
1469
- // `promptwheel conformance [file.json | -] [--cmd "<command>"]`
1470
- // Self-check a Contract implementation. With a file or stdin, validate that verdict JSON
1471
- // against the Section 2 schema. With --cmd, run the command, then validate its --json
1472
- // stdout (Section 2) AND that its exit code matches the verdict per Section 1. Exit 0 if
1473
- // it conforms, 1 otherwise, listing every problem.
1474
- function conformance(argv) {
1475
- let cmd = null, src = null;
1476
- for (let i = 0; i < argv.length; i++) {
1477
- if (argv[i] === '--cmd') cmd = argv[++i];
1478
- else if (argv[i] === '-' || !argv[i].startsWith('-')) src = argv[i];
1479
- }
1480
-
1481
- let text, exitCode = null;
1482
- if (cmd) {
1483
- try { text = execFileSync('sh', ['-c', cmd], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); exitCode = 0; }
1484
- catch (e) {
1485
- text = (e.stdout || '').toString();
1486
- exitCode = typeof e.status === 'number' ? e.status : null;
1487
- if (!text) { console.error(`conformance: command produced no stdout (exit ${exitCode}): ${e.message}`); process.exit(1); }
1488
- }
1489
- } else {
1490
- const file = src && src !== '-' ? src : 0; // fd 0 = stdin
1491
- try { text = readFileSync(file, 'utf8'); }
1492
- catch (e) { console.error(`conformance: cannot read ${!src || src === '-' ? 'stdin' : src}: ${e.message}`); process.exit(1); }
1493
- }
1494
-
1495
- let report;
1496
- try { report = JSON.parse(text); }
1497
- catch (e) { console.error(`conformance: not valid JSON: ${e.message}`); process.exit(1); }
1498
-
1499
- const { errors } = conformsToContract(report);
1500
- const problems = [...errors];
1501
- if (cmd && exitCode !== null && VERDICTS.includes(report.verdict)) {
1502
- const want = verdictExitCode(report.verdict);
1503
- if (exitCode !== want) problems.push(`exit code ${exitCode} does not match verdict '${report.verdict}' (Section 1 requires ${want})`);
1504
- }
1505
-
1506
- if (problems.length === 0) {
1507
- console.log(`ok, conforms to the Verified-Outcome Contract v${CONTRACT_VERSION} (verdict: ${report.verdict}${cmd ? `, exit ${exitCode}` : ''})`);
1508
- process.exit(0);
1509
- }
1510
- console.error(`does NOT conform to the Contract (${problems.length} problem${problems.length > 1 ? 's' : ''}):`);
1511
- for (const p of problems) console.error(` - ${p}`);
1512
- process.exit(1);
1513
- }
1514
-
1515
1436
  // the flags each command accepts. An unknown --flag is a usage error (exit 64), never silently
1516
1437
  // ignored: a typo in a CI step must not turn `--no-detect-gaming` into the default gate or `--allow`
1517
1438
  // into nothing, and INTEGRATE.md promises a bad flag exits 64. A flag that takes a value consumes
1518
- // the next argument; anything not starting with `-` (or a bare `-`, stdin for conformance) is positional.
1439
+ // the next argument; anything not starting with `-` is positional. --no-record does nothing now that
1440
+ // no record is written, and stays accepted because Sawbuck and existing CI steps pass it.
1519
1441
  const GATE_FLAGS = ['--base', '--head', '--repeat', '--working', '--no-record', '--no-auto', '--allow', '--detect-gaming', '--no-detect-gaming', '--json', '--markdown'];
1520
1442
  const COMMAND_FLAGS = {
1521
- run: GATE_FLAGS, improve: [...GATE_FLAGS, '--attempt'], insights: ['--json'], guards: ['--json'],
1522
- init: ['--preset', '--list', '--force', '--no-agent-block'], conformance: ['--cmd'], lock: [],
1443
+ run: GATE_FLAGS, init: ['--preset', '--list', '--force', '--no-agent-block'], lock: [],
1523
1444
  };
1524
- const VALUED_FLAGS = new Set(['--base', '--head', '--repeat', '--allow', '--attempt', '--preset', '--cmd']);
1445
+ // what each valued flag needs. A missing or unusable value is a usage error too: `--repeat abc` must not
1446
+ // measure nothing and read FAIL, and `--allow` with nothing after it must not run as if no reason was asked for.
1447
+ const VALUED_FLAGS = { '--base': 'a ref', '--head': 'a ref', '--repeat': 'a whole number, 1 or more', '--allow': 'a reason, e.g. --allow "the spec changed"', '--preset': 'a preset name (see promptwheel init --list)' };
1525
1448
  function checkFlags(cmd, argv) {
1526
1449
  const known = new Set(COMMAND_FLAGS[cmd]);
1527
1450
  for (let i = 0; i < argv.length; i++) {
1528
1451
  const a = argv[i];
1529
- if (a === '-' || !a.startsWith('-')) continue;
1452
+ if (!a.startsWith('-')) continue;
1530
1453
  if (!known.has(a)) bad(`promptwheel ${cmd}: unknown flag ${a} (see promptwheel --help)`);
1531
- if (VALUED_FLAGS.has(a)) i++;
1454
+ if (!(a in VALUED_FLAGS)) continue;
1455
+ const v = argv[++i];
1456
+ const gone = v === undefined || known.has(v); // nothing after the flag, or the next flag
1457
+ // an empty --base or --head is the default ref: the Action passes an empty base outside a pull request
1458
+ const blank = !gone && !v.trim() && !(v === '' && (a === '--base' || a === '--head'));
1459
+ if (gone || blank) bad(`promptwheel ${cmd}: ${a} needs ${VALUED_FLAGS[a]}`);
1460
+ if (a === '--repeat' && !/^[1-9]\d*$/.test(v)) bad(`promptwheel ${cmd}: --repeat needs ${VALUED_FLAGS[a]} (got "${v}")`);
1532
1461
  }
1533
1462
  }
1534
1463
 
1464
+ // read PR comments on stdin: a JSON array, or one comment per line as `gh api --paginate --jq '.[]'` prints every
1465
+ // page. Streamed: readFileSync(0) corrupted the heap on Node 24.22 reading a live paginated gh pipe.
1466
+ async function prAllow() {
1467
+ let t = ''; process.stdin.setEncoding('utf8');
1468
+ for await (const chunk of process.stdin) t += chunk;
1469
+ t = t.trim(); let c = [];
1470
+ try { c = [].concat(JSON.parse(t || '[]')); } catch { try { c = t.split('\n').filter(Boolean).map((l) => JSON.parse(l)); } catch { /* unreadable, no approval */ } }
1471
+ process.stdout.write(allowFromComments(c));
1472
+ }
1473
+
1535
1474
  // write or refresh the architecture lock from the tree as it stands. A person runs this: the lock file is
1536
1475
  // a mandate, so the refreshed file passes the gate only with --allow "<reason>".
1537
1476
  async function lock() {
@@ -1546,13 +1485,9 @@ function dispatch() {
1546
1485
  const cmd = (rest.includes('--help') || rest.includes('-h')) ? 'help' : first; // `promptwheel run --help` is a help request, not a gate run
1547
1486
  if (cmd in COMMAND_FLAGS) checkFlags(cmd, rest);
1548
1487
  if (cmd === 'run') run(rest);
1549
- else if (cmd === 'improve') improve(rest);
1550
- else if (cmd === 'pr-allow') { let c = []; try { c = JSON.parse(readFileSync(0, 'utf8') || '[]'); } catch { /* no comments, no approval */ } process.stdout.write(allowFromComments(c)); }
1551
- else if (cmd === 'insights') insights(rest);
1488
+ else if (cmd === 'pr-allow') return prAllow();
1552
1489
  else if (cmd === 'init') init(rest);
1553
- else if (cmd === 'guards') guards(rest);
1554
- else if (cmd === 'conformance') conformance(rest);
1555
- else if (cmd === 'lock') lock(rest);
1490
+ else if (cmd === 'lock') return lock(rest);
1556
1491
  else {
1557
1492
  const help = !cmd || cmd === '--help' || cmd === '-h' || cmd === 'help';
1558
1493
  if (!help) console.error(`unknown command: ${cmd}`);
@@ -1567,16 +1502,11 @@ function dispatch() {
1567
1502
  ' catches reward-hacking BY DEFAULT, re-proves each win from SOURCE edits alone (verdict GAMED,',
1568
1503
  ' exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
1569
1504
  ' promptwheel run --working gate uncommitted changes (incl. newly added files)',
1505
+ ' promptwheel pr-allow < comments.json print the newest `sawbuck-allow: <reason>` from a PR comment by someone with write access (the Action uses it)',
1570
1506
  ' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `Promptwheel-Allow: <reason>` commit trailer; `Sawbuck-Allow:` is an alias): recorded, not flagged GAMED',
1571
- ' promptwheel improve --attempt "<cmd>" run an agent/script; keep only if a metric improved',
1572
- ' exit 0=kept · 1=regression · 2=gamed · 3=plateau/inconclusive (all but 0 reverted) · add --json',
1573
- ' promptwheel insights [--json] which metrics actually respond (raw counts from .promptwheel/outcomes.jsonl)',
1574
- ' promptwheel guards [--json] show the effective guardrails (incl. inherited) + flag record',
1575
- ' promptwheel conformance [file|-] [--cmd C] check a verdict JSON (or a command) against the Contract (exit 0 = conforms)',
1576
1507
  '',
1577
1508
  'Exit codes: 0 pass · 1 fail (a guarded metric regressed, or a mandate was violated) · 2 gamed · 3 inconclusive · 64 usage or config error (never a verdict).',
1578
- 'Integrate (library / conform / shell / MCP): see INTEGRATE.md. Import: `import { check } from \'promptwheel\'`.',
1579
- 'Loop it: while promptwheel improve --attempt "$AGENT"; do :; done # stops on plateau/regression',
1509
+ 'Integrate (library / conform / shell): see INTEGRATE.md. Import: `import { check } from \'promptwheel\'`.',
1580
1510
  'Config: promptwheel.config.json → { extends?, metrics:[{ name, cmd, direction, extract?, guard? }] } (or: promptwheel init)',
1581
1511
  ' extends: a path (or array) to a shared base config, repos INHERIT its guardrails; local metrics override by name.',
1582
1512
  ].join('\n'));
@@ -1587,8 +1517,8 @@ function dispatch() {
1587
1517
  // run the CLI only when invoked directly (resolves symlinks so the npm bin still works)
1588
1518
  let isMain = false;
1589
1519
  try { isMain = !!process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { /* not main */ }
1590
- function main() {
1591
- try { dispatch(); }
1520
+ async function main() {
1521
+ try { await dispatch(); }
1592
1522
  catch (e) { if (e && e.name === 'ConfigError') die(e.message); throw e; }
1593
1523
  }
1594
1524
  if (isMain) main();