promptwheel 0.7.3 → 0.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INTEGRATE.md +29 -41
- package/README.md +102 -170
- package/bin/promptwheel.mjs +185 -286
- package/package.json +3 -5
- package/packs/security/README.md +2 -2
package/bin/promptwheel.mjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// PromptWheel,
|
|
2
|
+
// PromptWheel: a deterministic, zero-LLM check for your coding loop. It proves every change
|
|
3
|
+
// earned its win by re-running the result from the agent's source edits alone, so a green
|
|
4
|
+
// that came from editing the test reads GAMED.
|
|
3
5
|
//
|
|
4
|
-
// For any change (base ref
|
|
6
|
+
// For any change (base ref -> head ref), measure each configured metric in an
|
|
5
7
|
// isolated git worktree BEFORE and AFTER, enforce regression guards, and refuse
|
|
6
8
|
// to trust a delta that sits inside the measurement noise band. Emits a
|
|
7
9
|
// structured {verdict, metric, delta, confidence}. The thing base agents don't do:
|
|
@@ -29,9 +31,14 @@ const median = (xs) => {
|
|
|
29
31
|
};
|
|
30
32
|
const spread = (xs) => { const a = xs.filter((x) => x != null); return a.length ? Math.max(...a) - Math.min(...a) : 0; };
|
|
31
33
|
|
|
34
|
+
// usage / config errors exit 64 (EX_USAGE), so they can never be confused with a verdict
|
|
35
|
+
// (0 pass, 1 fail, 2 gamed, 3 inconclusive): in CI a config typo must not read as GAMED.
|
|
36
|
+
const EX_USAGE = 64;
|
|
37
|
+
function die(msg) { console.error(msg); process.exit(EX_USAGE); }
|
|
38
|
+
|
|
32
39
|
function readConfigFile(p) {
|
|
33
40
|
try { return JSON.parse(readFileSync(p, 'utf8')); }
|
|
34
|
-
catch (e) {
|
|
41
|
+
catch (e) { die(`invalid config ${p}: ${e.message}`); }
|
|
35
42
|
}
|
|
36
43
|
|
|
37
44
|
const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { return p; } };
|
|
@@ -41,42 +48,42 @@ const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { retu
|
|
|
41
48
|
// a repo INHERITS their guardrails, and a local metric of the same name overrides the inherited one.
|
|
42
49
|
function resolveConfig(p, repo, label, seen) {
|
|
43
50
|
const abs = realpathSync(p);
|
|
44
|
-
if (seen.has(abs))
|
|
51
|
+
if (seen.has(abs)) die(`config inheritance cycle at ${relOf(repo, p)}`);
|
|
45
52
|
seen.add(abs);
|
|
46
53
|
try {
|
|
47
54
|
const cfg = readConfigFile(p);
|
|
48
55
|
const byName = new Map();
|
|
56
|
+
const mandatesByName = new Map(); // hard rules accumulate down the extends chain (a parent's holds)
|
|
49
57
|
const scalars = {};
|
|
50
58
|
for (const ref of [].concat(cfg.extends || [])) {
|
|
51
59
|
const bp = isAbsolute(ref) ? ref : join(dirname(p), ref);
|
|
52
|
-
if (!existsSync(bp))
|
|
60
|
+
if (!existsSync(bp)) die(`extends target not found: ${ref} (from ${relOf(repo, p)})`);
|
|
53
61
|
const base = resolveConfig(bp, repo, relOf(repo, bp), seen);
|
|
54
62
|
for (const m of base.metrics) byName.set(m.name, m);
|
|
63
|
+
for (const md of (base.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
55
64
|
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'record', 'gamingThreshold', 'evaluatorPaths']) if (base[k] !== undefined) scalars[k] = base[k];
|
|
56
65
|
}
|
|
57
66
|
for (const m of (cfg.metrics || [])) {
|
|
58
67
|
byName.set(m.name, { ...m, __src: label, __override: byName.has(m.name) });
|
|
59
68
|
}
|
|
69
|
+
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, { ...md, __src: label });
|
|
60
70
|
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'record', 'gamingThreshold', 'evaluatorPaths']) if (cfg[k] !== undefined) scalars[k] = cfg[k];
|
|
61
|
-
return { ...scalars, metrics: [...byName.values()] };
|
|
71
|
+
return { ...scalars, metrics: [...byName.values()], mandates: [...mandatesByName.values()] };
|
|
62
72
|
} finally {
|
|
63
73
|
seen.delete(abs); // only ACTIVE ancestors are a cycle, a diamond (a base reached two ways) is fine
|
|
64
74
|
}
|
|
65
75
|
}
|
|
66
76
|
|
|
67
77
|
function loadConfig(repo) {
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
if (!p) { console.error('no promptwheel.config.json, run: promptwheel init (writes one for your stack)'); process.exit(2); }
|
|
78
|
+
const p = join(repo, 'promptwheel.config.json');
|
|
79
|
+
if (!existsSync(p)) die('no promptwheel.config.json, run: promptwheel init (writes one for your stack)');
|
|
71
80
|
const cfg = resolveConfig(p, repo, 'local', new Set());
|
|
72
|
-
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0)
|
|
73
|
-
console.error('config.metrics must be a non-empty array'); process.exit(2);
|
|
74
|
-
}
|
|
81
|
+
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0) die('config.metrics must be a non-empty array');
|
|
75
82
|
return cfg;
|
|
76
83
|
}
|
|
77
84
|
|
|
78
85
|
// does the repo have a committed gate config?
|
|
79
|
-
const hasConfigFile = (repo) =>
|
|
86
|
+
const hasConfigFile = (repo) => existsSync(join(repo, 'promptwheel.config.json'));
|
|
80
87
|
|
|
81
88
|
// the zero-config default: the gate `init` would write, built in memory from stack detection.
|
|
82
89
|
// It gates the detected test command (gaming detection on) with the stack's env/linkDirs so the
|
|
@@ -89,8 +96,13 @@ function autoConfig(repo) {
|
|
|
89
96
|
|
|
90
97
|
function resolveBase(repo, base) {
|
|
91
98
|
if (base) return base;
|
|
99
|
+
const head = git(['rev-parse', 'HEAD'], repo);
|
|
92
100
|
for (const b of ['origin/main', 'origin/master', 'main', 'master']) {
|
|
93
|
-
|
|
101
|
+
let mb = '';
|
|
102
|
+
try { mb = git(['merge-base', 'HEAD', b], repo); } catch { continue; }
|
|
103
|
+
// on the trunk itself the merge-base IS HEAD; comparing a commit to itself would be a
|
|
104
|
+
// trivial PASS that verified nothing, so fall through to the last commit instead.
|
|
105
|
+
if (mb && mb !== head) return mb;
|
|
94
106
|
}
|
|
95
107
|
return git(['rev-parse', 'HEAD~1'], repo);
|
|
96
108
|
}
|
|
@@ -267,6 +279,33 @@ function evaluatorMatcher(cfg) {
|
|
|
267
279
|
return (p) => res.some((re) => re.test(p));
|
|
268
280
|
}
|
|
269
281
|
|
|
282
|
+
// mandates: named hard rules that compile to a deterministic gate. A protected-path mandate
|
|
283
|
+
// forbids ANY change to its paths (same glob semantics as evaluatorPaths). A hit fails the gate
|
|
284
|
+
// regardless of metrics and rides at the top of the report; mandates inherit through `extends`,
|
|
285
|
+
// so a parent scope's rule holds for every repo beneath it.
|
|
286
|
+
// The mandates in force are the UNION of the head config's and the base ref's (`baseMandates`,
|
|
287
|
+
// resolved by gateDefAtRef): deleting a mandate in the same change that violates it does not
|
|
288
|
+
// clear it. A mandate removed at head but violated in the diff is reported with `removed: true`.
|
|
289
|
+
// The honest way to retire one is a separate change that only edits the config.
|
|
290
|
+
function checkMandates(repo, base, head, cfg, baseMandates) {
|
|
291
|
+
const headList = Array.isArray(cfg.mandates) ? cfg.mandates : [];
|
|
292
|
+
const headNames = new Set(headList.map((m) => m && m.name).filter(Boolean));
|
|
293
|
+
const removed = (Array.isArray(baseMandates) ? baseMandates : []).filter((m) => m && m.name && !headNames.has(m.name));
|
|
294
|
+
const mandates = [...headList, ...removed.map((m) => ({ ...m, __removed: true }))];
|
|
295
|
+
if (!mandates.length) return [];
|
|
296
|
+
let changed = [];
|
|
297
|
+
try { changed = git(['diff', '--name-only', base, head], repo).split('\n').filter(Boolean); } catch { return []; }
|
|
298
|
+
const out = [];
|
|
299
|
+
for (const m of mandates) {
|
|
300
|
+
const pats = (Array.isArray(m.paths) ? m.paths : []).filter((s) => typeof s === 'string' && s.trim());
|
|
301
|
+
if (!pats.length) continue;
|
|
302
|
+
const res = pats.map(globToRe);
|
|
303
|
+
const hit = changed.filter((f) => res.some((re) => re.test(f)));
|
|
304
|
+
if (hit.length) out.push({ name: String(m.name || 'mandate'), reason: m.reason ? String(m.reason).slice(0, 300) : '', files: hit.slice(0, 20), ...(m.__removed ? { removed: true } : {}) });
|
|
305
|
+
}
|
|
306
|
+
return out;
|
|
307
|
+
}
|
|
308
|
+
|
|
270
309
|
// split the agent's changed files into production-source vs test/config/grader/golden.
|
|
271
310
|
// `isEval` is the repo's opt-in evaluator-path matcher, treated as non-source alongside
|
|
272
311
|
// the built-in classifier, so a cheat that only edits a declared gate file is caught.
|
|
@@ -276,18 +315,6 @@ function changedSourcePaths(repo, base, head, isEval = () => false) {
|
|
|
276
315
|
return { source: all.filter((p) => !nonSrc(p)), nonSource: all.filter(nonSrc) };
|
|
277
316
|
}
|
|
278
317
|
|
|
279
|
-
// coarse change-location fingerprint for the outcome record: top source dirs of the diff
|
|
280
|
-
function subsystemsOf(repo, base, head, cfg) {
|
|
281
|
-
try {
|
|
282
|
-
const counts = {};
|
|
283
|
-
for (const p of changedSourcePaths(repo, base, head, evaluatorMatcher(cfg)).source) {
|
|
284
|
-
const d = p.includes('/') ? p.slice(0, p.indexOf('/')) : '(root)';
|
|
285
|
-
counts[d] = (counts[d] || 0) + 1;
|
|
286
|
-
}
|
|
287
|
-
return Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([d]) => d);
|
|
288
|
-
} catch { return []; }
|
|
289
|
-
}
|
|
290
|
-
|
|
291
318
|
// measure `metric` in a worktree at `base` with ONLY the source slice of base→head applied.
|
|
292
319
|
function measureSourceOnly(repo, base, head, metric, cfg, repeat) {
|
|
293
320
|
const { source } = changedSourcePaths(repo, base, head, evaluatorMatcher(cfg));
|
|
@@ -298,7 +325,9 @@ function measureSourceOnly(repo, base, head, metric, cfg, repeat) {
|
|
|
298
325
|
const env = bridgeEnv(repo, wt, cfg);
|
|
299
326
|
const patch = git(['diff', base, head, '--', ...source], repo);
|
|
300
327
|
if (patch.trim()) {
|
|
301
|
-
|
|
328
|
+
// stdio stderr 'ignore': git apply --3way prints "Applied patch ... cleanly" to stderr; that
|
|
329
|
+
// is noise on an otherwise-quiet run (and corrupts --json for callers that fold in stderr).
|
|
330
|
+
const tryApply = (extra) => { execFileSync('git', ['apply', '--whitespace=nowarn', ...extra], { cwd: wt, input: patch + '\n', encoding: 'utf8', stdio: ['pipe', 'ignore', 'ignore'] }); };
|
|
302
331
|
try { tryApply(['--3way']); }
|
|
303
332
|
catch { try { tryApply([]); } catch { return { samples: [], hadSourceChange: true, applyFailed: true }; } }
|
|
304
333
|
}
|
|
@@ -389,8 +418,9 @@ function selfHeal(repo) {
|
|
|
389
418
|
// Python editable installs (src-layout `pip install -e .`) can pin imports to the ORIGINAL
|
|
390
419
|
// checkout: the worktree's tests then import your current tree at BOTH refs and every delta
|
|
391
420
|
// reads 0, the gate is structurally blind on that repo shape. Undetectable from the verdict
|
|
392
|
-
// (it looks like "unchanged"), so
|
|
393
|
-
// controlled repro (CHANGELOG 0.2.2).
|
|
421
|
+
// (it looks like "unchanged"), so `init` detects the editable install itself and warns once.
|
|
422
|
+
// Verified by a controlled repro (CHANGELOG 0.2.2). Deliberately NOT run by gate(): it spawns
|
|
423
|
+
// python3 with a 5 s timeout and prints, and the library path (check) must do neither.
|
|
394
424
|
function warnEditableInstall(repo) {
|
|
395
425
|
if (!existsSync(join(repo, 'pyproject.toml')) && !existsSync(join(repo, 'setup.py'))) return;
|
|
396
426
|
try {
|
|
@@ -403,7 +433,7 @@ function warnEditableInstall(repo) {
|
|
|
403
433
|
'print("HIT" if hit else "")',
|
|
404
434
|
].join('\n');
|
|
405
435
|
const out = execFileSync('python3', ['-c', probe, repo], { encoding: 'utf8', timeout: 5000 }).trim();
|
|
406
|
-
if (out === 'HIT') console.error('⚠ editable install of this repo detected (pip install -e): worktree measurements
|
|
436
|
+
if (out === 'HIT') console.error('⚠ editable install of this repo detected (pip install -e): worktree measurements could import your ORIGINAL checkout instead of the measured ref, so deltas would read 0. The written config puts the worktree first on PYTHONPATH to counter this; if deltas still read 0, use a non-editable install in the measuring venv.');
|
|
407
437
|
} catch { /* no python / cannot tell, stay quiet */ }
|
|
408
438
|
}
|
|
409
439
|
|
|
@@ -415,10 +445,12 @@ function warnEditableInstall(repo) {
|
|
|
415
445
|
// that guard WOULD HAVE FAILED at head (a real regression hidden by moving the goalposts).
|
|
416
446
|
const GATE_SCALARS = ['gamingThreshold', 'evaluatorPaths'];
|
|
417
447
|
|
|
418
|
-
// Resolve the gate definition as of `ref` (metrics + scalars), following `extends` by
|
|
419
|
-
// every file from that ref via `git show
|
|
420
|
-
//
|
|
421
|
-
//
|
|
448
|
+
// Resolve the gate definition as of `ref` (metrics + mandates + scalars), following `extends` by
|
|
449
|
+
// reading every file from that ref via `git show`, mirroring resolveConfig (a parent's mandates
|
|
450
|
+
// hold beneath it; a local mandate of the same name overrides). Returns { metrics:Map, scalars }
|
|
451
|
+
// with scalars.mandates, or null when there is no config at `ref` (a gate ADDED in this range is
|
|
452
|
+
// not a weakening). Never exits: any missing/unreadable file or parse error yields null (that
|
|
453
|
+
// branch simply contributes nothing).
|
|
422
454
|
function gateDefAtRef(repo, ref) {
|
|
423
455
|
// quiet: a missing path makes `git show` exit nonzero and print "fatal:" to stderr; swallow it
|
|
424
456
|
const readAt = (p) => { try { return execFileSync('git', ['show', `${ref}:${p}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return null; } };
|
|
@@ -429,20 +461,22 @@ function gateDefAtRef(repo, ref) {
|
|
|
429
461
|
const raw = readAt(p);
|
|
430
462
|
if (raw == null) return null;
|
|
431
463
|
let cfg; try { cfg = JSON.parse(raw); } catch { return null; }
|
|
432
|
-
const byName = new Map(); const scalars = {};
|
|
464
|
+
const byName = new Map(); const mandatesByName = new Map(); const scalars = {};
|
|
433
465
|
for (const ext of [].concat(cfg.extends || [])) {
|
|
434
466
|
if (isAbsolute(ext)) continue; // outside the repo, cannot git-show; unchanged anyway
|
|
435
467
|
const base = resolve(join(dirname(p), ext).split('\\').join('/'));
|
|
436
468
|
if (!base) continue;
|
|
437
469
|
for (const m of base.metrics.values()) byName.set(m.name, m);
|
|
470
|
+
for (const md of base.scalars.mandates) mandatesByName.set(md.name, md);
|
|
438
471
|
for (const k of GATE_SCALARS) if (base.scalars[k] !== undefined) scalars[k] = base.scalars[k];
|
|
439
472
|
}
|
|
440
473
|
for (const m of (cfg.metrics || [])) if (m && m.name) byName.set(m.name, m);
|
|
474
|
+
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
441
475
|
for (const k of GATE_SCALARS) if (cfg[k] !== undefined) scalars[k] = cfg[k];
|
|
476
|
+
scalars.mandates = [...mandatesByName.values()];
|
|
442
477
|
return { metrics: byName, scalars };
|
|
443
478
|
};
|
|
444
|
-
|
|
445
|
-
return null;
|
|
479
|
+
return resolve('promptwheel.config.json');
|
|
446
480
|
}
|
|
447
481
|
|
|
448
482
|
// an order-independent SHA-256 (truncated) fingerprint of a gate definition, for the audit record
|
|
@@ -454,6 +488,7 @@ function gateFingerprint(metricsList, scalars) {
|
|
|
454
488
|
})).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
|
|
455
489
|
gamingThreshold: scalars.gamingThreshold ?? null,
|
|
456
490
|
evaluatorPaths: Array.isArray(scalars.evaluatorPaths) ? [...scalars.evaluatorPaths].sort() : [],
|
|
491
|
+
mandates: Array.isArray(scalars.mandates) ? scalars.mandates.map((m) => ({ name: String(m.name || ''), paths: [...(Array.isArray(m.paths) ? m.paths : [])].sort() })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) : [],
|
|
457
492
|
};
|
|
458
493
|
return createHash('sha256').update(JSON.stringify(norm)).digest('hex').slice(0, 16);
|
|
459
494
|
}
|
|
@@ -512,8 +547,8 @@ function gateWeakenings(baseGate, headMetrics) {
|
|
|
512
547
|
}
|
|
513
548
|
|
|
514
549
|
// A declared, auditable override (CONTRACT Section 3). The head commit can acknowledge an
|
|
515
|
-
// intentional test/config/grader change
|
|
516
|
-
// gate change, with a trailer:
|
|
550
|
+
// intentional test/config/grader change (an edit, a delete, or a new file) or an intentional
|
|
551
|
+
// gate change, with a trailer: Promptwheel-Allow: <reason> (Sawbuck-Allow: is an alias)
|
|
517
552
|
// so a change that would read GAMED reads PASS instead, with the acknowledgment RECORDED in
|
|
518
553
|
// report.allowed. The check does not judge intent; it forces intent to be explicit. An UNdeclared
|
|
519
554
|
// test-only win is still gamed. `--allow "<reason>"` (CLI) / `allow` (check) supplies the same for
|
|
@@ -521,13 +556,67 @@ function gateWeakenings(baseGate, headMetrics) {
|
|
|
521
556
|
function readAllowTrailer(repo, ref) {
|
|
522
557
|
let msg = '';
|
|
523
558
|
try { msg = execFileSync('git', ['log', '-1', '--format=%B', ref], { cwd: repo, encoding: 'utf8' }); } catch { return null; }
|
|
524
|
-
const m = msg.match(/^[ \t]*(?:
|
|
559
|
+
const m = msg.match(/^[ \t]*(?:Promptwheel-Allow|Sawbuck-Allow)[ \t]*:[ \t]*(\S.*?)[ \t]*$/im);
|
|
525
560
|
return m && m[1].trim() ? { reason: m[1].trim().slice(0, 300) } : null;
|
|
526
561
|
}
|
|
527
562
|
|
|
563
|
+
// The three Section 3 checks, run only with detectGaming, in order: (1) re-prove every actual
|
|
564
|
+
// win with the agent's source edits alone; (2) content-hash auto-fail: a declared evaluatorPath
|
|
565
|
+
// gate file edited in this diff is a moved goalpost, so a metric that improved alongside it is
|
|
566
|
+
// gamed by declaration, deterministic even if the source-only re-run read inert; (3) gate
|
|
567
|
+
// integrity: resolve the gate as of BASE, re-measure any guard HEAD dropped under its base
|
|
568
|
+
// definition (it only counts as tampering if it WOULD HAVE FAILED at head), and fingerprint the
|
|
569
|
+
// gate definition at both refs (metrics, evaluatorPaths, gamingThreshold, mandates) so a quietly
|
|
570
|
+
// deleted guard or mandate is tamper-evident: base != head is recorded as gate.changed and
|
|
571
|
+
// printed, whatever the verdict. Mutates `metrics` in place (gamed/sourceOnly/retained/
|
|
572
|
+
// gamingReason) and returns the report.gate record. An allow trailer can still clear a gamed
|
|
573
|
+
// metric afterwards, in gate().
|
|
574
|
+
function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
|
|
575
|
+
for (const m of metrics) {
|
|
576
|
+
if (m.status !== 'improved') continue;
|
|
577
|
+
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
578
|
+
if (cm.gamingCheck === false) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
|
|
579
|
+
const j = judgeGaming(m, measureSourceOnly(repo, base, head, cm, cfg, repeat), cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5);
|
|
580
|
+
m.gamed = j.gamed; m.sourceOnly = j.sourceOnly; m.retained = j.retained; m.gamingReason = j.reason;
|
|
581
|
+
}
|
|
582
|
+
const evalChanged = changedEvaluatorPaths(repo, base, head, cfg);
|
|
583
|
+
if (evalChanged.length) {
|
|
584
|
+
for (const m of metrics) {
|
|
585
|
+
if (m.status !== 'improved' || m.gamed === true) continue;
|
|
586
|
+
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
587
|
+
if (cm && cm.gamingCheck === false) continue; // tripwire/test-side guards legitimately live in gate files
|
|
588
|
+
m.gamed = true;
|
|
589
|
+
m.gamingReason = `a declared evaluatorPath gate file changed in this diff (${evalChanged.map((e) => e.path).join(', ')}); a win alongside a goalpost edit is gamed by declaration`;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
const baseGate = gateDefAtRef(repo, base);
|
|
593
|
+
const weak = gateWeakenings(baseGate, cfg.metrics);
|
|
594
|
+
const weakened = [];
|
|
595
|
+
if (weak.length) {
|
|
596
|
+
const defs = weak.map((w) => w.def);
|
|
597
|
+
const b = measureAt(repo, base, { ...cfg, metrics: defs }, repeat);
|
|
598
|
+
const a = measureAt(repo, head, { ...cfg, metrics: defs }, repeat);
|
|
599
|
+
for (const w of weak) {
|
|
600
|
+
const ev = evaluate(w.def, b[w.name], a[w.name], repeat);
|
|
601
|
+
if (!ev.ok) weakened.push({ name: w.name, change: w.change, status: ev.status });
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
const fpBase = baseGate ? gateFingerprint([...baseGate.metrics.values()], baseGate.scalars) : null;
|
|
605
|
+
const fpHead = gateFingerprint(cfg.metrics, cfg);
|
|
606
|
+
const info = {
|
|
607
|
+
fingerprint: { base: fpBase, head: fpHead },
|
|
608
|
+
...(fpBase && fpBase !== fpHead ? { changed: true } : {}),
|
|
609
|
+
...(weakened.length ? { weakened } : {}),
|
|
610
|
+
...(evalChanged.length ? { evaluatorChanged: evalChanged } : {}),
|
|
611
|
+
};
|
|
612
|
+
// hand the base ref's mandates to gate() (not part of the record) so checkMandates can hold a
|
|
613
|
+
// change to a mandate it deleted in the same diff.
|
|
614
|
+
Object.defineProperty(info, 'baseMandates', { value: baseGate ? baseGate.scalars.mandates || [] : null, enumerable: false });
|
|
615
|
+
return info;
|
|
616
|
+
}
|
|
617
|
+
|
|
528
618
|
function gate(repo, opts) {
|
|
529
619
|
selfHeal(repo);
|
|
530
|
-
warnEditableInstall(repo);
|
|
531
620
|
const cfg = opts.config ?? loadConfig(repo);
|
|
532
621
|
const repeat = Math.max(1, opts.repeat ?? cfg.repeat ?? 1);
|
|
533
622
|
|
|
@@ -537,7 +626,7 @@ function gate(repo, opts) {
|
|
|
537
626
|
// never touches the real index or working tree. (A loop agent's most common action
|
|
538
627
|
// is to ADD a file; `git stash create` omits untracked, which silently reverted them.)
|
|
539
628
|
try { git(['rev-parse', '--verify', 'HEAD'], repo); }
|
|
540
|
-
catch {
|
|
629
|
+
catch { die('no commits yet, make an initial commit before gating --working changes'); }
|
|
541
630
|
base = 'HEAD';
|
|
542
631
|
head = workingSnapshot(repo);
|
|
543
632
|
} else {
|
|
@@ -551,59 +640,9 @@ function gate(repo, opts) {
|
|
|
551
640
|
const ev = evaluate(m, before[m.name], after[m.name], repeat);
|
|
552
641
|
return { name: m.name, direction: m.direction || 'up', guard: !!m.guard, ...ev };
|
|
553
642
|
});
|
|
554
|
-
//
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
if (m.status !== 'improved') continue;
|
|
558
|
-
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
559
|
-
if (cm.gamingCheck === false) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
|
|
560
|
-
const j = judgeGaming(m, measureSourceOnly(repo, base, head, cm, cfg, repeat), cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5);
|
|
561
|
-
m.gamed = j.gamed; m.sourceOnly = j.sourceOnly; m.retained = j.retained; m.gamingReason = j.reason;
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
|
-
// content-hash auto-fail (Section 3): a declared evaluatorPath gate file edited in this diff is a
|
|
565
|
-
// moved goalpost, so a metric that improved alongside it is gamed by declaration - deterministic,
|
|
566
|
-
// even if the source-only re-run above read inert. An allow-trailer can still clear it, below.
|
|
567
|
-
let evalChanged = [];
|
|
568
|
-
if (opts.detectGaming) {
|
|
569
|
-
evalChanged = changedEvaluatorPaths(repo, base, head, cfg);
|
|
570
|
-
if (evalChanged.length) {
|
|
571
|
-
for (const m of metrics) {
|
|
572
|
-
if (m.status !== 'improved' || m.gamed === true) continue;
|
|
573
|
-
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
574
|
-
if (cm && cm.gamingCheck === false) continue; // tripwire/test-side guards legitimately live in gate files
|
|
575
|
-
m.gamed = true;
|
|
576
|
-
m.gamingReason = `a declared evaluatorPath gate file changed in this diff (${evalChanged.map((e) => e.path).join(', ')}); a win alongside a goalpost edit is gamed by declaration`;
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
// gate integrity (Section 3): did HEAD weaken the gate it is judged by? Resolve the gate as
|
|
581
|
-
// of BASE; a guard HEAD dropped is re-measured under its base definition and only counts as
|
|
582
|
-
// tampering if it WOULD HAVE FAILED at head (a real regression hidden by dropping the guard).
|
|
583
|
-
let gateInfo = null;
|
|
584
|
-
if (opts.detectGaming) {
|
|
585
|
-
const baseGate = gateDefAtRef(repo, base);
|
|
586
|
-
const weak = gateWeakenings(baseGate, cfg.metrics);
|
|
587
|
-
const weakened = [];
|
|
588
|
-
if (weak.length) {
|
|
589
|
-
const defs = weak.map((w) => w.def);
|
|
590
|
-
const b = measureAt(repo, base, { ...cfg, metrics: defs }, repeat);
|
|
591
|
-
const a = measureAt(repo, head, { ...cfg, metrics: defs }, repeat);
|
|
592
|
-
for (const w of weak) {
|
|
593
|
-
const ev = evaluate(w.def, b[w.name], a[w.name], repeat);
|
|
594
|
-
if (!ev.ok) weakened.push({ name: w.name, change: w.change, status: ev.status });
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
gateInfo = {
|
|
598
|
-
fingerprint: {
|
|
599
|
-
base: baseGate ? gateFingerprint([...baseGate.metrics.values()], baseGate.scalars) : null,
|
|
600
|
-
head: gateFingerprint(cfg.metrics, cfg),
|
|
601
|
-
},
|
|
602
|
-
...(weakened.length ? { weakened } : {}),
|
|
603
|
-
...(evalChanged.length ? { evaluatorChanged: evalChanged } : {}),
|
|
604
|
-
};
|
|
605
|
-
}
|
|
606
|
-
// declared override: a Sawbuck-Allow trailer (or --allow) acknowledges an intentional
|
|
643
|
+
// Section 3 (gaming detection), on by default: mutates `metrics` and yields the gate record
|
|
644
|
+
const gateInfo = opts.detectGaming ? detectGamingPass(repo, base, head, cfg, metrics, repeat) : null;
|
|
645
|
+
// declared override: a Promptwheel-Allow trailer (or --allow) acknowledges an intentional
|
|
607
646
|
// goalpost/gate change, so a would-be GAMED becomes PASS, recorded in report.allowed for review.
|
|
608
647
|
const allow = opts.allow ? { reason: String(opts.allow).slice(0, 300) } : (opts.working ? null : readAllowTrailer(repo, head));
|
|
609
648
|
let allowed = null;
|
|
@@ -616,6 +655,12 @@ function gate(repo, opts) {
|
|
|
616
655
|
if (gw) gateInfo.allowed = true;
|
|
617
656
|
}
|
|
618
657
|
}
|
|
658
|
+
// mandates run first: a protected-path hard rule fails the gate no matter what the metrics say.
|
|
659
|
+
// The base ref's mandates count too (resolved through `extends` by gateDefAtRef), so a change
|
|
660
|
+
// cannot clear a mandate by deleting it in the same diff that violates it.
|
|
661
|
+
const baseMandates = (gateInfo && gateInfo.baseMandates) || (!opts.detectGaming ? ((gateDefAtRef(repo, base) || {}).scalars || {}).mandates : null);
|
|
662
|
+
const mandateViolations = checkMandates(repo, base, head, cfg, baseMandates);
|
|
663
|
+
const mandated = mandateViolations.length > 0;
|
|
619
664
|
const failed = metrics.some((m) => m.guard && !m.ok);
|
|
620
665
|
const gamed = metrics.some((m) => m.gamed === true) || !!(gateInfo && gateInfo.weakened && !gateInfo.allowed);
|
|
621
666
|
// a guard that never runs (inert: a pass/fail metric stuck at 0 across both refs) verifies
|
|
@@ -623,15 +668,11 @@ function gate(repo, opts) {
|
|
|
623
668
|
// BUT scope it to a bare no-op run: an inert guard must NOT bury a genuine, measured win
|
|
624
669
|
// elsewhere (that would make `improve` revert good changes and misread them as a plateau).
|
|
625
670
|
const inconclusive = !failed && !gamed && metrics.some((m) => m.inert) && !metrics.some((m) => m.status === 'improved');
|
|
626
|
-
const verdict = failed ? 'fail' : gamed ? 'gamed' : inconclusive ? 'inconclusive' : 'pass';
|
|
671
|
+
const verdict = (mandated || failed) ? 'fail' : gamed ? 'gamed' : inconclusive ? 'inconclusive' : 'pass';
|
|
627
672
|
const report = {
|
|
628
673
|
base: short(repo, base), head: short(repo, head), repeat, mode: opts.working ? 'working' : 'refs',
|
|
629
|
-
// learning-substrate fields (Phase 5): cohort segments reliability by environment,
|
|
630
|
-
// label attributes a change-type, subsystems fingerprint WHERE the change landed.
|
|
631
|
-
cohort: opts.cohort ?? (process.env.CI ? 'ci' : 'local'),
|
|
632
|
-
...(opts.label ? { label: opts.label } : {}),
|
|
633
|
-
subsystems: subsystemsOf(repo, base, head, cfg),
|
|
634
674
|
verdict, metrics,
|
|
675
|
+
...(mandateViolations.length ? { mandates: mandateViolations } : {}),
|
|
635
676
|
...(gateInfo ? { gate: gateInfo } : {}),
|
|
636
677
|
...(allowed ? { allowed } : {}),
|
|
637
678
|
};
|
|
@@ -655,7 +696,7 @@ function run(argv) {
|
|
|
655
696
|
config = autoConfig(repo);
|
|
656
697
|
console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd} (promptwheel init to customize; --no-auto to require a config)`);
|
|
657
698
|
}
|
|
658
|
-
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming,
|
|
699
|
+
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, allow: args.allow, config });
|
|
659
700
|
if (args.json) console.log(JSON.stringify(report, null, 2));
|
|
660
701
|
else if (args.markdown) console.log(renderMarkdown(report));
|
|
661
702
|
else printHuman(report);
|
|
@@ -674,24 +715,25 @@ function recordOutcome(repo, report) {
|
|
|
674
715
|
// keep-if-improved: run any agent/script, gate the result, keep the change ONLY if a metric improved
|
|
675
716
|
function improve(argv) {
|
|
676
717
|
const args = parseArgs(argv);
|
|
677
|
-
if (!args.attempt)
|
|
718
|
+
if (!args.attempt) die('improve requires --attempt "<command that changes the repo>"');
|
|
678
719
|
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
679
720
|
const dirty = git(['status', '--porcelain'], repo).split('\n').filter((l) => l.trim() && !l.includes('.promptwheel'));
|
|
680
|
-
if (dirty.length)
|
|
721
|
+
if (dirty.length) die('working tree not clean, commit or stash first (improve needs a clean base to revert to)');
|
|
681
722
|
|
|
682
723
|
console.error(`▶ attempt: ${args.attempt}`);
|
|
683
724
|
try { execSync(args.attempt, { cwd: repo, stdio: 'inherit' }); }
|
|
684
725
|
catch (e) { console.error(` (attempt exited ${e.status ?? 1}, gating whatever it changed)`); }
|
|
685
726
|
|
|
686
|
-
const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming
|
|
727
|
+
const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
|
|
687
728
|
const noChange = report.metrics.every((m) => m.delta === 0 || m.delta == null);
|
|
688
729
|
const improvedNames = report.metrics.filter((m) => m.status === 'improved').map((m) => m.name);
|
|
689
730
|
|
|
690
|
-
// result + exit code express loop progress so `while improve; do :; done` converges
|
|
691
|
-
//
|
|
731
|
+
// result + exit code express loop progress so `while improve; do :; done` converges, with the
|
|
732
|
+
// same codes as `run`: 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed
|
|
733
|
+
// (reverted) · 3 = plateau/no-op OR inconclusive (reverted)
|
|
692
734
|
let result, exit, note;
|
|
693
735
|
if (report.verdict === 'fail') { result = 'regression'; exit = 1; revert(repo); note = '✗ guarded regression, reverted'; }
|
|
694
|
-
else if (report.verdict === 'gamed') { result = 'gamed'; exit =
|
|
736
|
+
else if (report.verdict === 'gamed') { result = 'gamed'; exit = 2; revert(repo); note = '🚩 gamed, a metric "improved" by editing tests/config, not source, reverted'; }
|
|
695
737
|
else if (report.verdict === 'inconclusive') { result = 'inconclusive'; exit = 3; revert(repo); note = '≈ inconclusive, a guard measured nothing (inert); reverted'; }
|
|
696
738
|
else if (noChange || improvedNames.length === 0) {
|
|
697
739
|
result = 'plateau'; exit = 3; revert(repo);
|
|
@@ -717,14 +759,14 @@ function revert(repo) {
|
|
|
717
759
|
try { git(['clean', '-fd', '-e', '.promptwheel'], repo); } catch { /* best effort */ }
|
|
718
760
|
}
|
|
719
761
|
|
|
720
|
-
//
|
|
721
|
-
//
|
|
722
|
-
//
|
|
762
|
+
// insights: honest aggregation of the per-repo outcome record (.promptwheel/outcomes.jsonl)
|
|
763
|
+
// into per-metric counts. No model, no decay, no claims: just which metrics actually respond.
|
|
764
|
+
// The reader is tolerant of older records that carry extra fields (cohort, label, subsystems).
|
|
723
765
|
function insights(argv) {
|
|
724
766
|
const args = parseArgs(argv);
|
|
725
767
|
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
726
768
|
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
727
|
-
if (!existsSync(f))
|
|
769
|
+
if (!existsSync(f)) die('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)');
|
|
728
770
|
const runs = readFileSync(f, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
729
771
|
const agg = {};
|
|
730
772
|
for (const r of runs) for (const m of (r.metrics || [])) {
|
|
@@ -746,149 +788,6 @@ function insights(argv) {
|
|
|
746
788
|
console.log(' highest-lever metrics are where an agent loop should spend its attempts.\n');
|
|
747
789
|
}
|
|
748
790
|
|
|
749
|
-
// ---------------------------------------------------------------------------
|
|
750
|
-
// Phase 5, outcome-curated playbook + UCB work-discovery.
|
|
751
|
-
// Unfrozen 2026-07-02 by explicit founder decision overriding D7 (see docs/LEARNING.md).
|
|
752
|
-
// Design constraints that survive the unfreeze:
|
|
753
|
-
// - PURE VIEW: the append-only ledger (.promptwheel/outcomes.jsonl) is the only store;
|
|
754
|
-
// the playbook is re-derived on every read (decay at read time, no curator state to rot).
|
|
755
|
-
// - EVIDENCE-GATED: a key renders a claim only past MIN_MOVED weighted observations;
|
|
756
|
-
// everything below that is counted, not asserted.
|
|
757
|
-
// - CLAIM-GATED: no public compounding claim until bench/compounding-ab.mjs passes on
|
|
758
|
-
// real usage data. The gate on the CLAIM outlived the gate on the CODE.
|
|
759
|
-
// ---------------------------------------------------------------------------
|
|
760
|
-
const HALF_LIFE = 20; // runs, an entry's weight halves every HALF_LIFE runs unless re-earned
|
|
761
|
-
const MIN_MOVED = 3; // weighted moved-observations before a key earns a rendered claim
|
|
762
|
-
const EXPLORE_C = 1.0; // UCB exploration constant
|
|
763
|
-
|
|
764
|
-
function readLedger(repo) {
|
|
765
|
-
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
766
|
-
if (!existsSync(f)) return [];
|
|
767
|
-
return readFileSync(f, 'utf8').split('\n').filter(Boolean)
|
|
768
|
-
.map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
// fold the ledger into decayed per-key stats. keys: `metric` · `metric @ subsystem` · `metric # label`
|
|
772
|
-
function foldOutcomes(runs) {
|
|
773
|
-
const keys = new Map();
|
|
774
|
-
const N = runs.length;
|
|
775
|
-
runs.forEach((r, idx) => {
|
|
776
|
-
const w = Math.pow(0.5, (N - 1 - idx) / HALF_LIFE); // idx N-1 = newest → weight 1
|
|
777
|
-
for (const m of (r.metrics || [])) {
|
|
778
|
-
const tags = [m.name];
|
|
779
|
-
for (const s of (r.subsystems || [])) tags.push(`${m.name} @ ${s}`);
|
|
780
|
-
if (r.label) tags.push(`${m.name} # ${r.label}`);
|
|
781
|
-
for (const key of tags) {
|
|
782
|
-
const k = keys.get(key) ?? { key, metric: m.name, w: 0, n: 0, imp: 0, reg: 0, moved: 0, effects: [], cohorts: {}, last: null };
|
|
783
|
-
k.n += 1; k.w += w;
|
|
784
|
-
const coh = (k.cohorts[r.cohort || 'unknown'] ??= { imp: 0, reg: 0 });
|
|
785
|
-
if (m.status === 'improved') {
|
|
786
|
-
k.imp += w; k.moved += w; coh.imp += 1;
|
|
787
|
-
if (typeof m.delta === 'number' && m.delta !== 0) k.effects.push(Math.abs(m.delta));
|
|
788
|
-
} else if (m.status === 'regressed') { k.reg += w; k.moved += w; coh.reg += 1; }
|
|
789
|
-
k.last = r.ts || k.last;
|
|
790
|
-
keys.set(key, k);
|
|
791
|
-
}
|
|
792
|
-
}
|
|
793
|
-
});
|
|
794
|
-
return [...keys.values()];
|
|
795
|
-
}
|
|
796
|
-
|
|
797
|
-
const betaMean = (imp, reg) => (1 + imp) / (2 + imp + reg); // Beta(1,1)-smoothed helpfulness
|
|
798
|
-
|
|
799
|
-
// composite lever = smoothed helpfulness × responsiveness; UCB adds an exploration bonus
|
|
800
|
-
function scoreKey(k, totalW) {
|
|
801
|
-
const p = betaMean(k.imp, k.reg);
|
|
802
|
-
const move = k.w ? k.moved / k.w : 0;
|
|
803
|
-
const lever = p * move;
|
|
804
|
-
const ucb = lever + EXPLORE_C * Math.sqrt(Math.log(Math.max(Math.E, totalW)) / Math.max(1e-6, k.w));
|
|
805
|
-
return { p: +p.toFixed(3), move: +move.toFixed(3), lever: +lever.toFixed(3), ucb: +ucb.toFixed(3), effect: +(median(k.effects) ?? 0).toFixed(6) };
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
// cohorts that disagree in sign are flagged, never averaged into a lie
|
|
809
|
-
function cohortNote(k) {
|
|
810
|
-
const cs = Object.entries(k.cohorts).filter(([, c]) => c.imp + c.reg > 0);
|
|
811
|
-
if (cs.length < 2) return '';
|
|
812
|
-
const signs = new Set(cs.map(([, c]) => Math.sign(c.imp - c.reg)));
|
|
813
|
-
return signs.size > 1 ? ' · ⚠ cohort-dependent (ci vs local disagree)' : '';
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
function loadEntries(repo) {
|
|
817
|
-
const runs = readLedger(repo);
|
|
818
|
-
if (!runs.length) { console.error('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)'); process.exit(2); }
|
|
819
|
-
const keys = foldOutcomes(runs);
|
|
820
|
-
const totalW = keys.filter((k) => k.key === k.metric).reduce((s, k) => s + k.w, 0);
|
|
821
|
-
const entries = keys.map((k) => ({ ...k, ...scoreKey(k, totalW), sufficient: k.moved >= MIN_MOVED }))
|
|
822
|
-
.sort((a, b) => b.lever - a.lever || b.w - a.w);
|
|
823
|
-
return { runs, entries };
|
|
824
|
-
}
|
|
825
|
-
|
|
826
|
-
// the distilled, execution-earned context an agent (or CLAUDE.md) can consume
|
|
827
|
-
function playbook(argv) {
|
|
828
|
-
const args = parseArgs(argv);
|
|
829
|
-
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
830
|
-
const { runs, entries } = loadEntries(repo);
|
|
831
|
-
if (args.json) { console.log(JSON.stringify({ runs: runs.length, halfLife: HALF_LIFE, minMoved: MIN_MOVED, entries }, null, 2)); return; }
|
|
832
|
-
const suff = entries.filter((e) => e.sufficient);
|
|
833
|
-
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
834
|
-
const out = [];
|
|
835
|
-
out.push(`## Earned playbook, ${runs.length} gated runs, decay half-life ${HALF_LIFE} runs`);
|
|
836
|
-
out.push('*(every line below was measured by the gate, not asserted; entries decay unless re-earned)*', '');
|
|
837
|
-
if (!suff.length) {
|
|
838
|
-
out.push(`_Nothing has earned a claim yet, a key needs ≥${MIN_MOVED} weighted moved-observations. Keep gating; ${entries.length} keys are accumulating._`);
|
|
839
|
-
} else {
|
|
840
|
-
for (const e of suff.slice(0, 20)) {
|
|
841
|
-
out.push(`- **${e.key}**, helpful ${pct(e.p)} when it moves (${e.imp.toFixed(1)}✓/${e.reg.toFixed(1)}✗ weighted), responds in ${pct(e.move)} of runs, median effect ${e.effect}${cohortNote(e)}`);
|
|
842
|
-
}
|
|
843
|
-
const hidden = entries.length - suff.length;
|
|
844
|
-
if (hidden > 0) out.push('', `_${hidden} more keys below the evidence threshold, counted, not asserted._`);
|
|
845
|
-
}
|
|
846
|
-
console.log(out.join('\n'));
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
// UCB work-discovery: where should the loop spend its NEXT attempt? Proposes measured
|
|
850
|
-
// targets (metric/subsystem arms), never code advice; the measurement stays the message.
|
|
851
|
-
function suggest(argv) {
|
|
852
|
-
const args = parseArgs(argv);
|
|
853
|
-
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
854
|
-
const { runs, entries } = loadEntries(repo);
|
|
855
|
-
const arms = entries.filter((e) => e.key === e.metric || e.key.includes(' @ ')).sort((a, b) => b.ucb - a.ucb);
|
|
856
|
-
if (args.json) { console.log(JSON.stringify({ runs: runs.length, thin: runs.length < 10, arms: arms.slice(0, 10) }, null, 2)); return; }
|
|
857
|
-
console.log(`\nPromptWheel suggest, UCB over ${runs.length} gated runs${runs.length < 10 ? ' (thin record: exploration dominates, treat as a coin with opinions)' : ''}\n`);
|
|
858
|
-
for (const a of arms.slice(0, 5)) {
|
|
859
|
-
console.log(` ${a.ucb.toFixed(3)} ${a.key.padEnd(28)} lever ${a.lever.toFixed(3)} (helpful ${a.p}, responds ${a.move}) · evidence ${a.moved.toFixed(1)} moved / ${a.n} runs${a.sufficient ? '' : ' · below claim threshold'}`);
|
|
860
|
-
}
|
|
861
|
-
console.log('\n score = lever + exploration bonus, high scores are either proven levers or under-explored arms.\n');
|
|
862
|
-
}
|
|
863
|
-
|
|
864
|
-
// backfill, seed the ledger from git history: replay past commits through the CURRENT
|
|
865
|
-
// metrics (LEARNING.md harvest path 1). Deterministic, no LLM. Rows are cohort-tagged
|
|
866
|
-
// 'backfill', historical human commits are NOT live agent-loop evidence; the cohort
|
|
867
|
-
// machinery segments them and flags disagreement rather than averaging it away. The
|
|
868
|
-
// conventional-commit type (fix/feat/refactor/…) becomes the change-type label for free.
|
|
869
|
-
function backfill(argv) {
|
|
870
|
-
const args = parseArgs(argv);
|
|
871
|
-
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
872
|
-
const listArgs = ['rev-list', '--no-merges'];
|
|
873
|
-
if (args.since) listArgs.push(`${args.since}..HEAD`); else listArgs.push('-n', String(args.n ?? 30), 'HEAD');
|
|
874
|
-
const commits = git(listArgs, repo).split('\n').filter(Boolean).reverse(); // oldest first → ledger stays chronological, decay stays honest
|
|
875
|
-
if (!commits.length) { console.error('no commits to backfill'); process.exit(2); }
|
|
876
|
-
const seen = new Set(readLedger(repo).filter((r) => r.cohort === 'backfill').map((r) => r.head));
|
|
877
|
-
let done = 0, skipped = 0;
|
|
878
|
-
console.log(`\nbackfilling ${commits.length} commits through the current metrics (gaming detection ${args.dgExplicit && args.detectGaming ? 'ON' : 'off, pass --detect-gaming to audit history too'})\n`);
|
|
879
|
-
for (const c of commits) {
|
|
880
|
-
try { git(['rev-parse', `${c}~1`], repo); } catch { skipped++; continue; } // root commit
|
|
881
|
-
if (seen.has(short(repo, c))) { skipped++; continue; } // already recorded
|
|
882
|
-
const subj = git(['log', '-1', '--format=%s', c], repo);
|
|
883
|
-
const type = /^(feat|fix|docs|test|chore|refactor|perf|ci|build|style)\b/i.exec(subj)?.[1]?.toLowerCase();
|
|
884
|
-
const report = gate(repo, { base: `${c}~1`, head: c, repeat: args.repeat, detectGaming: args.dgExplicit ? args.detectGaming : false, label: type, cohort: 'backfill' });
|
|
885
|
-
console.log(` ${report.head} ${report.verdict.toUpperCase().padEnd(6)} ${type ? `#${type}`.padEnd(10) : ''.padEnd(10)} ${subj.slice(0, 56)}`);
|
|
886
|
-
done++;
|
|
887
|
-
}
|
|
888
|
-
console.log(`\n backfilled ${done}${skipped ? ` (skipped ${skipped}: root or already recorded)` : ''}, old commits that no longer build record as unmeasurable, never faked.`);
|
|
889
|
-
console.log(' next: promptwheel playbook · promptwheel suggest\n');
|
|
890
|
-
}
|
|
891
|
-
|
|
892
791
|
// ---------------------------------------------------------------------------
|
|
893
792
|
// init, write a starter config so a newcomer isn't staring at a blank page
|
|
894
793
|
// ---------------------------------------------------------------------------
|
|
@@ -993,15 +892,13 @@ function init(argv) {
|
|
|
993
892
|
}
|
|
994
893
|
const repo = (() => { try { return git(['rev-parse', '--show-toplevel'], process.cwd()); } catch { return process.cwd(); } })();
|
|
995
894
|
const out = join(repo, 'promptwheel.config.json');
|
|
996
|
-
if (existsSync(out) && !argv.includes('--force'))
|
|
997
|
-
console.error('promptwheel.config.json already exists, pass --force to overwrite'); process.exit(2);
|
|
998
|
-
}
|
|
895
|
+
if (existsSync(out) && !argv.includes('--force')) die('promptwheel.config.json already exists, pass --force to overwrite');
|
|
999
896
|
const pi = argv.indexOf('--preset');
|
|
1000
897
|
const presetName = pi >= 0 ? argv[pi + 1] : null;
|
|
1001
898
|
let metrics, note, evaluatorPaths;
|
|
1002
899
|
if (presetName) {
|
|
1003
900
|
const p = PRESETS[presetName];
|
|
1004
|
-
if (!p)
|
|
901
|
+
if (!p) die(`unknown preset "${presetName}", try: promptwheel init --list`);
|
|
1005
902
|
evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (e.g. security-antihack)
|
|
1006
903
|
metrics = p.metrics || [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }];
|
|
1007
904
|
metrics = metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
|
|
@@ -1034,6 +931,7 @@ function init(argv) {
|
|
|
1034
931
|
if (envCfg.linkDirs) note += ` · linking ${envCfg.linkDirs.join(', ')}`;
|
|
1035
932
|
writeFileSync(out, JSON.stringify({ repeat: 1, ...envCfg, ...(evaluatorPaths ? { evaluatorPaths } : {}), metrics }, null, 2) + '\n');
|
|
1036
933
|
console.log(`✓ wrote promptwheel.config.json, ${note}`);
|
|
934
|
+
warnEditableInstall(repo); // one-time advice at setup, never on the per-turn path
|
|
1037
935
|
if (!argv.includes('--no-agent-block')) {
|
|
1038
936
|
const wrote = writeAgentBlock(repo);
|
|
1039
937
|
if (wrote.length) console.log(`✓ wrote the agent guide (when to run the gate) to ${wrote.join(', ')}`);
|
|
@@ -1095,6 +993,7 @@ const short = (repo, ref) => { try { return git(['rev-parse', '--short', ref], r
|
|
|
1095
993
|
function printHuman(r) {
|
|
1096
994
|
const arrowFor = (m) => (m.delta == null ? '?' : m.delta > 0 ? '▲' : m.delta < 0 ? '▼' : '=');
|
|
1097
995
|
console.log(`\nPromptWheel ${r.base} → ${r.head}${r.repeat > 1 ? ` (×${r.repeat})` : ''}\n`);
|
|
996
|
+
if (r.mandates) for (const md of r.mandates) console.log(` ⛔ MANDATE "${md.name}" violated${md.reason ? ': ' + md.reason : ''} (touched: ${md.files.join(', ')})${md.removed ? ' [deleted in this same change; a mandate stays in force for the change that removes it]' : ''}`);
|
|
1098
997
|
for (const m of r.metrics) {
|
|
1099
998
|
const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
|
|
1100
999
|
const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
|
|
@@ -1103,9 +1002,12 @@ function printHuman(r) {
|
|
|
1103
1002
|
if (m.gamed === true) console.log(` 🚩 GAMED, ${m.gamingReason}`);
|
|
1104
1003
|
}
|
|
1105
1004
|
if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(` 🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : 'disabled'} in this change and it would have ${w.status} at head (goalpost moved, not earned)`);
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1005
|
+
if (r.gate && r.gate.changed) console.log(` ⚠ gate definition changed in this change (fingerprint ${r.gate.fingerprint.base} -> ${r.gate.fingerprint.head}): a metric, guard, evaluatorPath or mandate was added, removed or edited; review promptwheel.config.json as part of the diff`);
|
|
1006
|
+
const failReason = r.mandates ? ', a mandate was violated (a protected path changed)' : ', a guarded metric regressed (beyond noise)';
|
|
1007
|
+
console.log(`\n VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? failReason : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
|
|
1008
|
+
if (r.verdict === 'fail' && r.mandates) console.log(' a mandate is a hard rule; if the change is truly intended, edit the mandate in promptwheel.config.json (a tracked, tamper-evident change), do not work around it');
|
|
1009
|
+
else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name), `promptwheel guards` shows the effective set');
|
|
1010
|
+
if (r.allowed) console.log(` ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Promptwheel-Allow: ${r.allowed.reason})`);
|
|
1109
1011
|
console.log('');
|
|
1110
1012
|
}
|
|
1111
1013
|
|
|
@@ -1123,12 +1025,14 @@ function renderMarkdown(r) {
|
|
|
1123
1025
|
'',
|
|
1124
1026
|
`\`${r.base} → ${r.head}\` · ${r.mode} mode${r.repeat > 1 ? ` · ×${r.repeat}` : ''}`,
|
|
1125
1027
|
'',
|
|
1028
|
+
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **Mandate violated:** \`${md.name}\`${md.reason ? ': ' + md.reason : ''} (touched \`${md.files.join('`, `')}\`)${md.removed ? ' (deleted in this same change; it stays in force for the change that removes it)' : ''}`).join('\n'), ''] : []),
|
|
1126
1029
|
'| metric | before | after | Δ | status | confidence |',
|
|
1127
1030
|
'|---|--:|--:|--:|---|---|',
|
|
1128
1031
|
rows,
|
|
1129
1032
|
'',
|
|
1130
|
-
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? '> ❌ A 🛡️ guarded metric regressed beyond the noise band.' : r.verdict === 'inconclusive' ? '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.' : '> ✅ No guarded metric regressed.',
|
|
1033
|
+
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A mandate was violated: a protected path changed. This is a hard rule, enforced before any metric.' : '> ❌ A 🛡️ guarded metric regressed beyond the noise band.') : r.verdict === 'inconclusive' ? '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.' : '> ✅ No guarded metric regressed.',
|
|
1131
1034
|
...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : 'disabled'} (would have ${w.status} at head)`).join('; ')}. The pass was bought by dropping a guard, not by earning it.`] : []),
|
|
1035
|
+
...(r.gate && r.gate.changed ? ['', `> ⚠ **Gate definition changed** in this diff (fingerprint \`${r.gate.fingerprint.base}\` -> \`${r.gate.fingerprint.head}\`): a metric, guard, evaluatorPath or mandate was added, removed or edited. Review the config change with the code.`] : []),
|
|
1132
1036
|
'',
|
|
1133
1037
|
'<sub>🛡️ = guard · _prove every change moved a metric_ · [PromptWheel](https://github.com/promptwheel-ai/promptwheel)</sub>',
|
|
1134
1038
|
].join('\n');
|
|
@@ -1144,21 +1048,18 @@ function parseArgs(argv) {
|
|
|
1144
1048
|
else if (argv[i] === '--no-record') a.noRecord = true;
|
|
1145
1049
|
else if (argv[i] === '--no-auto') a.noAuto = true;
|
|
1146
1050
|
else if (argv[i] === '--allow') a.allow = argv[++i];
|
|
1147
|
-
else if (argv[i] === '--detect-gaming'
|
|
1148
|
-
else if (argv[i] === '--no-detect-gaming'
|
|
1149
|
-
else if (argv[i] === '--since') a.since = argv[++i];
|
|
1150
|
-
else if (argv[i] === '-n' || argv[i] === '--limit') a.n = parseInt(argv[++i], 10);
|
|
1051
|
+
else if (argv[i] === '--detect-gaming') a.detectGaming = true;
|
|
1052
|
+
else if (argv[i] === '--no-detect-gaming') a.detectGaming = false;
|
|
1151
1053
|
else if (argv[i] === '--attempt') a.attempt = argv[++i];
|
|
1152
|
-
else if (argv[i] === '--label') a.label = argv[++i];
|
|
1153
1054
|
else if (argv[i] === '--json') a.json = true;
|
|
1154
|
-
else if (argv[i] === '--markdown'
|
|
1055
|
+
else if (argv[i] === '--markdown') a.markdown = true;
|
|
1155
1056
|
}
|
|
1156
1057
|
return a;
|
|
1157
1058
|
}
|
|
1158
1059
|
|
|
1159
1060
|
// pure, side-effect-free helpers are exported for unit testing; the CLI below only
|
|
1160
1061
|
// runs when this file is executed directly (not when imported by the test suite).
|
|
1161
|
-
export { extract, evaluate, median, spread, renderMarkdown, isNonSource, evaluatorMatcher, gain, judgeGaming,
|
|
1062
|
+
export { extract, evaluate, median, spread, renderMarkdown, isNonSource, evaluatorMatcher, gain, judgeGaming, verdictFingerprint };
|
|
1162
1063
|
|
|
1163
1064
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
1164
1065
|
// Public library API: the Verified-Outcome Contract, in process.
|
|
@@ -1201,8 +1102,7 @@ export function verdictExitCode(verdict) {
|
|
|
1201
1102
|
// auto boolean allow the zero-config auto-detect fallback (default true; false throws if no config)
|
|
1202
1103
|
// repeat number measurement repeats for the noise band (default: config.repeat ?? 1)
|
|
1203
1104
|
// detectGaming boolean run the Section 3 source-only re-proof (default: true)
|
|
1204
|
-
//
|
|
1205
|
-
// label string change-type label
|
|
1105
|
+
// allow string a declared override reason (same as --allow / a Promptwheel-Allow: trailer)
|
|
1206
1106
|
// record boolean append the run to .promptwheel/outcomes.jsonl (default: false)
|
|
1207
1107
|
export function check(options = {}) {
|
|
1208
1108
|
let repo;
|
|
@@ -1227,8 +1127,6 @@ export function check(options = {}) {
|
|
|
1227
1127
|
config,
|
|
1228
1128
|
repeat: options.repeat,
|
|
1229
1129
|
detectGaming: options.detectGaming ?? true,
|
|
1230
|
-
cohort: options.cohort,
|
|
1231
|
-
label: options.label,
|
|
1232
1130
|
allow: options.allow,
|
|
1233
1131
|
noRecord: options.record !== true,
|
|
1234
1132
|
});
|
|
@@ -1283,6 +1181,7 @@ export function conformsToContract(report) {
|
|
|
1283
1181
|
if (!report.gate || typeof report.gate !== 'object' || Array.isArray(report.gate)) E('gate must be an object');
|
|
1284
1182
|
else {
|
|
1285
1183
|
if (report.gate.fingerprint !== undefined && (typeof report.gate.fingerprint !== 'object' || report.gate.fingerprint === null)) E('gate.fingerprint must be an object { base, head }');
|
|
1184
|
+
if (report.gate.changed !== undefined && typeof report.gate.changed !== 'boolean') E('gate.changed must be a boolean');
|
|
1286
1185
|
if (report.gate.weakened !== undefined && !Array.isArray(report.gate.weakened)) E('gate.weakened must be an array');
|
|
1287
1186
|
}
|
|
1288
1187
|
}
|
|
@@ -1363,40 +1262,40 @@ function conformance(argv) {
|
|
|
1363
1262
|
|
|
1364
1263
|
function main() {
|
|
1365
1264
|
const [cmd, ...rest] = process.argv.slice(2);
|
|
1265
|
+
if (cmd === '--version' || cmd === '-v') { console.log(TOOL_VERSION ?? 'unknown'); process.exit(0); }
|
|
1366
1266
|
if (cmd === 'run') run(rest);
|
|
1367
1267
|
else if (cmd === 'improve') improve(rest);
|
|
1368
1268
|
else if (cmd === 'insights') insights(rest);
|
|
1369
|
-
else if (cmd === 'playbook') playbook(rest);
|
|
1370
|
-
else if (cmd === 'suggest') suggest(rest);
|
|
1371
|
-
else if (cmd === 'backfill') backfill(rest);
|
|
1372
1269
|
else if (cmd === 'init') init(rest);
|
|
1373
1270
|
else if (cmd === 'guards') guards(rest);
|
|
1374
1271
|
else if (cmd === 'conformance') conformance(rest);
|
|
1375
1272
|
else {
|
|
1273
|
+
const help = !cmd || cmd === '--help' || cmd === '-h' || cmd === 'help';
|
|
1274
|
+
if (!help) console.error(`unknown command: ${cmd}`);
|
|
1376
1275
|
console.log([
|
|
1377
|
-
'PromptWheel,
|
|
1276
|
+
'PromptWheel: a deterministic, zero-LLM check for your coding loop. It proves every change earned its win by re-running the result from the agent\'s source edits alone, so a green that came from editing the test reads GAMED.',
|
|
1378
1277
|
'',
|
|
1278
|
+
' promptwheel --version | -v print the version',
|
|
1279
|
+
' promptwheel --help | -h this banner',
|
|
1379
1280
|
' promptwheel init [--preset <name> | --list] [--no-agent-block] write a starter config + an AGENTS.md guide so your agent runs the gate (OPTIONAL: run works zero-config, auto-detecting your test command)',
|
|
1380
1281
|
' promptwheel run [--base R] [--head R] [--repeat N] [--json|--markdown]',
|
|
1381
1282
|
' catches reward-hacking BY DEFAULT, re-proves each win from SOURCE edits alone (verdict GAMED,',
|
|
1382
1283
|
' exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
|
|
1383
1284
|
' promptwheel run --working gate uncommitted changes (incl. newly added files)',
|
|
1384
|
-
' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `
|
|
1285
|
+
' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `Promptwheel-Allow: <reason>` commit trailer; `Sawbuck-Allow:` is an alias): recorded, not flagged GAMED',
|
|
1385
1286
|
' promptwheel improve --attempt "<cmd>" run an agent/script; keep only if a metric improved',
|
|
1386
|
-
' exit 0=kept · 1=regression · 3=plateau · add --json',
|
|
1387
|
-
' promptwheel insights
|
|
1388
|
-
' promptwheel
|
|
1389
|
-
' promptwheel suggest [--json] UCB work-discovery: where the next attempt should go (experimental)',
|
|
1390
|
-
' promptwheel backfill [-n N | --since <ref>] seed the ledger from git history (cohort-tagged; commit types become labels)',
|
|
1391
|
-
' promptwheel guards show the effective guardrails (incl. inherited) + flag record',
|
|
1287
|
+
' exit 0=kept · 1=regression · 2=gamed · 3=plateau/inconclusive (all but 0 reverted) · add --json',
|
|
1288
|
+
' promptwheel insights [--json] which metrics actually respond (raw counts from .promptwheel/outcomes.jsonl)',
|
|
1289
|
+
' promptwheel guards [--json] show the effective guardrails (incl. inherited) + flag record',
|
|
1392
1290
|
' promptwheel conformance [file|-] [--cmd C] check a verdict JSON (or a command) against the Contract (exit 0 = conforms)',
|
|
1393
1291
|
'',
|
|
1292
|
+
'Exit codes: 0 pass · 1 fail (a guarded metric regressed, or a mandate was violated) · 2 gamed · 3 inconclusive · 64 usage or config error (never a verdict).',
|
|
1394
1293
|
'Integrate (library / conform / shell / MCP): see INTEGRATE.md. Import: `import { check } from \'promptwheel\'`.',
|
|
1395
1294
|
'Loop it: while promptwheel improve --attempt "$AGENT"; do :; done # stops on plateau/regression',
|
|
1396
1295
|
'Config: promptwheel.config.json → { extends?, metrics:[{ name, cmd, direction, extract?, guard? }] } (or: promptwheel init)',
|
|
1397
1296
|
' extends: a path (or array) to a shared base config, repos INHERIT its guardrails; local metrics override by name.',
|
|
1398
1297
|
].join('\n'));
|
|
1399
|
-
process.exit(
|
|
1298
|
+
process.exit(help ? 0 : EX_USAGE);
|
|
1400
1299
|
}
|
|
1401
1300
|
}
|
|
1402
1301
|
|