promptwheel 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INTEGRATE.md +32 -42
- package/README.md +95 -181
- package/bin/promptwheel.d.ts +306 -0
- package/bin/promptwheel.mjs +165 -287
- package/package.json +8 -6
- package/packs/security/README.md +16 -6
package/bin/promptwheel.mjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// PromptWheel,
|
|
2
|
+
// PromptWheel: a deterministic, zero-LLM check for your coding loop. It proves every change
|
|
3
|
+
// earned its win by re-running the result from the agent's source edits alone, so a green
|
|
4
|
+
// that came from editing the test reads GAMED.
|
|
3
5
|
//
|
|
4
|
-
// For any change (base ref
|
|
6
|
+
// For any change (base ref -> head ref), measure each configured metric in an
|
|
5
7
|
// isolated git worktree BEFORE and AFTER, enforce regression guards, and refuse
|
|
6
8
|
// to trust a delta that sits inside the measurement noise band. Emits a
|
|
7
9
|
// structured {verdict, metric, delta, confidence}. The thing base agents don't do:
|
|
@@ -29,9 +31,14 @@ const median = (xs) => {
|
|
|
29
31
|
};
|
|
30
32
|
const spread = (xs) => { const a = xs.filter((x) => x != null); return a.length ? Math.max(...a) - Math.min(...a) : 0; };
|
|
31
33
|
|
|
34
|
+
// usage / config errors exit 64 (EX_USAGE), so they can never be confused with a verdict
|
|
35
|
+
// (0 pass, 1 fail, 2 gamed, 3 inconclusive): in CI a config typo must not read as GAMED.
|
|
36
|
+
const EX_USAGE = 64;
|
|
37
|
+
function die(msg) { console.error(msg); process.exit(EX_USAGE); }
|
|
38
|
+
|
|
32
39
|
function readConfigFile(p) {
|
|
33
40
|
try { return JSON.parse(readFileSync(p, 'utf8')); }
|
|
34
|
-
catch (e) {
|
|
41
|
+
catch (e) { die(`invalid config ${p}: ${e.message}`); }
|
|
35
42
|
}
|
|
36
43
|
|
|
37
44
|
const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { return p; } };
|
|
@@ -41,7 +48,7 @@ const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { retu
|
|
|
41
48
|
// a repo INHERITS their guardrails, and a local metric of the same name overrides the inherited one.
|
|
42
49
|
function resolveConfig(p, repo, label, seen) {
|
|
43
50
|
const abs = realpathSync(p);
|
|
44
|
-
if (seen.has(abs))
|
|
51
|
+
if (seen.has(abs)) die(`config inheritance cycle at ${relOf(repo, p)}`);
|
|
45
52
|
seen.add(abs);
|
|
46
53
|
try {
|
|
47
54
|
const cfg = readConfigFile(p);
|
|
@@ -50,7 +57,7 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
50
57
|
const scalars = {};
|
|
51
58
|
for (const ref of [].concat(cfg.extends || [])) {
|
|
52
59
|
const bp = isAbsolute(ref) ? ref : join(dirname(p), ref);
|
|
53
|
-
if (!existsSync(bp))
|
|
60
|
+
if (!existsSync(bp)) die(`extends target not found: ${ref} (from ${relOf(repo, p)})`);
|
|
54
61
|
const base = resolveConfig(bp, repo, relOf(repo, bp), seen);
|
|
55
62
|
for (const m of base.metrics) byName.set(m.name, m);
|
|
56
63
|
for (const md of (base.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
@@ -68,18 +75,15 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
68
75
|
}
|
|
69
76
|
|
|
70
77
|
function loadConfig(repo) {
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
if (!p) { console.error('no promptwheel.config.json, run: promptwheel init (writes one for your stack)'); process.exit(2); }
|
|
78
|
+
const p = join(repo, 'promptwheel.config.json');
|
|
79
|
+
if (!existsSync(p)) die('no promptwheel.config.json, run: promptwheel init (writes one for your stack)');
|
|
74
80
|
const cfg = resolveConfig(p, repo, 'local', new Set());
|
|
75
|
-
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0)
|
|
76
|
-
console.error('config.metrics must be a non-empty array'); process.exit(2);
|
|
77
|
-
}
|
|
81
|
+
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0) die('config.metrics must be a non-empty array');
|
|
78
82
|
return cfg;
|
|
79
83
|
}
|
|
80
84
|
|
|
81
85
|
// does the repo have a committed gate config?
|
|
82
|
-
const hasConfigFile = (repo) =>
|
|
86
|
+
const hasConfigFile = (repo) => existsSync(join(repo, 'promptwheel.config.json'));
|
|
83
87
|
|
|
84
88
|
// the zero-config default: the gate `init` would write, built in memory from stack detection.
|
|
85
89
|
// It gates the detected test command (gaming detection on) with the stack's env/linkDirs so the
|
|
@@ -92,8 +96,13 @@ function autoConfig(repo) {
|
|
|
92
96
|
|
|
93
97
|
function resolveBase(repo, base) {
|
|
94
98
|
if (base) return base;
|
|
99
|
+
const head = git(['rev-parse', 'HEAD'], repo);
|
|
95
100
|
for (const b of ['origin/main', 'origin/master', 'main', 'master']) {
|
|
96
|
-
|
|
101
|
+
let mb = '';
|
|
102
|
+
try { mb = git(['merge-base', 'HEAD', b], repo); } catch { continue; }
|
|
103
|
+
// on the trunk itself the merge-base IS HEAD; comparing a commit to itself would be a
|
|
104
|
+
// trivial PASS that verified nothing, so fall through to the last commit instead.
|
|
105
|
+
if (mb && mb !== head) return mb;
|
|
97
106
|
}
|
|
98
107
|
return git(['rev-parse', 'HEAD~1'], repo);
|
|
99
108
|
}
|
|
@@ -274,8 +283,15 @@ function evaluatorMatcher(cfg) {
|
|
|
274
283
|
// forbids ANY change to its paths (same glob semantics as evaluatorPaths). A hit fails the gate
|
|
275
284
|
// regardless of metrics and rides at the top of the report; mandates inherit through `extends`,
|
|
276
285
|
// so a parent scope's rule holds for every repo beneath it.
|
|
277
|
-
|
|
278
|
-
|
|
286
|
+
// The mandates in force are the UNION of the head config's and the base ref's (`baseMandates`,
|
|
287
|
+
// resolved by gateDefAtRef): deleting a mandate in the same change that violates it does not
|
|
288
|
+
// clear it. A mandate removed at head but violated in the diff is reported with `removed: true`.
|
|
289
|
+
// The honest way to retire one is a separate change that only edits the config.
|
|
290
|
+
function checkMandates(repo, base, head, cfg, baseMandates) {
|
|
291
|
+
const headList = Array.isArray(cfg.mandates) ? cfg.mandates : [];
|
|
292
|
+
const headNames = new Set(headList.map((m) => m && m.name).filter(Boolean));
|
|
293
|
+
const removed = (Array.isArray(baseMandates) ? baseMandates : []).filter((m) => m && m.name && !headNames.has(m.name));
|
|
294
|
+
const mandates = [...headList, ...removed.map((m) => ({ ...m, __removed: true }))];
|
|
279
295
|
if (!mandates.length) return [];
|
|
280
296
|
let changed = [];
|
|
281
297
|
try { changed = git(['diff', '--name-only', base, head], repo).split('\n').filter(Boolean); } catch { return []; }
|
|
@@ -285,7 +301,7 @@ function checkMandates(repo, base, head, cfg) {
|
|
|
285
301
|
if (!pats.length) continue;
|
|
286
302
|
const res = pats.map(globToRe);
|
|
287
303
|
const hit = changed.filter((f) => res.some((re) => re.test(f)));
|
|
288
|
-
if (hit.length) out.push({ name: String(m.name || 'mandate'), reason: m.reason ? String(m.reason).slice(0, 300) : '', files: hit.slice(0, 20) });
|
|
304
|
+
if (hit.length) out.push({ name: String(m.name || 'mandate'), reason: m.reason ? String(m.reason).slice(0, 300) : '', files: hit.slice(0, 20), ...(m.__removed ? { removed: true } : {}) });
|
|
289
305
|
}
|
|
290
306
|
return out;
|
|
291
307
|
}
|
|
@@ -299,18 +315,6 @@ function changedSourcePaths(repo, base, head, isEval = () => false) {
|
|
|
299
315
|
return { source: all.filter((p) => !nonSrc(p)), nonSource: all.filter(nonSrc) };
|
|
300
316
|
}
|
|
301
317
|
|
|
302
|
-
// coarse change-location fingerprint for the outcome record: top source dirs of the diff
|
|
303
|
-
function subsystemsOf(repo, base, head, cfg) {
|
|
304
|
-
try {
|
|
305
|
-
const counts = {};
|
|
306
|
-
for (const p of changedSourcePaths(repo, base, head, evaluatorMatcher(cfg)).source) {
|
|
307
|
-
const d = p.includes('/') ? p.slice(0, p.indexOf('/')) : '(root)';
|
|
308
|
-
counts[d] = (counts[d] || 0) + 1;
|
|
309
|
-
}
|
|
310
|
-
return Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([d]) => d);
|
|
311
|
-
} catch { return []; }
|
|
312
|
-
}
|
|
313
|
-
|
|
314
318
|
// measure `metric` in a worktree at `base` with ONLY the source slice of base→head applied.
|
|
315
319
|
function measureSourceOnly(repo, base, head, metric, cfg, repeat) {
|
|
316
320
|
const { source } = changedSourcePaths(repo, base, head, evaluatorMatcher(cfg));
|
|
@@ -414,8 +418,9 @@ function selfHeal(repo) {
|
|
|
414
418
|
// Python editable installs (src-layout `pip install -e .`) can pin imports to the ORIGINAL
|
|
415
419
|
// checkout: the worktree's tests then import your current tree at BOTH refs and every delta
|
|
416
420
|
// reads 0, the gate is structurally blind on that repo shape. Undetectable from the verdict
|
|
417
|
-
// (it looks like "unchanged"), so
|
|
418
|
-
// controlled repro (CHANGELOG 0.2.2).
|
|
421
|
+
// (it looks like "unchanged"), so `init` detects the editable install itself and warns once.
|
|
422
|
+
// Verified by a controlled repro (CHANGELOG 0.2.2). Deliberately NOT run by gate(): it spawns
|
|
423
|
+
// python3 with a 5 s timeout and prints, and the library path (check) must do neither.
|
|
419
424
|
function warnEditableInstall(repo) {
|
|
420
425
|
if (!existsSync(join(repo, 'pyproject.toml')) && !existsSync(join(repo, 'setup.py'))) return;
|
|
421
426
|
try {
|
|
@@ -428,7 +433,7 @@ function warnEditableInstall(repo) {
|
|
|
428
433
|
'print("HIT" if hit else "")',
|
|
429
434
|
].join('\n');
|
|
430
435
|
const out = execFileSync('python3', ['-c', probe, repo], { encoding: 'utf8', timeout: 5000 }).trim();
|
|
431
|
-
if (out === 'HIT') console.error('⚠ editable install of this repo detected (pip install -e): worktree measurements
|
|
436
|
+
if (out === 'HIT') console.error('⚠ editable install of this repo detected (pip install -e): worktree measurements could import your ORIGINAL checkout instead of the measured ref, so deltas would read 0. The written config puts the worktree first on PYTHONPATH to counter this; if deltas still read 0, use a non-editable install in the measuring venv.');
|
|
432
437
|
} catch { /* no python / cannot tell, stay quiet */ }
|
|
433
438
|
}
|
|
434
439
|
|
|
@@ -440,10 +445,12 @@ function warnEditableInstall(repo) {
|
|
|
440
445
|
// that guard WOULD HAVE FAILED at head (a real regression hidden by moving the goalposts).
|
|
441
446
|
const GATE_SCALARS = ['gamingThreshold', 'evaluatorPaths'];
|
|
442
447
|
|
|
443
|
-
// Resolve the gate definition as of `ref` (metrics + scalars), following `extends` by
|
|
444
|
-
// every file from that ref via `git show
|
|
445
|
-
//
|
|
446
|
-
//
|
|
448
|
+
// Resolve the gate definition as of `ref` (metrics + mandates + scalars), following `extends` by
|
|
449
|
+
// reading every file from that ref via `git show`, mirroring resolveConfig (a parent's mandates
|
|
450
|
+
// hold beneath it; a local mandate of the same name overrides). Returns { metrics:Map, scalars }
|
|
451
|
+
// with scalars.mandates, or null when there is no config at `ref` (a gate ADDED in this range is
|
|
452
|
+
// not a weakening). Never exits: any missing/unreadable file or parse error yields null (that
|
|
453
|
+
// branch simply contributes nothing).
|
|
447
454
|
function gateDefAtRef(repo, ref) {
|
|
448
455
|
// quiet: a missing path makes `git show` exit nonzero and print "fatal:" to stderr; swallow it
|
|
449
456
|
const readAt = (p) => { try { return execFileSync('git', ['show', `${ref}:${p}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return null; } };
|
|
@@ -454,20 +461,22 @@ function gateDefAtRef(repo, ref) {
|
|
|
454
461
|
const raw = readAt(p);
|
|
455
462
|
if (raw == null) return null;
|
|
456
463
|
let cfg; try { cfg = JSON.parse(raw); } catch { return null; }
|
|
457
|
-
const byName = new Map(); const scalars = {};
|
|
464
|
+
const byName = new Map(); const mandatesByName = new Map(); const scalars = {};
|
|
458
465
|
for (const ext of [].concat(cfg.extends || [])) {
|
|
459
466
|
if (isAbsolute(ext)) continue; // outside the repo, cannot git-show; unchanged anyway
|
|
460
467
|
const base = resolve(join(dirname(p), ext).split('\\').join('/'));
|
|
461
468
|
if (!base) continue;
|
|
462
469
|
for (const m of base.metrics.values()) byName.set(m.name, m);
|
|
470
|
+
for (const md of base.scalars.mandates) mandatesByName.set(md.name, md);
|
|
463
471
|
for (const k of GATE_SCALARS) if (base.scalars[k] !== undefined) scalars[k] = base.scalars[k];
|
|
464
472
|
}
|
|
465
473
|
for (const m of (cfg.metrics || [])) if (m && m.name) byName.set(m.name, m);
|
|
474
|
+
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
466
475
|
for (const k of GATE_SCALARS) if (cfg[k] !== undefined) scalars[k] = cfg[k];
|
|
476
|
+
scalars.mandates = [...mandatesByName.values()];
|
|
467
477
|
return { metrics: byName, scalars };
|
|
468
478
|
};
|
|
469
|
-
|
|
470
|
-
return null;
|
|
479
|
+
return resolve('promptwheel.config.json');
|
|
471
480
|
}
|
|
472
481
|
|
|
473
482
|
// an order-independent SHA-256 (truncated) fingerprint of a gate definition, for the audit record
|
|
@@ -538,8 +547,8 @@ function gateWeakenings(baseGate, headMetrics) {
|
|
|
538
547
|
}
|
|
539
548
|
|
|
540
549
|
// A declared, auditable override (CONTRACT Section 3). The head commit can acknowledge an
|
|
541
|
-
// intentional test/config/grader change
|
|
542
|
-
// gate change, with a trailer:
|
|
550
|
+
// intentional test/config/grader change (an edit, a delete, or a new file) or an intentional
|
|
551
|
+
// gate change, with a trailer: Promptwheel-Allow: <reason> (Sawbuck-Allow: is an alias)
|
|
543
552
|
// so a change that would read GAMED reads PASS instead, with the acknowledgment RECORDED in
|
|
544
553
|
// report.allowed. The check does not judge intent; it forces intent to be explicit. An UNdeclared
|
|
545
554
|
// test-only win is still gamed. `--allow "<reason>"` (CLI) / `allow` (check) supplies the same for
|
|
@@ -547,13 +556,67 @@ function gateWeakenings(baseGate, headMetrics) {
|
|
|
547
556
|
function readAllowTrailer(repo, ref) {
|
|
548
557
|
let msg = '';
|
|
549
558
|
try { msg = execFileSync('git', ['log', '-1', '--format=%B', ref], { cwd: repo, encoding: 'utf8' }); } catch { return null; }
|
|
550
|
-
const m = msg.match(/^[ \t]*(?:
|
|
559
|
+
const m = msg.match(/^[ \t]*(?:Promptwheel-Allow|Sawbuck-Allow)[ \t]*:[ \t]*(\S.*?)[ \t]*$/im);
|
|
551
560
|
return m && m[1].trim() ? { reason: m[1].trim().slice(0, 300) } : null;
|
|
552
561
|
}
|
|
553
562
|
|
|
563
|
+
// The three Section 3 checks, run only with detectGaming, in order: (1) re-prove every actual
|
|
564
|
+
// win with the agent's source edits alone; (2) content-hash auto-fail: a declared evaluatorPath
|
|
565
|
+
// gate file edited in this diff is a moved goalpost, so a metric that improved alongside it is
|
|
566
|
+
// gamed by declaration, deterministic even if the source-only re-run read inert; (3) gate
|
|
567
|
+
// integrity: resolve the gate as of BASE, re-measure any guard HEAD dropped under its base
|
|
568
|
+
// definition (it only counts as tampering if it WOULD HAVE FAILED at head), and fingerprint the
|
|
569
|
+
// gate definition at both refs (metrics, evaluatorPaths, gamingThreshold, mandates) so a quietly
|
|
570
|
+
// deleted guard or mandate is tamper-evident: base != head is recorded as gate.changed and
|
|
571
|
+
// printed, whatever the verdict. Mutates `metrics` in place (gamed/sourceOnly/retained/
|
|
572
|
+
// gamingReason) and returns the report.gate record. An allow trailer can still clear a gamed
|
|
573
|
+
// metric afterwards, in gate().
|
|
574
|
+
function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
|
|
575
|
+
for (const m of metrics) {
|
|
576
|
+
if (m.status !== 'improved') continue;
|
|
577
|
+
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
578
|
+
if (cm.gamingCheck === false) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
|
|
579
|
+
const j = judgeGaming(m, measureSourceOnly(repo, base, head, cm, cfg, repeat), cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5);
|
|
580
|
+
m.gamed = j.gamed; m.sourceOnly = j.sourceOnly; m.retained = j.retained; m.gamingReason = j.reason;
|
|
581
|
+
}
|
|
582
|
+
const evalChanged = changedEvaluatorPaths(repo, base, head, cfg);
|
|
583
|
+
if (evalChanged.length) {
|
|
584
|
+
for (const m of metrics) {
|
|
585
|
+
if (m.status !== 'improved' || m.gamed === true) continue;
|
|
586
|
+
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
587
|
+
if (cm && cm.gamingCheck === false) continue; // tripwire/test-side guards legitimately live in gate files
|
|
588
|
+
m.gamed = true;
|
|
589
|
+
m.gamingReason = `a declared evaluatorPath gate file changed in this diff (${evalChanged.map((e) => e.path).join(', ')}); a win alongside a goalpost edit is gamed by declaration`;
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
const baseGate = gateDefAtRef(repo, base);
|
|
593
|
+
const weak = gateWeakenings(baseGate, cfg.metrics);
|
|
594
|
+
const weakened = [];
|
|
595
|
+
if (weak.length) {
|
|
596
|
+
const defs = weak.map((w) => w.def);
|
|
597
|
+
const b = measureAt(repo, base, { ...cfg, metrics: defs }, repeat);
|
|
598
|
+
const a = measureAt(repo, head, { ...cfg, metrics: defs }, repeat);
|
|
599
|
+
for (const w of weak) {
|
|
600
|
+
const ev = evaluate(w.def, b[w.name], a[w.name], repeat);
|
|
601
|
+
if (!ev.ok) weakened.push({ name: w.name, change: w.change, status: ev.status });
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
const fpBase = baseGate ? gateFingerprint([...baseGate.metrics.values()], baseGate.scalars) : null;
|
|
605
|
+
const fpHead = gateFingerprint(cfg.metrics, cfg);
|
|
606
|
+
const info = {
|
|
607
|
+
fingerprint: { base: fpBase, head: fpHead },
|
|
608
|
+
...(fpBase && fpBase !== fpHead ? { changed: true } : {}),
|
|
609
|
+
...(weakened.length ? { weakened } : {}),
|
|
610
|
+
...(evalChanged.length ? { evaluatorChanged: evalChanged } : {}),
|
|
611
|
+
};
|
|
612
|
+
// hand the base ref's mandates to gate() (not part of the record) so checkMandates can hold a
|
|
613
|
+
// change to a mandate it deleted in the same diff.
|
|
614
|
+
Object.defineProperty(info, 'baseMandates', { value: baseGate ? baseGate.scalars.mandates || [] : null, enumerable: false });
|
|
615
|
+
return info;
|
|
616
|
+
}
|
|
617
|
+
|
|
554
618
|
function gate(repo, opts) {
|
|
555
619
|
selfHeal(repo);
|
|
556
|
-
warnEditableInstall(repo);
|
|
557
620
|
const cfg = opts.config ?? loadConfig(repo);
|
|
558
621
|
const repeat = Math.max(1, opts.repeat ?? cfg.repeat ?? 1);
|
|
559
622
|
|
|
@@ -563,7 +626,7 @@ function gate(repo, opts) {
|
|
|
563
626
|
// never touches the real index or working tree. (A loop agent's most common action
|
|
564
627
|
// is to ADD a file; `git stash create` omits untracked, which silently reverted them.)
|
|
565
628
|
try { git(['rev-parse', '--verify', 'HEAD'], repo); }
|
|
566
|
-
catch {
|
|
629
|
+
catch { die('no commits yet, make an initial commit before gating --working changes'); }
|
|
567
630
|
base = 'HEAD';
|
|
568
631
|
head = workingSnapshot(repo);
|
|
569
632
|
} else {
|
|
@@ -577,59 +640,9 @@ function gate(repo, opts) {
|
|
|
577
640
|
const ev = evaluate(m, before[m.name], after[m.name], repeat);
|
|
578
641
|
return { name: m.name, direction: m.direction || 'up', guard: !!m.guard, ...ev };
|
|
579
642
|
});
|
|
580
|
-
//
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
if (m.status !== 'improved') continue;
|
|
584
|
-
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
585
|
-
if (cm.gamingCheck === false) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
|
|
586
|
-
const j = judgeGaming(m, measureSourceOnly(repo, base, head, cm, cfg, repeat), cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5);
|
|
587
|
-
m.gamed = j.gamed; m.sourceOnly = j.sourceOnly; m.retained = j.retained; m.gamingReason = j.reason;
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
// content-hash auto-fail (Section 3): a declared evaluatorPath gate file edited in this diff is a
|
|
591
|
-
// moved goalpost, so a metric that improved alongside it is gamed by declaration - deterministic,
|
|
592
|
-
// even if the source-only re-run above read inert. An allow-trailer can still clear it, below.
|
|
593
|
-
let evalChanged = [];
|
|
594
|
-
if (opts.detectGaming) {
|
|
595
|
-
evalChanged = changedEvaluatorPaths(repo, base, head, cfg);
|
|
596
|
-
if (evalChanged.length) {
|
|
597
|
-
for (const m of metrics) {
|
|
598
|
-
if (m.status !== 'improved' || m.gamed === true) continue;
|
|
599
|
-
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
600
|
-
if (cm && cm.gamingCheck === false) continue; // tripwire/test-side guards legitimately live in gate files
|
|
601
|
-
m.gamed = true;
|
|
602
|
-
m.gamingReason = `a declared evaluatorPath gate file changed in this diff (${evalChanged.map((e) => e.path).join(', ')}); a win alongside a goalpost edit is gamed by declaration`;
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
}
|
|
606
|
-
// gate integrity (Section 3): did HEAD weaken the gate it is judged by? Resolve the gate as
|
|
607
|
-
// of BASE; a guard HEAD dropped is re-measured under its base definition and only counts as
|
|
608
|
-
// tampering if it WOULD HAVE FAILED at head (a real regression hidden by dropping the guard).
|
|
609
|
-
let gateInfo = null;
|
|
610
|
-
if (opts.detectGaming) {
|
|
611
|
-
const baseGate = gateDefAtRef(repo, base);
|
|
612
|
-
const weak = gateWeakenings(baseGate, cfg.metrics);
|
|
613
|
-
const weakened = [];
|
|
614
|
-
if (weak.length) {
|
|
615
|
-
const defs = weak.map((w) => w.def);
|
|
616
|
-
const b = measureAt(repo, base, { ...cfg, metrics: defs }, repeat);
|
|
617
|
-
const a = measureAt(repo, head, { ...cfg, metrics: defs }, repeat);
|
|
618
|
-
for (const w of weak) {
|
|
619
|
-
const ev = evaluate(w.def, b[w.name], a[w.name], repeat);
|
|
620
|
-
if (!ev.ok) weakened.push({ name: w.name, change: w.change, status: ev.status });
|
|
621
|
-
}
|
|
622
|
-
}
|
|
623
|
-
gateInfo = {
|
|
624
|
-
fingerprint: {
|
|
625
|
-
base: baseGate ? gateFingerprint([...baseGate.metrics.values()], baseGate.scalars) : null,
|
|
626
|
-
head: gateFingerprint(cfg.metrics, cfg),
|
|
627
|
-
},
|
|
628
|
-
...(weakened.length ? { weakened } : {}),
|
|
629
|
-
...(evalChanged.length ? { evaluatorChanged: evalChanged } : {}),
|
|
630
|
-
};
|
|
631
|
-
}
|
|
632
|
-
// declared override: a Sawbuck-Allow trailer (or --allow) acknowledges an intentional
|
|
643
|
+
// Section 3 (gaming detection), on by default: mutates `metrics` and yields the gate record
|
|
644
|
+
const gateInfo = opts.detectGaming ? detectGamingPass(repo, base, head, cfg, metrics, repeat) : null;
|
|
645
|
+
// declared override: a Promptwheel-Allow trailer (or --allow) acknowledges an intentional
|
|
633
646
|
// goalpost/gate change, so a would-be GAMED becomes PASS, recorded in report.allowed for review.
|
|
634
647
|
const allow = opts.allow ? { reason: String(opts.allow).slice(0, 300) } : (opts.working ? null : readAllowTrailer(repo, head));
|
|
635
648
|
let allowed = null;
|
|
@@ -643,7 +656,10 @@ function gate(repo, opts) {
|
|
|
643
656
|
}
|
|
644
657
|
}
|
|
645
658
|
// mandates run first: a protected-path hard rule fails the gate no matter what the metrics say.
|
|
646
|
-
|
|
659
|
+
// The base ref's mandates count too (resolved through `extends` by gateDefAtRef), so a change
|
|
660
|
+
// cannot clear a mandate by deleting it in the same diff that violates it.
|
|
661
|
+
const baseMandates = (gateInfo && gateInfo.baseMandates) || (!opts.detectGaming ? ((gateDefAtRef(repo, base) || {}).scalars || {}).mandates : null);
|
|
662
|
+
const mandateViolations = checkMandates(repo, base, head, cfg, baseMandates);
|
|
647
663
|
const mandated = mandateViolations.length > 0;
|
|
648
664
|
const failed = metrics.some((m) => m.guard && !m.ok);
|
|
649
665
|
const gamed = metrics.some((m) => m.gamed === true) || !!(gateInfo && gateInfo.weakened && !gateInfo.allowed);
|
|
@@ -655,11 +671,6 @@ function gate(repo, opts) {
|
|
|
655
671
|
const verdict = (mandated || failed) ? 'fail' : gamed ? 'gamed' : inconclusive ? 'inconclusive' : 'pass';
|
|
656
672
|
const report = {
|
|
657
673
|
base: short(repo, base), head: short(repo, head), repeat, mode: opts.working ? 'working' : 'refs',
|
|
658
|
-
// learning-substrate fields (Phase 5): cohort segments reliability by environment,
|
|
659
|
-
// label attributes a change-type, subsystems fingerprint WHERE the change landed.
|
|
660
|
-
cohort: opts.cohort ?? (process.env.CI ? 'ci' : 'local'),
|
|
661
|
-
...(opts.label ? { label: opts.label } : {}),
|
|
662
|
-
subsystems: subsystemsOf(repo, base, head, cfg),
|
|
663
674
|
verdict, metrics,
|
|
664
675
|
...(mandateViolations.length ? { mandates: mandateViolations } : {}),
|
|
665
676
|
...(gateInfo ? { gate: gateInfo } : {}),
|
|
@@ -685,7 +696,7 @@ function run(argv) {
|
|
|
685
696
|
config = autoConfig(repo);
|
|
686
697
|
console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd} (promptwheel init to customize; --no-auto to require a config)`);
|
|
687
698
|
}
|
|
688
|
-
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming,
|
|
699
|
+
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, allow: args.allow, config });
|
|
689
700
|
if (args.json) console.log(JSON.stringify(report, null, 2));
|
|
690
701
|
else if (args.markdown) console.log(renderMarkdown(report));
|
|
691
702
|
else printHuman(report);
|
|
@@ -704,24 +715,25 @@ function recordOutcome(repo, report) {
|
|
|
704
715
|
// keep-if-improved: run any agent/script, gate the result, keep the change ONLY if a metric improved
|
|
705
716
|
function improve(argv) {
|
|
706
717
|
const args = parseArgs(argv);
|
|
707
|
-
if (!args.attempt)
|
|
718
|
+
if (!args.attempt) die('improve requires --attempt "<command that changes the repo>"');
|
|
708
719
|
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
709
720
|
const dirty = git(['status', '--porcelain'], repo).split('\n').filter((l) => l.trim() && !l.includes('.promptwheel'));
|
|
710
|
-
if (dirty.length)
|
|
721
|
+
if (dirty.length) die('working tree not clean, commit or stash first (improve needs a clean base to revert to)');
|
|
711
722
|
|
|
712
723
|
console.error(`▶ attempt: ${args.attempt}`);
|
|
713
724
|
try { execSync(args.attempt, { cwd: repo, stdio: 'inherit' }); }
|
|
714
725
|
catch (e) { console.error(` (attempt exited ${e.status ?? 1}, gating whatever it changed)`); }
|
|
715
726
|
|
|
716
|
-
const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming
|
|
727
|
+
const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
|
|
717
728
|
const noChange = report.metrics.every((m) => m.delta === 0 || m.delta == null);
|
|
718
729
|
const improvedNames = report.metrics.filter((m) => m.status === 'improved').map((m) => m.name);
|
|
719
730
|
|
|
720
|
-
// result + exit code express loop progress so `while improve; do :; done` converges
|
|
721
|
-
//
|
|
731
|
+
// result + exit code express loop progress so `while improve; do :; done` converges, with the
|
|
732
|
+
// same codes as `run`: 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed
|
|
733
|
+
// (reverted) · 3 = plateau/no-op OR inconclusive (reverted)
|
|
722
734
|
let result, exit, note;
|
|
723
735
|
if (report.verdict === 'fail') { result = 'regression'; exit = 1; revert(repo); note = '✗ guarded regression, reverted'; }
|
|
724
|
-
else if (report.verdict === 'gamed') { result = 'gamed'; exit =
|
|
736
|
+
else if (report.verdict === 'gamed') { result = 'gamed'; exit = 2; revert(repo); note = '🚩 gamed, a metric "improved" by editing tests/config, not source, reverted'; }
|
|
725
737
|
else if (report.verdict === 'inconclusive') { result = 'inconclusive'; exit = 3; revert(repo); note = '≈ inconclusive, a guard measured nothing (inert); reverted'; }
|
|
726
738
|
else if (noChange || improvedNames.length === 0) {
|
|
727
739
|
result = 'plateau'; exit = 3; revert(repo);
|
|
@@ -747,14 +759,14 @@ function revert(repo) {
|
|
|
747
759
|
try { git(['clean', '-fd', '-e', '.promptwheel'], repo); } catch { /* best effort */ }
|
|
748
760
|
}
|
|
749
761
|
|
|
750
|
-
//
|
|
751
|
-
//
|
|
752
|
-
//
|
|
762
|
+
// insights: honest aggregation of the per-repo outcome record (.promptwheel/outcomes.jsonl)
|
|
763
|
+
// into per-metric counts. No model, no decay, no claims: just which metrics actually respond.
|
|
764
|
+
// The reader is tolerant of older records that carry extra fields (cohort, label, subsystems).
|
|
753
765
|
function insights(argv) {
|
|
754
766
|
const args = parseArgs(argv);
|
|
755
767
|
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
756
768
|
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
757
|
-
if (!existsSync(f))
|
|
769
|
+
if (!existsSync(f)) die('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)');
|
|
758
770
|
const runs = readFileSync(f, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
759
771
|
const agg = {};
|
|
760
772
|
for (const r of runs) for (const m of (r.metrics || [])) {
|
|
@@ -776,149 +788,6 @@ function insights(argv) {
|
|
|
776
788
|
console.log(' highest-lever metrics are where an agent loop should spend its attempts.\n');
|
|
777
789
|
}
|
|
778
790
|
|
|
779
|
-
// ---------------------------------------------------------------------------
|
|
780
|
-
// Phase 5, outcome-curated playbook + UCB work-discovery.
|
|
781
|
-
// Unfrozen 2026-07-02 by explicit founder decision overriding D7 (see docs/LEARNING.md).
|
|
782
|
-
// Design constraints that survive the unfreeze:
|
|
783
|
-
// - PURE VIEW: the append-only ledger (.promptwheel/outcomes.jsonl) is the only store;
|
|
784
|
-
// the playbook is re-derived on every read (decay at read time, no curator state to rot).
|
|
785
|
-
// - EVIDENCE-GATED: a key renders a claim only past MIN_MOVED weighted observations;
|
|
786
|
-
// everything below that is counted, not asserted.
|
|
787
|
-
// - CLAIM-GATED: no public compounding claim until bench/compounding-ab.mjs passes on
|
|
788
|
-
// real usage data. The gate on the CLAIM outlived the gate on the CODE.
|
|
789
|
-
// ---------------------------------------------------------------------------
|
|
790
|
-
const HALF_LIFE = 20; // runs, an entry's weight halves every HALF_LIFE runs unless re-earned
|
|
791
|
-
const MIN_MOVED = 3; // weighted moved-observations before a key earns a rendered claim
|
|
792
|
-
const EXPLORE_C = 1.0; // UCB exploration constant
|
|
793
|
-
|
|
794
|
-
function readLedger(repo) {
|
|
795
|
-
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
796
|
-
if (!existsSync(f)) return [];
|
|
797
|
-
return readFileSync(f, 'utf8').split('\n').filter(Boolean)
|
|
798
|
-
.map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
799
|
-
}
|
|
800
|
-
|
|
801
|
-
// fold the ledger into decayed per-key stats. keys: `metric` · `metric @ subsystem` · `metric # label`
|
|
802
|
-
function foldOutcomes(runs) {
|
|
803
|
-
const keys = new Map();
|
|
804
|
-
const N = runs.length;
|
|
805
|
-
runs.forEach((r, idx) => {
|
|
806
|
-
const w = Math.pow(0.5, (N - 1 - idx) / HALF_LIFE); // idx N-1 = newest → weight 1
|
|
807
|
-
for (const m of (r.metrics || [])) {
|
|
808
|
-
const tags = [m.name];
|
|
809
|
-
for (const s of (r.subsystems || [])) tags.push(`${m.name} @ ${s}`);
|
|
810
|
-
if (r.label) tags.push(`${m.name} # ${r.label}`);
|
|
811
|
-
for (const key of tags) {
|
|
812
|
-
const k = keys.get(key) ?? { key, metric: m.name, w: 0, n: 0, imp: 0, reg: 0, moved: 0, effects: [], cohorts: {}, last: null };
|
|
813
|
-
k.n += 1; k.w += w;
|
|
814
|
-
const coh = (k.cohorts[r.cohort || 'unknown'] ??= { imp: 0, reg: 0 });
|
|
815
|
-
if (m.status === 'improved') {
|
|
816
|
-
k.imp += w; k.moved += w; coh.imp += 1;
|
|
817
|
-
if (typeof m.delta === 'number' && m.delta !== 0) k.effects.push(Math.abs(m.delta));
|
|
818
|
-
} else if (m.status === 'regressed') { k.reg += w; k.moved += w; coh.reg += 1; }
|
|
819
|
-
k.last = r.ts || k.last;
|
|
820
|
-
keys.set(key, k);
|
|
821
|
-
}
|
|
822
|
-
}
|
|
823
|
-
});
|
|
824
|
-
return [...keys.values()];
|
|
825
|
-
}
|
|
826
|
-
|
|
827
|
-
const betaMean = (imp, reg) => (1 + imp) / (2 + imp + reg); // Beta(1,1)-smoothed helpfulness
|
|
828
|
-
|
|
829
|
-
// composite lever = smoothed helpfulness × responsiveness; UCB adds an exploration bonus
|
|
830
|
-
function scoreKey(k, totalW) {
|
|
831
|
-
const p = betaMean(k.imp, k.reg);
|
|
832
|
-
const move = k.w ? k.moved / k.w : 0;
|
|
833
|
-
const lever = p * move;
|
|
834
|
-
const ucb = lever + EXPLORE_C * Math.sqrt(Math.log(Math.max(Math.E, totalW)) / Math.max(1e-6, k.w));
|
|
835
|
-
return { p: +p.toFixed(3), move: +move.toFixed(3), lever: +lever.toFixed(3), ucb: +ucb.toFixed(3), effect: +(median(k.effects) ?? 0).toFixed(6) };
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
// cohorts that disagree in sign are flagged, never averaged into a lie
|
|
839
|
-
function cohortNote(k) {
|
|
840
|
-
const cs = Object.entries(k.cohorts).filter(([, c]) => c.imp + c.reg > 0);
|
|
841
|
-
if (cs.length < 2) return '';
|
|
842
|
-
const signs = new Set(cs.map(([, c]) => Math.sign(c.imp - c.reg)));
|
|
843
|
-
return signs.size > 1 ? ' · ⚠ cohort-dependent (ci vs local disagree)' : '';
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
function loadEntries(repo) {
|
|
847
|
-
const runs = readLedger(repo);
|
|
848
|
-
if (!runs.length) { console.error('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)'); process.exit(2); }
|
|
849
|
-
const keys = foldOutcomes(runs);
|
|
850
|
-
const totalW = keys.filter((k) => k.key === k.metric).reduce((s, k) => s + k.w, 0);
|
|
851
|
-
const entries = keys.map((k) => ({ ...k, ...scoreKey(k, totalW), sufficient: k.moved >= MIN_MOVED }))
|
|
852
|
-
.sort((a, b) => b.lever - a.lever || b.w - a.w);
|
|
853
|
-
return { runs, entries };
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
// the distilled, execution-earned context an agent (or CLAUDE.md) can consume
|
|
857
|
-
function playbook(argv) {
|
|
858
|
-
const args = parseArgs(argv);
|
|
859
|
-
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
860
|
-
const { runs, entries } = loadEntries(repo);
|
|
861
|
-
if (args.json) { console.log(JSON.stringify({ runs: runs.length, halfLife: HALF_LIFE, minMoved: MIN_MOVED, entries }, null, 2)); return; }
|
|
862
|
-
const suff = entries.filter((e) => e.sufficient);
|
|
863
|
-
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
864
|
-
const out = [];
|
|
865
|
-
out.push(`## Earned playbook, ${runs.length} gated runs, decay half-life ${HALF_LIFE} runs`);
|
|
866
|
-
out.push('*(every line below was measured by the gate, not asserted; entries decay unless re-earned)*', '');
|
|
867
|
-
if (!suff.length) {
|
|
868
|
-
out.push(`_Nothing has earned a claim yet, a key needs ≥${MIN_MOVED} weighted moved-observations. Keep gating; ${entries.length} keys are accumulating._`);
|
|
869
|
-
} else {
|
|
870
|
-
for (const e of suff.slice(0, 20)) {
|
|
871
|
-
out.push(`- **${e.key}**, helpful ${pct(e.p)} when it moves (${e.imp.toFixed(1)}✓/${e.reg.toFixed(1)}✗ weighted), responds in ${pct(e.move)} of runs, median effect ${e.effect}${cohortNote(e)}`);
|
|
872
|
-
}
|
|
873
|
-
const hidden = entries.length - suff.length;
|
|
874
|
-
if (hidden > 0) out.push('', `_${hidden} more keys below the evidence threshold, counted, not asserted._`);
|
|
875
|
-
}
|
|
876
|
-
console.log(out.join('\n'));
|
|
877
|
-
}
|
|
878
|
-
|
|
879
|
-
// UCB work-discovery: where should the loop spend its NEXT attempt? Proposes measured
|
|
880
|
-
// targets (metric/subsystem arms), never code advice; the measurement stays the message.
|
|
881
|
-
function suggest(argv) {
|
|
882
|
-
const args = parseArgs(argv);
|
|
883
|
-
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
884
|
-
const { runs, entries } = loadEntries(repo);
|
|
885
|
-
const arms = entries.filter((e) => e.key === e.metric || e.key.includes(' @ ')).sort((a, b) => b.ucb - a.ucb);
|
|
886
|
-
if (args.json) { console.log(JSON.stringify({ runs: runs.length, thin: runs.length < 10, arms: arms.slice(0, 10) }, null, 2)); return; }
|
|
887
|
-
console.log(`\nPromptWheel suggest, UCB over ${runs.length} gated runs${runs.length < 10 ? ' (thin record: exploration dominates, treat as a coin with opinions)' : ''}\n`);
|
|
888
|
-
for (const a of arms.slice(0, 5)) {
|
|
889
|
-
console.log(` ${a.ucb.toFixed(3)} ${a.key.padEnd(28)} lever ${a.lever.toFixed(3)} (helpful ${a.p}, responds ${a.move}) · evidence ${a.moved.toFixed(1)} moved / ${a.n} runs${a.sufficient ? '' : ' · below claim threshold'}`);
|
|
890
|
-
}
|
|
891
|
-
console.log('\n score = lever + exploration bonus, high scores are either proven levers or under-explored arms.\n');
|
|
892
|
-
}
|
|
893
|
-
|
|
894
|
-
// backfill, seed the ledger from git history: replay past commits through the CURRENT
|
|
895
|
-
// metrics (LEARNING.md harvest path 1). Deterministic, no LLM. Rows are cohort-tagged
|
|
896
|
-
// 'backfill', historical human commits are NOT live agent-loop evidence; the cohort
|
|
897
|
-
// machinery segments them and flags disagreement rather than averaging it away. The
|
|
898
|
-
// conventional-commit type (fix/feat/refactor/…) becomes the change-type label for free.
|
|
899
|
-
function backfill(argv) {
|
|
900
|
-
const args = parseArgs(argv);
|
|
901
|
-
const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
|
|
902
|
-
const listArgs = ['rev-list', '--no-merges'];
|
|
903
|
-
if (args.since) listArgs.push(`${args.since}..HEAD`); else listArgs.push('-n', String(args.n ?? 30), 'HEAD');
|
|
904
|
-
const commits = git(listArgs, repo).split('\n').filter(Boolean).reverse(); // oldest first → ledger stays chronological, decay stays honest
|
|
905
|
-
if (!commits.length) { console.error('no commits to backfill'); process.exit(2); }
|
|
906
|
-
const seen = new Set(readLedger(repo).filter((r) => r.cohort === 'backfill').map((r) => r.head));
|
|
907
|
-
let done = 0, skipped = 0;
|
|
908
|
-
console.log(`\nbackfilling ${commits.length} commits through the current metrics (gaming detection ${args.dgExplicit && args.detectGaming ? 'ON' : 'off, pass --detect-gaming to audit history too'})\n`);
|
|
909
|
-
for (const c of commits) {
|
|
910
|
-
try { git(['rev-parse', `${c}~1`], repo); } catch { skipped++; continue; } // root commit
|
|
911
|
-
if (seen.has(short(repo, c))) { skipped++; continue; } // already recorded
|
|
912
|
-
const subj = git(['log', '-1', '--format=%s', c], repo);
|
|
913
|
-
const type = /^(feat|fix|docs|test|chore|refactor|perf|ci|build|style)\b/i.exec(subj)?.[1]?.toLowerCase();
|
|
914
|
-
const report = gate(repo, { base: `${c}~1`, head: c, repeat: args.repeat, detectGaming: args.dgExplicit ? args.detectGaming : false, label: type, cohort: 'backfill' });
|
|
915
|
-
console.log(` ${report.head} ${report.verdict.toUpperCase().padEnd(6)} ${type ? `#${type}`.padEnd(10) : ''.padEnd(10)} ${subj.slice(0, 56)}`);
|
|
916
|
-
done++;
|
|
917
|
-
}
|
|
918
|
-
console.log(`\n backfilled ${done}${skipped ? ` (skipped ${skipped}: root or already recorded)` : ''}, old commits that no longer build record as unmeasurable, never faked.`);
|
|
919
|
-
console.log(' next: promptwheel playbook · promptwheel suggest\n');
|
|
920
|
-
}
|
|
921
|
-
|
|
922
791
|
// ---------------------------------------------------------------------------
|
|
923
792
|
// init, write a starter config so a newcomer isn't staring at a blank page
|
|
924
793
|
// ---------------------------------------------------------------------------
|
|
@@ -1015,6 +884,13 @@ function hasEslint(repo) {
|
|
|
1015
884
|
} catch { return false; }
|
|
1016
885
|
}
|
|
1017
886
|
|
|
887
|
+
// does git ignore this repo-relative path? `check-ignore -q` exits 0 when it does; 1 when it does
|
|
888
|
+
// not; anything else (not a git repo) counts as not ignored, so init never fails on the check.
|
|
889
|
+
function gitIgnores(repo, relPath) {
|
|
890
|
+
try { execFileSync('git', ['check-ignore', '-q', relPath], { cwd: repo, stdio: 'ignore' }); return true; }
|
|
891
|
+
catch { return false; }
|
|
892
|
+
}
|
|
893
|
+
|
|
1018
894
|
function init(argv) {
|
|
1019
895
|
if (argv.includes('--list')) {
|
|
1020
896
|
console.log('PromptWheel presets (promptwheel init --preset <name>):\n');
|
|
@@ -1023,15 +899,13 @@ function init(argv) {
|
|
|
1023
899
|
}
|
|
1024
900
|
const repo = (() => { try { return git(['rev-parse', '--show-toplevel'], process.cwd()); } catch { return process.cwd(); } })();
|
|
1025
901
|
const out = join(repo, 'promptwheel.config.json');
|
|
1026
|
-
if (existsSync(out) && !argv.includes('--force'))
|
|
1027
|
-
console.error('promptwheel.config.json already exists, pass --force to overwrite'); process.exit(2);
|
|
1028
|
-
}
|
|
902
|
+
if (existsSync(out) && !argv.includes('--force')) die('promptwheel.config.json already exists, pass --force to overwrite');
|
|
1029
903
|
const pi = argv.indexOf('--preset');
|
|
1030
904
|
const presetName = pi >= 0 ? argv[pi + 1] : null;
|
|
1031
905
|
let metrics, note, evaluatorPaths;
|
|
1032
906
|
if (presetName) {
|
|
1033
907
|
const p = PRESETS[presetName];
|
|
1034
|
-
if (!p)
|
|
908
|
+
if (!p) die(`unknown preset "${presetName}", try: promptwheel init --list`);
|
|
1035
909
|
evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (e.g. security-antihack)
|
|
1036
910
|
metrics = p.metrics || [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }];
|
|
1037
911
|
metrics = metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
|
|
@@ -1044,8 +918,14 @@ function init(argv) {
|
|
|
1044
918
|
const src = fileURLToPath(new URL(`../${pk[2]}/`, import.meta.url));
|
|
1045
919
|
const dst = join(repo, '.promptwheel', pk[2]);
|
|
1046
920
|
mkdirSync(dst, { recursive: true });
|
|
1047
|
-
|
|
921
|
+
const files = readdirSync(src);
|
|
922
|
+
for (const f of files) copyFileSync(join(src, f), join(dst, f));
|
|
1048
923
|
console.log(` ✓ installed ${pk[2]} → .promptwheel/${pk[2]} (commit it with the config)`);
|
|
924
|
+
// the gate measures in clean worktrees, so a pack that git ignores (common: the outcome
|
|
925
|
+
// record lives in .promptwheel/ too) is absent at both refs and the metric reads
|
|
926
|
+
// unmeasurable, failing the guard with no hint why. Say so now, while it is one edit.
|
|
927
|
+
if (files.length && gitIgnores(repo, `.promptwheel/${pk[2]}/${files[0]}`))
|
|
928
|
+
console.log(` ⚠ .promptwheel/${pk[2]} is gitignored: the gate measures in clean worktrees, so an ignored pack is absent at both refs and ${m.name} reads unmeasurable (the guard fails). Replace ".promptwheel/" in .gitignore with ".promptwheel/*" and "!.promptwheel/packs/", then commit the pack`);
|
|
1049
929
|
return { ...m, cmd: m.cmd.replaceAll(pk[2] + '/', `.promptwheel/${pk[2]}/`) };
|
|
1050
930
|
});
|
|
1051
931
|
note = presetName;
|
|
@@ -1064,6 +944,7 @@ function init(argv) {
|
|
|
1064
944
|
if (envCfg.linkDirs) note += ` · linking ${envCfg.linkDirs.join(', ')}`;
|
|
1065
945
|
writeFileSync(out, JSON.stringify({ repeat: 1, ...envCfg, ...(evaluatorPaths ? { evaluatorPaths } : {}), metrics }, null, 2) + '\n');
|
|
1066
946
|
console.log(`✓ wrote promptwheel.config.json, ${note}`);
|
|
947
|
+
warnEditableInstall(repo); // one-time advice at setup, never on the per-turn path
|
|
1067
948
|
if (!argv.includes('--no-agent-block')) {
|
|
1068
949
|
const wrote = writeAgentBlock(repo);
|
|
1069
950
|
if (wrote.length) console.log(`✓ wrote the agent guide (when to run the gate) to ${wrote.join(', ')}`);
|
|
@@ -1125,7 +1006,7 @@ const short = (repo, ref) => { try { return git(['rev-parse', '--short', ref], r
|
|
|
1125
1006
|
function printHuman(r) {
|
|
1126
1007
|
const arrowFor = (m) => (m.delta == null ? '?' : m.delta > 0 ? '▲' : m.delta < 0 ? '▼' : '=');
|
|
1127
1008
|
console.log(`\nPromptWheel ${r.base} → ${r.head}${r.repeat > 1 ? ` (×${r.repeat})` : ''}\n`);
|
|
1128
|
-
if (r.mandates) for (const md of r.mandates) console.log(` ⛔ MANDATE "${md.name}" violated${md.reason ? ': ' + md.reason : ''} (touched: ${md.files.join(', ')})`);
|
|
1009
|
+
if (r.mandates) for (const md of r.mandates) console.log(` ⛔ MANDATE "${md.name}" violated${md.reason ? ': ' + md.reason : ''} (touched: ${md.files.join(', ')})${md.removed ? ' [deleted in this same change; a mandate stays in force for the change that removes it]' : ''}`);
|
|
1129
1010
|
for (const m of r.metrics) {
|
|
1130
1011
|
const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
|
|
1131
1012
|
const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
|
|
@@ -1134,11 +1015,12 @@ function printHuman(r) {
|
|
|
1134
1015
|
if (m.gamed === true) console.log(` 🚩 GAMED, ${m.gamingReason}`);
|
|
1135
1016
|
}
|
|
1136
1017
|
if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(` 🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : 'disabled'} in this change and it would have ${w.status} at head (goalpost moved, not earned)`);
|
|
1018
|
+
if (r.gate && r.gate.changed) console.log(` ⚠ gate definition changed in this change (fingerprint ${r.gate.fingerprint.base} -> ${r.gate.fingerprint.head}): a metric, guard, evaluatorPath or mandate was added, removed or edited; review promptwheel.config.json as part of the diff`);
|
|
1137
1019
|
const failReason = r.mandates ? ', a mandate was violated (a protected path changed)' : ', a guarded metric regressed (beyond noise)';
|
|
1138
1020
|
console.log(`\n VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? failReason : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
|
|
1139
1021
|
if (r.verdict === 'fail' && r.mandates) console.log(' a mandate is a hard rule; if the change is truly intended, edit the mandate in promptwheel.config.json (a tracked, tamper-evident change), do not work around it');
|
|
1140
1022
|
else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name), `promptwheel guards` shows the effective set');
|
|
1141
|
-
if (r.allowed) console.log(` ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (
|
|
1023
|
+
if (r.allowed) console.log(` ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Promptwheel-Allow: ${r.allowed.reason})`);
|
|
1142
1024
|
console.log('');
|
|
1143
1025
|
}
|
|
1144
1026
|
|
|
@@ -1156,13 +1038,14 @@ function renderMarkdown(r) {
|
|
|
1156
1038
|
'',
|
|
1157
1039
|
`\`${r.base} → ${r.head}\` · ${r.mode} mode${r.repeat > 1 ? ` · ×${r.repeat}` : ''}`,
|
|
1158
1040
|
'',
|
|
1159
|
-
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **Mandate violated:** \`${md.name}\`${md.reason ? ': ' + md.reason : ''} (touched \`${md.files.join('`, `')}\`)`).join('\n'), ''] : []),
|
|
1041
|
+
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **Mandate violated:** \`${md.name}\`${md.reason ? ': ' + md.reason : ''} (touched \`${md.files.join('`, `')}\`)${md.removed ? ' (deleted in this same change; it stays in force for the change that removes it)' : ''}`).join('\n'), ''] : []),
|
|
1160
1042
|
'| metric | before | after | Δ | status | confidence |',
|
|
1161
1043
|
'|---|--:|--:|--:|---|---|',
|
|
1162
1044
|
rows,
|
|
1163
1045
|
'',
|
|
1164
1046
|
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A mandate was violated: a protected path changed. This is a hard rule, enforced before any metric.' : '> ❌ A 🛡️ guarded metric regressed beyond the noise band.') : r.verdict === 'inconclusive' ? '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.' : '> ✅ No guarded metric regressed.',
|
|
1165
1047
|
...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : 'disabled'} (would have ${w.status} at head)`).join('; ')}. The pass was bought by dropping a guard, not by earning it.`] : []),
|
|
1048
|
+
...(r.gate && r.gate.changed ? ['', `> ⚠ **Gate definition changed** in this diff (fingerprint \`${r.gate.fingerprint.base}\` -> \`${r.gate.fingerprint.head}\`): a metric, guard, evaluatorPath or mandate was added, removed or edited. Review the config change with the code.`] : []),
|
|
1166
1049
|
'',
|
|
1167
1050
|
'<sub>🛡️ = guard · _prove every change moved a metric_ · [PromptWheel](https://github.com/promptwheel-ai/promptwheel)</sub>',
|
|
1168
1051
|
].join('\n');
|
|
@@ -1178,21 +1061,18 @@ function parseArgs(argv) {
|
|
|
1178
1061
|
else if (argv[i] === '--no-record') a.noRecord = true;
|
|
1179
1062
|
else if (argv[i] === '--no-auto') a.noAuto = true;
|
|
1180
1063
|
else if (argv[i] === '--allow') a.allow = argv[++i];
|
|
1181
|
-
else if (argv[i] === '--detect-gaming'
|
|
1182
|
-
else if (argv[i] === '--no-detect-gaming'
|
|
1183
|
-
else if (argv[i] === '--since') a.since = argv[++i];
|
|
1184
|
-
else if (argv[i] === '-n' || argv[i] === '--limit') a.n = parseInt(argv[++i], 10);
|
|
1064
|
+
else if (argv[i] === '--detect-gaming') a.detectGaming = true;
|
|
1065
|
+
else if (argv[i] === '--no-detect-gaming') a.detectGaming = false;
|
|
1185
1066
|
else if (argv[i] === '--attempt') a.attempt = argv[++i];
|
|
1186
|
-
else if (argv[i] === '--label') a.label = argv[++i];
|
|
1187
1067
|
else if (argv[i] === '--json') a.json = true;
|
|
1188
|
-
else if (argv[i] === '--markdown'
|
|
1068
|
+
else if (argv[i] === '--markdown') a.markdown = true;
|
|
1189
1069
|
}
|
|
1190
1070
|
return a;
|
|
1191
1071
|
}
|
|
1192
1072
|
|
|
1193
1073
|
// pure, side-effect-free helpers are exported for unit testing; the CLI below only
|
|
1194
1074
|
// runs when this file is executed directly (not when imported by the test suite).
|
|
1195
|
-
export { extract, evaluate, median, spread, renderMarkdown, isNonSource, evaluatorMatcher, gain, judgeGaming,
|
|
1075
|
+
export { extract, evaluate, median, spread, renderMarkdown, isNonSource, evaluatorMatcher, gain, judgeGaming, verdictFingerprint };
|
|
1196
1076
|
|
|
1197
1077
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
1198
1078
|
// Public library API: the Verified-Outcome Contract, in process.
|
|
@@ -1235,8 +1115,7 @@ export function verdictExitCode(verdict) {
|
|
|
1235
1115
|
// auto boolean allow the zero-config auto-detect fallback (default true; false throws if no config)
|
|
1236
1116
|
// repeat number measurement repeats for the noise band (default: config.repeat ?? 1)
|
|
1237
1117
|
// detectGaming boolean run the Section 3 source-only re-proof (default: true)
|
|
1238
|
-
//
|
|
1239
|
-
// label string change-type label
|
|
1118
|
+
// allow string a declared override reason (same as --allow / a Promptwheel-Allow: trailer)
|
|
1240
1119
|
// record boolean append the run to .promptwheel/outcomes.jsonl (default: false)
|
|
1241
1120
|
export function check(options = {}) {
|
|
1242
1121
|
let repo;
|
|
@@ -1261,8 +1140,6 @@ export function check(options = {}) {
|
|
|
1261
1140
|
config,
|
|
1262
1141
|
repeat: options.repeat,
|
|
1263
1142
|
detectGaming: options.detectGaming ?? true,
|
|
1264
|
-
cohort: options.cohort,
|
|
1265
|
-
label: options.label,
|
|
1266
1143
|
allow: options.allow,
|
|
1267
1144
|
noRecord: options.record !== true,
|
|
1268
1145
|
});
|
|
@@ -1317,6 +1194,7 @@ export function conformsToContract(report) {
|
|
|
1317
1194
|
if (!report.gate || typeof report.gate !== 'object' || Array.isArray(report.gate)) E('gate must be an object');
|
|
1318
1195
|
else {
|
|
1319
1196
|
if (report.gate.fingerprint !== undefined && (typeof report.gate.fingerprint !== 'object' || report.gate.fingerprint === null)) E('gate.fingerprint must be an object { base, head }');
|
|
1197
|
+
if (report.gate.changed !== undefined && typeof report.gate.changed !== 'boolean') E('gate.changed must be a boolean');
|
|
1320
1198
|
if (report.gate.weakened !== undefined && !Array.isArray(report.gate.weakened)) E('gate.weakened must be an array');
|
|
1321
1199
|
}
|
|
1322
1200
|
}
|
|
@@ -1397,40 +1275,40 @@ function conformance(argv) {
|
|
|
1397
1275
|
|
|
1398
1276
|
function main() {
|
|
1399
1277
|
const [cmd, ...rest] = process.argv.slice(2);
|
|
1278
|
+
if (cmd === '--version' || cmd === '-v') { console.log(TOOL_VERSION ?? 'unknown'); process.exit(0); }
|
|
1400
1279
|
if (cmd === 'run') run(rest);
|
|
1401
1280
|
else if (cmd === 'improve') improve(rest);
|
|
1402
1281
|
else if (cmd === 'insights') insights(rest);
|
|
1403
|
-
else if (cmd === 'playbook') playbook(rest);
|
|
1404
|
-
else if (cmd === 'suggest') suggest(rest);
|
|
1405
|
-
else if (cmd === 'backfill') backfill(rest);
|
|
1406
1282
|
else if (cmd === 'init') init(rest);
|
|
1407
1283
|
else if (cmd === 'guards') guards(rest);
|
|
1408
1284
|
else if (cmd === 'conformance') conformance(rest);
|
|
1409
1285
|
else {
|
|
1286
|
+
const help = !cmd || cmd === '--help' || cmd === '-h' || cmd === 'help';
|
|
1287
|
+
if (!help) console.error(`unknown command: ${cmd}`);
|
|
1410
1288
|
console.log([
|
|
1411
|
-
'PromptWheel,
|
|
1289
|
+
'PromptWheel: a deterministic, zero-LLM check for your coding loop. It proves every change earned its win by re-running the result from the agent\'s source edits alone, so a green that came from editing the test reads GAMED.',
|
|
1412
1290
|
'',
|
|
1291
|
+
' promptwheel --version | -v print the version',
|
|
1292
|
+
' promptwheel --help | -h this banner',
|
|
1413
1293
|
' promptwheel init [--preset <name> | --list] [--no-agent-block] write a starter config + an AGENTS.md guide so your agent runs the gate (OPTIONAL: run works zero-config, auto-detecting your test command)',
|
|
1414
1294
|
' promptwheel run [--base R] [--head R] [--repeat N] [--json|--markdown]',
|
|
1415
1295
|
' catches reward-hacking BY DEFAULT, re-proves each win from SOURCE edits alone (verdict GAMED,',
|
|
1416
1296
|
' exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
|
|
1417
1297
|
' promptwheel run --working gate uncommitted changes (incl. newly added files)',
|
|
1418
|
-
' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `
|
|
1298
|
+
' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `Promptwheel-Allow: <reason>` commit trailer; `Sawbuck-Allow:` is an alias): recorded, not flagged GAMED',
|
|
1419
1299
|
' promptwheel improve --attempt "<cmd>" run an agent/script; keep only if a metric improved',
|
|
1420
|
-
' exit 0=kept · 1=regression · 3=plateau · add --json',
|
|
1421
|
-
' promptwheel insights
|
|
1422
|
-
' promptwheel
|
|
1423
|
-
' promptwheel suggest [--json] UCB work-discovery: where the next attempt should go (experimental)',
|
|
1424
|
-
' promptwheel backfill [-n N | --since <ref>] seed the ledger from git history (cohort-tagged; commit types become labels)',
|
|
1425
|
-
' promptwheel guards show the effective guardrails (incl. inherited) + flag record',
|
|
1300
|
+
' exit 0=kept · 1=regression · 2=gamed · 3=plateau/inconclusive (all but 0 reverted) · add --json',
|
|
1301
|
+
' promptwheel insights [--json] which metrics actually respond (raw counts from .promptwheel/outcomes.jsonl)',
|
|
1302
|
+
' promptwheel guards [--json] show the effective guardrails (incl. inherited) + flag record',
|
|
1426
1303
|
' promptwheel conformance [file|-] [--cmd C] check a verdict JSON (or a command) against the Contract (exit 0 = conforms)',
|
|
1427
1304
|
'',
|
|
1305
|
+
'Exit codes: 0 pass · 1 fail (a guarded metric regressed, or a mandate was violated) · 2 gamed · 3 inconclusive · 64 usage or config error (never a verdict).',
|
|
1428
1306
|
'Integrate (library / conform / shell / MCP): see INTEGRATE.md. Import: `import { check } from \'promptwheel\'`.',
|
|
1429
1307
|
'Loop it: while promptwheel improve --attempt "$AGENT"; do :; done # stops on plateau/regression',
|
|
1430
1308
|
'Config: promptwheel.config.json → { extends?, metrics:[{ name, cmd, direction, extract?, guard? }] } (or: promptwheel init)',
|
|
1431
1309
|
' extends: a path (or array) to a shared base config, repos INHERIT its guardrails; local metrics override by name.',
|
|
1432
1310
|
].join('\n'));
|
|
1433
|
-
process.exit(
|
|
1311
|
+
process.exit(help ? 0 : EX_USAGE);
|
|
1434
1312
|
}
|
|
1435
1313
|
}
|
|
1436
1314
|
|