promptwheel 0.7.16 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INTEGRATE.md +22 -23
- package/README.md +25 -28
- package/bin/promptwheel.d.ts +15 -13
- package/bin/promptwheel.mjs +241 -311
- package/package.json +1 -1
- package/packs/supply-chain/README.md +16 -12
- package/packs/supply-chain/pack.json +1 -1
- package/packs/security/README.md +0 -62
- package/packs/security/pack.json +0 -12
- package/packs/security/patterns.json +0 -209
- package/packs/security/scan.mjs +0 -57
package/bin/promptwheel.mjs
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import { execSync, execFileSync } from 'node:child_process';
|
|
16
16
|
import { createHash } from 'node:crypto';
|
|
17
|
-
import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync,
|
|
17
|
+
import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, realpathSync, readdirSync, lstatSync, copyFileSync } from 'node:fs';
|
|
18
18
|
import { tmpdir } from 'node:os';
|
|
19
19
|
import { join, dirname, isAbsolute, relative } from 'node:path';
|
|
20
20
|
import { fileURLToPath } from 'node:url';
|
|
@@ -36,6 +36,9 @@ const spread = (xs) => { const a = xs.filter((x) => x != null); return a.length
|
|
|
36
36
|
// repo is still measuring in (a long gate elsewhere must not lose its worktree mid-run)
|
|
37
37
|
function tmpOwnerTag(repo) { let r = repo; try { r = realpathSync(repo); } catch { /* keep as given */ } return createHash('sha1').update(String(r)).digest('hex').slice(0, 8); }
|
|
38
38
|
const tmpPrefix = (repo, kind) => `${kind}-${tmpOwnerTag(repo)}-`;
|
|
39
|
+
// exactly the names mkdtemp makes from tmpPrefix: the kind, the 8-hex owner tag, then mkdtemp's 6 random
|
|
40
|
+
// characters. Self-heal removes nothing else, so a folder that only starts with "promptwheel-" is never ours.
|
|
41
|
+
const ENGINE_TMP = /^(promptwheel-src|promptwheel|pw-idx)-([0-9a-f]{8})-[A-Za-z0-9]{6}$/;
|
|
39
42
|
|
|
40
43
|
// usage / config errors exit 64 (EX_USAGE), so they can never be confused with a verdict
|
|
41
44
|
// (0 pass, 1 fail, 2 gamed, 3 inconclusive): in CI a config typo must not read as GAMED.
|
|
@@ -68,7 +71,7 @@ const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { retu
|
|
|
68
71
|
// resolve a config + its `extends` chain into a flat, provenance-tagged metric list.
|
|
69
72
|
// `extends` is a path (or array of paths) to base configs, relative to the config file:
|
|
70
73
|
// a repo INHERITS their guardrails, and a local metric of the same name overrides the inherited one.
|
|
71
|
-
function resolveConfig(p, repo,
|
|
74
|
+
function resolveConfig(p, repo, seen) {
|
|
72
75
|
const abs = realpathSync(p);
|
|
73
76
|
if (seen.has(abs)) bad(`config inheritance cycle at ${relOf(repo, p)}`);
|
|
74
77
|
seen.add(abs);
|
|
@@ -80,16 +83,16 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
80
83
|
for (const ref of [].concat(cfg.extends || [])) {
|
|
81
84
|
const bp = isAbsolute(ref) ? ref : join(dirname(p), ref);
|
|
82
85
|
if (!existsSync(bp)) bad(`extends target not found: ${ref} (from ${relOf(repo, p)})`);
|
|
83
|
-
const base = resolveConfig(bp, repo,
|
|
86
|
+
const base = resolveConfig(bp, repo, seen);
|
|
84
87
|
for (const m of base.metrics) byName.set(m.name, m);
|
|
85
88
|
for (const md of (base.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
86
|
-
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', '
|
|
89
|
+
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (base[k] !== undefined) scalars[k] = base[k];
|
|
87
90
|
}
|
|
88
91
|
for (const m of (cfg.metrics || [])) {
|
|
89
|
-
byName.set(m.name,
|
|
92
|
+
byName.set(m.name, m);
|
|
90
93
|
}
|
|
91
|
-
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name,
|
|
92
|
-
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', '
|
|
94
|
+
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
95
|
+
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (cfg[k] !== undefined) scalars[k] = cfg[k];
|
|
93
96
|
return { ...scalars, metrics: [...byName.values()], mandates: [...mandatesByName.values()] };
|
|
94
97
|
} finally {
|
|
95
98
|
seen.delete(abs); // only ACTIVE ancestors are a cycle, a diamond (a base reached two ways) is fine
|
|
@@ -99,7 +102,7 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
99
102
|
function loadConfig(repo) {
|
|
100
103
|
const p = join(repo, 'promptwheel.config.json');
|
|
101
104
|
if (!existsSync(p)) bad('no promptwheel.config.json, run: promptwheel init (writes one for your stack)');
|
|
102
|
-
const cfg = resolveConfig(p, repo,
|
|
105
|
+
const cfg = resolveConfig(p, repo, new Set());
|
|
103
106
|
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0) bad('config.metrics must be a non-empty array');
|
|
104
107
|
return cfg;
|
|
105
108
|
}
|
|
@@ -112,8 +115,14 @@ const hasConfigFile = (repo) => existsSync(join(repo, 'promptwheel.config.json')
|
|
|
112
115
|
// measurement actually runs in a throwaway worktree. Lets `run` and `check()` work with no config
|
|
113
116
|
// file; `init` is for customizing. A repo with no test command gets a sentinel that reads inert ->
|
|
114
117
|
// INCONCLUSIVE (never a fake green), with a message pointing at the fix.
|
|
115
|
-
function autoConfig(repo) {
|
|
116
|
-
return { ...detectEnv(repo), metrics: [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }] };
|
|
118
|
+
function autoConfig(repo, at = stackAt(repo)) {
|
|
119
|
+
return { ...detectEnv(repo, at), metrics: [{ name: 'tests_pass', cmd: detectTestCmd(repo, at), extract: 'exit', direction: 'pass', guard: true }] };
|
|
120
|
+
}
|
|
121
|
+
// where stack detection looks: the working tree, or the tracked files as of `ref` (the repo as it was before a change)
|
|
122
|
+
function stackAt(repo, ref) {
|
|
123
|
+
if (!ref) return { has: (f) => existsSync(join(repo, f)), read: (f) => readFileSync(join(repo, f), 'utf8') };
|
|
124
|
+
const read = (f) => execFileSync('git', ['show', `${ref}:${f}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
|
|
125
|
+
return { has: (f) => { try { read(f); return true; } catch { return false; } }, read };
|
|
117
126
|
}
|
|
118
127
|
|
|
119
128
|
function resolveBase(repo, base) {
|
|
@@ -123,7 +132,8 @@ function resolveBase(repo, base) {
|
|
|
123
132
|
catch { bad('promptwheel: no commits yet, make an initial commit before gating'); }
|
|
124
133
|
for (const b of ['origin/main', 'origin/master', 'main', 'master']) {
|
|
125
134
|
let mb = '';
|
|
126
|
-
|
|
135
|
+
// stderr ignored: a ref that is not there (no origin/main in this clone) is expected, not an error to print
|
|
136
|
+
try { mb = execFileSync('git', ['merge-base', 'HEAD', b], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { continue; }
|
|
127
137
|
// on the trunk itself the merge-base IS HEAD; comparing a commit to itself would be a
|
|
128
138
|
// trivial PASS that verified nothing, so fall through to the last commit instead.
|
|
129
139
|
if (mb && mb !== head) return mb;
|
|
@@ -132,15 +142,21 @@ function resolveBase(repo, base) {
|
|
|
132
142
|
catch { bad('promptwheel: no commit to compare against (one commit, or a shallow clone): pass --base, or --working'); }
|
|
133
143
|
}
|
|
134
144
|
|
|
135
|
-
|
|
136
|
-
|
|
145
|
+
// exit statuses of a sample list, kept beside it: 127 (the shell found no such command), 126 (it could not
|
|
146
|
+
// execute it) or no status (killed: a timeout) mean the metric never ran, which needs a different fix from a
|
|
147
|
+
// test that ran and failed
|
|
148
|
+
const exitCodes = new WeakMap();
|
|
149
|
+
const neverRan = (samples) => { const c = exitCodes.get(samples) || []; return c.length > 0 && c.every((x) => x == null || x === 126 || x === 127); };
|
|
150
|
+
function runMetric(cwd, m, env, codes) {
|
|
151
|
+
let stdout = '', code = 0, status = 0;
|
|
137
152
|
// Never leak the harness's own node:test context into the measured gate: a metric like
|
|
138
153
|
// `node --test` would see NODE_TEST_CONTEXT, act as a child reporter, and report a bogus
|
|
139
154
|
// exit code (so the gate can't be trusted when PromptWheel itself runs under a test runner).
|
|
140
155
|
const childEnv = { ...(env || process.env) }; delete childEnv.NODE_TEST_CONTEXT;
|
|
141
156
|
try {
|
|
142
157
|
stdout = execSync(m.cmd, { cwd, env: childEnv, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: (m.timeoutSec ?? 300) * 1000 });
|
|
143
|
-
} catch (e) { code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; }
|
|
158
|
+
} catch (e) { status = e.status ?? null; code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; }
|
|
159
|
+
if (codes) codes.push(status);
|
|
144
160
|
return extract(stdout, code, m.extract, m.allowNonzero);
|
|
145
161
|
}
|
|
146
162
|
|
|
@@ -169,9 +185,9 @@ function extract(stdout, code, mode, allowNonzero) {
|
|
|
169
185
|
// with linkNodeModules:false → link nothing); `env` sets vars ({wt} → the worktree path, so a
|
|
170
186
|
// Python run can put the measured ref FIRST on PYTHONPATH). Ecosystem knowledge lives in config,
|
|
171
187
|
// not here, the engine stays ecosystem-agnostic.
|
|
188
|
+
const linkDirsOf = (cfg) => (Array.isArray(cfg.linkDirs) ? cfg.linkDirs : (cfg.linkNodeModules === false ? [] : ['node_modules']));
|
|
172
189
|
function bridgeEnv(repo, wt, cfg) {
|
|
173
|
-
const
|
|
174
|
-
for (const d of linkDirs) {
|
|
190
|
+
for (const d of linkDirsOf(cfg)) {
|
|
175
191
|
if (existsSync(join(repo, d)) && !existsSync(join(wt, d))) {
|
|
176
192
|
try { symlinkSync(join(repo, d), join(wt, d)); } catch { /* best effort */ }
|
|
177
193
|
}
|
|
@@ -196,8 +212,9 @@ function measureAt(repo, ref, cfg, repeat) {
|
|
|
196
212
|
runSetup(wt, cfg, env);
|
|
197
213
|
const out = {};
|
|
198
214
|
for (const m of cfg.metrics) {
|
|
199
|
-
const samples = [];
|
|
200
|
-
for (let i = 0; i < repeat; i++) samples.push(runMetric(wt, m, env));
|
|
215
|
+
const samples = [], codes = [];
|
|
216
|
+
for (let i = 0; i < repeat; i++) samples.push(runMetric(wt, m, env, codes));
|
|
217
|
+
exitCodes.set(samples, codes);
|
|
201
218
|
out[m.name] = samples;
|
|
202
219
|
}
|
|
203
220
|
return out;
|
|
@@ -266,9 +283,12 @@ const NON_SOURCE = [
|
|
|
266
283
|
/_spec\.rb$/i,
|
|
267
284
|
/[a-z0-9](Test|Tests)\.(java|kt|kts|scala)$/,
|
|
268
285
|
/\.snap$/i, /(^|\/)golden[^/]*$/i, /\.golden$/i,
|
|
269
|
-
// a grader/eval/score FILE by name (eval.
|
|
270
|
-
// evaluate.ts, evaluation.
|
|
271
|
-
|
|
286
|
+
// a grader/eval/score FILE by name in a scripting language (eval.py, grader-x.js, grade.sh, scoring.ts), never a
|
|
287
|
+
// prefix: evaluate.ts, evaluation.py, scoreboard.js stay source. Compiled names stay source too (eval.c, eval.go
|
|
288
|
+
// are interpreters; Score.java is a type), and so do .tsx/.vue files (a Score component, a /score page).
|
|
289
|
+
/(^|\/)(eval|evals|grade|grader|graders|score|scorer|scoring)([._-][^/]*)?\.([cm]?[jt]s|py|sh|bash|zsh|rb|pl|php|lua|r|jl)$/i,
|
|
290
|
+
// anything under an evals/ or grader(s)/ directory. Not eval/ (an interpreter's package) or scoring/ (a product module).
|
|
291
|
+
/(^|\/)(evals|graders?)\//i,
|
|
272
292
|
// config files only, require a .config/.conf/.setup segment so PRODUCTION source that
|
|
273
293
|
// happens to be named after a tool (e.g. src/installers/eslint.ts) is NOT swept out of
|
|
274
294
|
// the source slice (a false sweep here can flag an honest win as GAMED).
|
|
@@ -279,7 +299,7 @@ const NON_SOURCE = [
|
|
|
279
299
|
// promptwheel's OWN evaluator surface: the config that defines the gate
|
|
280
300
|
// and any pack copied into the repo by `init --preset`. Editing these is
|
|
281
301
|
// moving the goalposts, never a production improvement, an agent that
|
|
282
|
-
// "improved"
|
|
302
|
+
// "improved" a scanner count by deleting its rules must show 0%
|
|
283
303
|
// retained in the source-only re-run, not 100%.
|
|
284
304
|
/(^|\/)promptwheel\.config\.json$/i,
|
|
285
305
|
/(^|\/)\.promptwheel\//i,
|
|
@@ -311,9 +331,9 @@ function evaluatorMatcher(cfg) {
|
|
|
311
331
|
// forbids ANY change to its paths (same glob semantics as evaluatorPaths). A hit fails the gate
|
|
312
332
|
// regardless of metrics and rides at the top of the report; mandates inherit through `extends`,
|
|
313
333
|
// so a parent scope's rule holds for every repo beneath it.
|
|
314
|
-
// The mandates in force are the
|
|
315
|
-
//
|
|
316
|
-
// clear it. A mandate removed at head but violated in the diff is reported with `removed: true`; one
|
|
334
|
+
// The mandates in force are the base ref's (`baseMandates`, resolved by gateDefAtRef), in their head
|
|
335
|
+
// version when both have them: deleting a mandate in the same change that violates it does not
|
|
336
|
+
// clear it, and one the change adds does not apply to it (see below). A mandate removed at head but violated in the diff is reported with `removed: true`; one
|
|
317
337
|
// whose paths were edited at head (same name, different paths) is tested in BOTH versions and reported
|
|
318
338
|
// with `edited: true`, so narrowing a rule in the change that breaks it does not clear it either.
|
|
319
339
|
// The honest way to retire one is a separate change that only edits the config.
|
|
@@ -457,7 +477,7 @@ function judgeGaming(m, srcResult, threshold = 0.5) {
|
|
|
457
477
|
// main
|
|
458
478
|
// ---------------------------------------------------------------------------
|
|
459
479
|
// snapshot the working tree (tracked + untracked) into a dangling commit WITHOUT
|
|
460
|
-
// touching the real index or files, so a gate
|
|
480
|
+
// touching the real index or files, so a gate sees a file an agent just added.
|
|
461
481
|
// Returns 'HEAD' when nothing changed.
|
|
462
482
|
function workingSnapshot(repo) {
|
|
463
483
|
const idxDir = mkdtempSync(join(tmpdir(), tmpPrefix(repo, 'pw-idx')));
|
|
@@ -482,12 +502,16 @@ function workingSnapshot(repo) {
|
|
|
482
502
|
}
|
|
483
503
|
}
|
|
484
504
|
|
|
485
|
-
// self-heal: a hard-killed run (SIGKILL / power loss) can leave an orphaned worktree,
|
|
505
|
+
// self-heal: a hard-killed run (SIGKILL / power loss) can leave an orphaned worktree,
|
|
486
506
|
// its `finally` never ran. On the next run, drop stale worktree registry entries and any
|
|
487
|
-
// abandoned /tmp checkout, so nothing accumulates in your repo or temp dir.
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
//
|
|
507
|
+
// abandoned /tmp checkout, so nothing accumulates in your repo or temp dir. It only ever removes a
|
|
508
|
+
// directory whose name is exactly what this engine generates (ENGINE_TMP): a folder named
|
|
509
|
+
// promptwheel-notes, or another repo's worktree named promptwheel-site-wt, is not ours at any age.
|
|
510
|
+
// Ages, by dir mtime: this repo's unregistered checkout after 1h; this repo's still-registered worktree
|
|
511
|
+
// after a full day (a killed run leaves it registered, and a younger one may be a concurrent gate still
|
|
512
|
+
// measuring); a dir tagged for ANOTHER repo after a full day (an orphan of a repo that was deleted or
|
|
513
|
+
// moved: a new real path is a new tag nobody will ever claim). A registered worktree that is not
|
|
514
|
+
// tagged for this repo is never touched: it may be yours.
|
|
491
515
|
function selfHeal(repo) {
|
|
492
516
|
try {
|
|
493
517
|
git(['worktree', 'prune'], repo); // remove registry entries whose dir is already gone (always safe)
|
|
@@ -496,16 +520,18 @@ function selfHeal(repo) {
|
|
|
496
520
|
.split('\n').filter((l) => l.startsWith('worktree ')).map((l) => l.slice(9).trim()),
|
|
497
521
|
);
|
|
498
522
|
const tmp = tmpdir(), now = Date.now(), mine = tmpOwnerTag(repo);
|
|
499
|
-
const TAGGED = /^(promptwheel-src|promptwheel|pw-idx)-([0-9a-f]{8})-/;
|
|
500
523
|
for (const name of readdirSync(tmp)) {
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
524
|
+
const made = ENGINE_TMP.exec(name);
|
|
525
|
+
if (!made) continue; // not a name this engine generates (worktrees + temp-index dirs): never removed
|
|
526
|
+
const tag = made[2];
|
|
527
|
+
// ours after an hour; another repo's only after a full day
|
|
505
528
|
const p = join(tmp, name);
|
|
506
|
-
if (tracked.has(p)) continue;
|
|
507
|
-
|
|
529
|
+
if (tracked.has(p) && tag !== mine) continue; // a registered worktree this repo's gate did not make
|
|
530
|
+
const maxAge = tag === mine && !tracked.has(p) ? 3600_000 : 86_400_000;
|
|
531
|
+
// lstat: the engine makes directories, so a file or a symlink under one of its names is left alone
|
|
532
|
+
try { const st = lstatSync(p); if (st.isDirectory() && st.mtimeMs < now - maxAge) rmSync(p, { recursive: true, force: true }); } catch { /* */ }
|
|
508
533
|
}
|
|
534
|
+
git(['worktree', 'prune'], repo); // drop the registry entries of anything removed above
|
|
509
535
|
} catch { /* cleanup must never block the gate */ }
|
|
510
536
|
}
|
|
511
537
|
|
|
@@ -537,18 +563,25 @@ function warnEditableInstall(repo) {
|
|
|
537
563
|
// guarded metric in the very diff it "passes". We resolve the gate as of BASE too and, if a
|
|
538
564
|
// guard was dropped, re-measure it under its BASE definition; the pass only reads GAMED when
|
|
539
565
|
// that guard WOULD HAVE FAILED at head (a real regression hidden by moving the goalposts).
|
|
540
|
-
const GATE_SCALARS = ['gamingThreshold', 'evaluatorPaths', 'allowTrailer'];
|
|
566
|
+
const GATE_SCALARS = ['gamingThreshold', 'evaluatorPaths', 'allowTrailer', 'setup', 'env', 'linkDirs', 'linkNodeModules'];
|
|
567
|
+
// what runs or shapes every measurement besides the metric's own command. A setup can rewrite a test and an
|
|
568
|
+
// env var can skip one, so these are part of the gate: fingerprinted, and held at their base values for the
|
|
569
|
+
// change that edits them (see detectGamingPass).
|
|
570
|
+
const measureOf = (s) => ({ setup: s.setup, env: s.env, linkDirs: s.linkDirs, linkNodeModules: s.linkNodeModules });
|
|
571
|
+
const measuredWith = (s) => JSON.stringify([s.setup ?? null, Object.entries(s.env || {}).map(([k, v]) => `${k}=${v}`).sort(), [...linkDirsOf(s)].sort()]);
|
|
541
572
|
|
|
542
573
|
// Resolve the gate definition as of `ref` (metrics + mandates + scalars), following `extends` by
|
|
543
574
|
// reading every file from that ref via `git show`, mirroring resolveConfig (a parent's mandates
|
|
544
575
|
// hold beneath it; a local mandate of the same name overrides). Returns { metrics:Map, scalars }
|
|
545
|
-
// with scalars.mandates, or null when there is no config at `ref` (
|
|
546
|
-
//
|
|
547
|
-
// branch simply contributes nothing)
|
|
576
|
+
// with scalars.mandates, or null when there is no config at `ref` (gate() then falls back to the
|
|
577
|
+
// zero-config gate the repo ran under, see zeroConfigGate). Never exits: any missing/unreadable file or
|
|
578
|
+
// parse error yields null (that branch simply contributes nothing), and `partial` says an extends target
|
|
579
|
+
// could not be read at `ref` (it lives outside the repo), so how the base measured is not fully known.
|
|
548
580
|
function gateDefAtRef(repo, ref) {
|
|
549
581
|
// quiet: a missing path makes `git show` exit nonzero and print "fatal:" to stderr; swallow it
|
|
550
582
|
const readAt = (p) => { try { return execFileSync('git', ['show', `${ref}:${p}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return null; } };
|
|
551
583
|
const seen = new Set();
|
|
584
|
+
let partial = false;
|
|
552
585
|
const resolve = (p) => {
|
|
553
586
|
if (seen.has(p)) return null;
|
|
554
587
|
seen.add(p);
|
|
@@ -557,9 +590,11 @@ function gateDefAtRef(repo, ref) {
|
|
|
557
590
|
let cfg; try { cfg = JSON.parse(raw); } catch { return null; }
|
|
558
591
|
const byName = new Map(); const mandatesByName = new Map(); const scalars = {};
|
|
559
592
|
for (const ext of [].concat(cfg.extends || [])) {
|
|
560
|
-
if (isAbsolute(ext)) continue;
|
|
561
|
-
const
|
|
562
|
-
|
|
593
|
+
if (isAbsolute(ext)) { partial = true; continue; } // outside the repo, cannot git-show; unchanged anyway
|
|
594
|
+
const bp = join(dirname(p), ext).split('\\').join('/');
|
|
595
|
+
const again = seen.has(bp); // a diamond: already merged, not a missing piece
|
|
596
|
+
const base = resolve(bp);
|
|
597
|
+
if (!base) { if (!again) partial = true; continue; }
|
|
563
598
|
for (const m of base.metrics.values()) byName.set(m.name, m);
|
|
564
599
|
for (const md of base.scalars.mandates) mandatesByName.set(md.name, md);
|
|
565
600
|
for (const k of GATE_SCALARS) if (base.scalars[k] !== undefined) scalars[k] = base.scalars[k];
|
|
@@ -570,18 +605,35 @@ function gateDefAtRef(repo, ref) {
|
|
|
570
605
|
scalars.mandates = [...mandatesByName.values()];
|
|
571
606
|
return { metrics: byName, scalars };
|
|
572
607
|
};
|
|
573
|
-
|
|
608
|
+
const def = resolve('promptwheel.config.json');
|
|
609
|
+
return def && { ...def, partial };
|
|
574
610
|
}
|
|
575
611
|
|
|
612
|
+
// The gate a repo with no config ran under: the detected test command (source-only re-run on) with the
|
|
613
|
+
// stack's env and linkDirs, detected from the tracked files as of `ref`. A change that adds the first
|
|
614
|
+
// config is judged against this, so `tests_pass.cmd: "true"` in a new config is an edited guard, not a
|
|
615
|
+
// fresh start. `initSetup` is the setup `init` would add: a first config that carries exactly it has not
|
|
616
|
+
// changed how the gate measures. With no test command to detect (the placeholder) the old gate measured
|
|
617
|
+
// nothing, so its setup is no baseline (`partial`).
|
|
618
|
+
function zeroConfigGate(repo, ref) {
|
|
619
|
+
const at = stackAt(repo, ref), auto = autoConfig(repo, at);
|
|
620
|
+
return { metrics: new Map(auto.metrics.map((m) => [m.name, m])), scalars: { mandates: [], ...measureOf(auto) }, zeroConfig: true, initSetup: detectSetup(repo, at), partial: auto.metrics[0].cmd === PLACEHOLDER_TEST };
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// how a metric is measured and judged: change any of these on a guard and it is a different guard
|
|
624
|
+
// what init writes when it finds no test command. It guards nothing, so replacing it is setup, not a weakened gate.
|
|
625
|
+
const PLACEHOLDER_TEST = 'echo "set your test command in promptwheel.config.json" && false';
|
|
626
|
+
const measuredBy = (m) => JSON.stringify([m.cmd ?? null, m.extract ?? 'number', m.direction || 'up', m.exact === true, m.report === true]);
|
|
576
627
|
// an order-independent SHA-256 (truncated) fingerprint of a gate definition, for the audit record
|
|
577
628
|
function gateFingerprint(metricsList, scalars) {
|
|
578
629
|
const norm = {
|
|
579
630
|
metrics: metricsList.map((m) => ({
|
|
580
|
-
name: m.name, guard: m.guard === true,
|
|
631
|
+
name: m.name, guard: m.guard === true, measuredBy: measuredBy(m),
|
|
581
632
|
gamingThreshold: m.gamingThreshold ?? null, gamingCheck: m.gamingCheck !== false,
|
|
582
633
|
})).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
|
|
583
634
|
gamingThreshold: scalars.gamingThreshold ?? null,
|
|
584
635
|
evaluatorPaths: Array.isArray(scalars.evaluatorPaths) ? [...scalars.evaluatorPaths].sort() : [],
|
|
636
|
+
measuredWith: measuredWith(scalars),
|
|
585
637
|
mandates: Array.isArray(scalars.mandates) ? scalars.mandates.map((m) => ({ name: String(m.name || ''), paths: [...(Array.isArray(m.paths) ? m.paths : [])].sort() })).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) : [],
|
|
586
638
|
};
|
|
587
639
|
return createHash('sha256').update(JSON.stringify(norm)).digest('hex').slice(0, 16);
|
|
@@ -626,7 +678,8 @@ function verdictFingerprint(report) {
|
|
|
626
678
|
return createHash('sha256').update(JSON.stringify(norm)).digest('hex').slice(0, 16);
|
|
627
679
|
}
|
|
628
680
|
|
|
629
|
-
// guarded metrics that HEAD dropped relative to BASE (removed entirely, or guard turned off)
|
|
681
|
+
// guarded metrics that HEAD dropped relative to BASE (removed entirely, or guard turned off), or
|
|
682
|
+
// measures another way (a new cmd, extract, direction, exact or report flag): `cmd: "true"` is a dropped guard too
|
|
630
683
|
function gateWeakenings(baseGate, headMetrics) {
|
|
631
684
|
if (!baseGate) return [];
|
|
632
685
|
const head = new Map(headMetrics.map((m) => [m.name, m]));
|
|
@@ -636,6 +689,7 @@ function gateWeakenings(baseGate, headMetrics) {
|
|
|
636
689
|
const hm = head.get(bm.name);
|
|
637
690
|
if (!hm) weak.push({ name: bm.name, change: 'guard-removed', def: bm });
|
|
638
691
|
else if (hm.guard !== true) weak.push({ name: bm.name, change: 'guard-disabled', def: bm });
|
|
692
|
+
else if (!bm.report && bm.cmd !== PLACEHOLDER_TEST && measuredBy(hm) !== measuredBy(bm)) weak.push({ name: bm.name, change: 'guard-edited', def: bm });
|
|
639
693
|
}
|
|
640
694
|
return weak;
|
|
641
695
|
}
|
|
@@ -654,29 +708,51 @@ function readAllowTrailer(repo, ref) {
|
|
|
654
708
|
return m && m[1].trim() ? { reason: m[1].trim().slice(0, 300) } : null;
|
|
655
709
|
}
|
|
656
710
|
|
|
711
|
+
// A change is judged by the gate that was in force before it (`baseGate`: the config at base, or the
|
|
712
|
+
// zero-config gate when this change adds the first config). So a change cannot switch off the source-only
|
|
713
|
+
// re-run for a metric that had it, loosen gamingThreshold, or swap the setup, env or linked dirs it is
|
|
714
|
+
// measured with: the base values hold for the change that edits them, and apply from the next one.
|
|
715
|
+
const gamingCheckOff = (cm, baseGate) => {
|
|
716
|
+
if (!cm || cm.gamingCheck !== false) return false;
|
|
717
|
+
const bm = baseGate && baseGate.metrics.get(cm.name);
|
|
718
|
+
return !bm || bm.gamingCheck === false; // off before this change too, or a metric this change adds
|
|
719
|
+
};
|
|
720
|
+
function gamingThresholdFor(cm, cfg, baseGate) {
|
|
721
|
+
const now = cm.gamingThreshold ?? cfg.gamingThreshold ?? 0.5;
|
|
722
|
+
const bm = baseGate && baseGate.metrics.get(cm.name);
|
|
723
|
+
const before = bm ? (bm.gamingThreshold ?? baseGate.scalars.gamingThreshold ?? (baseGate.partial ? now : 0.5)) : now;
|
|
724
|
+
return Math.max(now, before); // the stricter of before and after
|
|
725
|
+
}
|
|
726
|
+
|
|
657
727
|
// The three Section 3 checks, run only with detectGaming, in order: (1) re-prove every actual
|
|
658
728
|
// win with the agent's source edits alone; (2) content-hash auto-fail: a declared evaluatorPath
|
|
659
729
|
// gate file edited in this diff is a moved goalpost, so a metric that improved alongside it is
|
|
660
730
|
// gamed by declaration, deterministic even if the source-only re-run read inert; (3) gate
|
|
661
|
-
// integrity:
|
|
662
|
-
// definition (it only counts as tampering if it
|
|
663
|
-
// gate definition at both refs (metrics,
|
|
664
|
-
//
|
|
731
|
+
// integrity: take the gate as of BASE, re-measure any guard HEAD dropped, edited or measures with
|
|
732
|
+
// another setup, env or linked dirs, under its base definition (it only counts as tampering if it
|
|
733
|
+
// WOULD HAVE FAILED at head), and fingerprint the gate definition at both refs (metrics,
|
|
734
|
+
// evaluatorPaths, gamingThreshold, mandates, setup, env, linked dirs) so a quietly deleted guard
|
|
735
|
+
// or mandate is tamper-evident: base != head is recorded as gate.changed and
|
|
665
736
|
// printed, whatever the verdict. Mutates `metrics` in place (gamed/sourceOnly/retained/
|
|
666
737
|
// gamingReason) and returns the report.gate record. An allow trailer can still clear a gamed
|
|
667
738
|
// metric afterwards, in gate().
|
|
668
|
-
function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
|
|
739
|
+
function detectGamingPass(repo, base, head, cfg, metrics, repeat, baseGate) {
|
|
740
|
+
// how the base measured. Unknown (`partial`) reads as unchanged; so does a first config that adds only the setup `init` detects
|
|
741
|
+
const bs = baseGate ? (baseGate.partial ? { ...baseGate.scalars, ...measureOf(cfg) } : baseGate.scalars) : null;
|
|
742
|
+
const asInit = !!bs && baseGate.zeroConfig && measuredWith({ ...bs, setup: baseGate.initSetup }) === measuredWith(cfg);
|
|
743
|
+
const measureChanged = !!bs && !asInit && measuredWith(bs) !== measuredWith(cfg);
|
|
744
|
+
const baseRun = measureChanged ? { ...cfg, ...measureOf(bs) } : cfg; // the head gate, run with the base setup, env and linked dirs
|
|
669
745
|
const toProve = [];
|
|
670
746
|
for (const m of metrics) {
|
|
671
747
|
if (m.status !== 'improved') continue;
|
|
672
748
|
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
673
|
-
if (cm
|
|
749
|
+
if (gamingCheckOff(cm, baseGate)) continue; // tripwire / test-side guards aren't re-proven from source (their gain legitimately lives in test files)
|
|
674
750
|
toProve.push(cm);
|
|
675
751
|
}
|
|
676
|
-
const srcResults = measureSourceOnlyAll(repo, base, head, toProve,
|
|
752
|
+
const srcResults = measureSourceOnlyAll(repo, base, head, toProve, baseRun, repeat); // one patched checkout for all of them
|
|
677
753
|
for (const cm of toProve) {
|
|
678
754
|
const m = metrics.find((x) => x.name === cm.name);
|
|
679
|
-
const j = judgeGaming(m, srcResults[cm.name], cm
|
|
755
|
+
const j = judgeGaming(m, srcResults[cm.name], gamingThresholdFor(cm, cfg, baseGate));
|
|
680
756
|
m.gamed = j.gamed; m.sourceOnly = j.sourceOnly; m.retained = j.retained; m.gamingReason = j.reason;
|
|
681
757
|
if (j.unprovable) m.unprovable = true;
|
|
682
758
|
}
|
|
@@ -687,32 +763,55 @@ function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
|
|
|
687
763
|
if (split.source.length && split.nonSource.length) {
|
|
688
764
|
const held = metrics.filter((m) => m.status === 'unchanged' && m.direction === 'pass' && m.after === 1 && !m.tripwire);
|
|
689
765
|
const defs = held.map((m) => cfg.metrics.find((c) => c.name === m.name)).filter(Boolean);
|
|
690
|
-
const res = measureSourceOnlyAll(repo, base, head, defs,
|
|
766
|
+
const res = measureSourceOnlyAll(repo, base, head, defs, baseRun, repeat);
|
|
691
767
|
for (const m of held) { const r = res[m.name]; if (r && r.samples && r.samples.length && median(r.samples) !== 1) m.heldByTestEdit = true; }
|
|
692
768
|
}
|
|
693
|
-
|
|
769
|
+
// a file declared as a goalpost before this change still is one for this change: dropping the declaration in the
|
|
770
|
+
// same diff that edits the file does not make the edit count as source
|
|
771
|
+
const declared = [...new Set([...(cfg.evaluatorPaths || []), ...((baseGate && baseGate.scalars.evaluatorPaths) || [])])];
|
|
772
|
+
const evalChanged = changedEvaluatorPaths(repo, base, head, { ...cfg, evaluatorPaths: declared });
|
|
694
773
|
if (evalChanged.length) {
|
|
695
774
|
for (const m of metrics) {
|
|
696
775
|
if (m.status !== 'improved' || m.gamed === true) continue;
|
|
697
776
|
const cm = cfg.metrics.find((c) => c.name === m.name);
|
|
698
|
-
if (cm
|
|
777
|
+
if (gamingCheckOff(cm, baseGate)) continue; // tripwire/test-side guards legitimately live in gate files
|
|
699
778
|
m.gamed = true;
|
|
700
779
|
m.gamingReason = `a declared evaluatorPath gate file changed in this diff (${evalChanged.map((e) => e.path).join(', ')}); a win alongside a goalpost edit is gamed by declaration`;
|
|
701
780
|
}
|
|
702
781
|
}
|
|
703
|
-
const baseGate = gateDefAtRef(repo, base);
|
|
704
782
|
const weak = gateWeakenings(baseGate, cfg.metrics);
|
|
783
|
+
// the setup, env or linked dirs changed: every guard the base had is measured again the way it was measured before
|
|
784
|
+
if (measureChanged) for (const bm of baseGate.metrics.values()) {
|
|
785
|
+
if (bm.guard === true && !bm.report && bm.cmd !== PLACEHOLDER_TEST && !weak.some((w) => w.name === bm.name)) weak.push({ name: bm.name, change: 'measurement-edited', def: bm });
|
|
786
|
+
}
|
|
705
787
|
const weakened = [];
|
|
706
788
|
if (weak.length) {
|
|
707
789
|
const defs = weak.map((w) => w.def);
|
|
708
|
-
const b = measureAt(repo, base, { ...
|
|
709
|
-
const a = measureAt(repo, head, { ...
|
|
790
|
+
const b = measureAt(repo, base, { ...baseRun, metrics: defs }, repeat);
|
|
791
|
+
const a = measureAt(repo, head, { ...baseRun, metrics: defs }, repeat);
|
|
792
|
+
const evs = new Map();
|
|
710
793
|
for (const w of weak) {
|
|
711
794
|
const ev = evaluate(w.def, b[w.name], a[w.name], repeat);
|
|
712
|
-
|
|
795
|
+
evs.set(w.name, ev);
|
|
796
|
+
// an edited guard that still fails under its old definition was turned green by the edit, not the code
|
|
797
|
+
let hidden = !ev.ok || (w.change === 'guard-edited' && ev.inert);
|
|
798
|
+
// a guard measured another way counts only when the new measurement hides it: it fails the old way and reads fine the new way
|
|
799
|
+
if (w.change === 'measurement-edited') { const hm = metrics.find((x) => x.name === w.name); hidden = (!ev.ok || !!ev.inert) && !!hm && hm.ok && !hm.inert; }
|
|
800
|
+
if (hidden) weakened.push({ name: w.name, change: w.change, status: ev.inert ? 'failed' : ev.status });
|
|
801
|
+
}
|
|
802
|
+
// a win under the old definition still has to be earned in the source: swapping or renaming the guard in the same
|
|
803
|
+
// change does not skip the source-only re-run
|
|
804
|
+
const won = weak.filter((w) => w.def.gamingCheck !== false && !weakened.some((x) => x.name === w.name) && evs.get(w.name).status === 'improved');
|
|
805
|
+
if (won.length) {
|
|
806
|
+
const src = measureSourceOnlyAll(repo, base, head, won.map((w) => w.def), baseRun, repeat);
|
|
807
|
+
for (const w of won) {
|
|
808
|
+
const j = judgeGaming({ name: w.name, direction: w.def.direction || 'up', guard: true, ...evs.get(w.name) }, src[w.name], w.def.gamingThreshold ?? (bs && bs.gamingThreshold) ?? 0.5);
|
|
809
|
+
if (j.gamed) weakened.push({ name: w.name, change: w.change, status: 'gamed' });
|
|
810
|
+
}
|
|
713
811
|
}
|
|
714
812
|
}
|
|
715
|
-
|
|
813
|
+
// the fingerprint is of a config file: `base` stays null when there was none there (the zero-config gate is not a file)
|
|
814
|
+
const fpBase = baseGate && !baseGate.zeroConfig ? gateFingerprint([...baseGate.metrics.values()], bs) : null;
|
|
716
815
|
const fpHead = gateFingerprint(cfg.metrics, cfg);
|
|
717
816
|
const info = {
|
|
718
817
|
fingerprint: { base: fpBase, head: fpHead },
|
|
@@ -731,7 +830,7 @@ function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
|
|
|
731
830
|
// (or a weakened gate) is GAMED; INCONCLUSIVE when nothing could be certified. A guard that never
|
|
732
831
|
// runs (inert: a pass/fail metric stuck at 0 across both refs) verifies nothing, so a bare no-op
|
|
733
832
|
// run is not laundered into a green PASS, BUT an inert guard must not bury a genuine, measured win
|
|
734
|
-
// elsewhere (
|
|
833
|
+
// elsewhere (a loop reading the verdict would throw away a good change). And a
|
|
735
834
|
// win whose source-only re-run could not be applied (`unprovable`) is never a PASS either: a green
|
|
736
835
|
// the engine could not re-prove is exactly the green it exists to doubt.
|
|
737
836
|
export function deriveVerdict(metrics, gateInfo = null, mandated = false) {
|
|
@@ -769,17 +868,26 @@ function gate(repo, opts) {
|
|
|
769
868
|
// a guard this change adds has no value at base, because its scanner did not exist there. That is adoption,
|
|
770
869
|
// not a failure: it starts counting from head. Removing a guard is still caught (gateWeakenings), and a guard the
|
|
771
870
|
// base contract already had still fails when it cannot be measured. A zero-config run has no contract to adopt.
|
|
772
|
-
const
|
|
871
|
+
const fileGate = gateDefAtRef(repo, base);
|
|
872
|
+
const baseDefs = opts.config ? null : fileGate;
|
|
773
873
|
const adopted = (name) => !opts.config && !(baseDefs && baseDefs.metrics.has(name));
|
|
874
|
+
// the gate in force before this change: the config committed at base, or, when this change adds the
|
|
875
|
+
// first config, the zero-config gate the repo ran under until now
|
|
876
|
+
const baseGate = opts.detectGaming ? (fileGate || (opts.config ? null : zeroConfigGate(repo, base))) : null;
|
|
774
877
|
const metrics = cfg.metrics.map((m) => {
|
|
775
878
|
const ev = evaluate(m, before[m.name], after[m.name], repeat);
|
|
776
879
|
if (ev.status === 'unmeasurable' && ev.before == null && ev.after != null && adopted(m.name)) { ev.ok = true; ev.adopted = true; }
|
|
777
880
|
// a `report` guard tells you and does not fail: a contract can start by watching and tighten later
|
|
778
881
|
if (m.report && !ev.ok && ev.status === 'regressed') { ev.ok = true; ev.noted = true; }
|
|
779
|
-
|
|
882
|
+
// an inert guard: did its command run and fail, or never run? `ran` says where it ran (absent: at neither ref)
|
|
883
|
+
if (ev.inert && m.cmd !== PLACEHOLDER_TEST) {
|
|
884
|
+
const nb = neverRan(before[m.name]), na = neverRan(after[m.name]);
|
|
885
|
+
if (!nb || !na) ev.ran = !nb && !na ? 'both' : nb ? 'head' : 'base';
|
|
886
|
+
}
|
|
887
|
+
return { name: m.name, direction: m.direction || 'up', guard: !!m.guard, ...(gamingCheckOff(m, baseGate) ? { tripwire: true } : {}), ...ev };
|
|
780
888
|
});
|
|
781
889
|
// Section 3 (gaming detection), on by default: mutates `metrics` and yields the gate record
|
|
782
|
-
const gateInfo = opts.detectGaming ? detectGamingPass(repo, base, head, cfg, metrics, repeat) : null;
|
|
890
|
+
const gateInfo = opts.detectGaming ? detectGamingPass(repo, base, head, cfg, metrics, repeat, baseGate) : null;
|
|
783
891
|
// declared override: a Promptwheel-Allow trailer (or --allow) acknowledges an intentional
|
|
784
892
|
// goalpost/gate change, so a would-be GAMED becomes PASS, recorded in report.allowed for review.
|
|
785
893
|
// It covers an `unprovable` win too (the source-only re-run could not be applied): the declared
|
|
@@ -820,7 +928,7 @@ function gate(repo, opts) {
|
|
|
820
928
|
for (const m of ga) { m.ok = true; m.allowed = true; allowed.cleared.push(m.name); }
|
|
821
929
|
} else for (const m of ga) m.allowable = true; // failed, and a declared reason would clear it: say so in the report
|
|
822
930
|
}
|
|
823
|
-
// mandates
|
|
931
|
+
// mandates: a protected-path hard rule fails the gate no matter what the metrics say.
|
|
824
932
|
// The base ref's mandates count too (resolved through `extends` by gateDefAtRef), so a change
|
|
825
933
|
// cannot clear a mandate by deleting it in the same diff that violates it.
|
|
826
934
|
const baseMandates = (gateInfo && gateInfo.baseMandates) || (!opts.detectGaming ? ((gateDefAtRef(repo, base) || {}).scalars || {}).mandates : null);
|
|
@@ -854,7 +962,6 @@ function gate(repo, opts) {
|
|
|
854
962
|
configHash: gateFingerprint(cfg.metrics, cfg),
|
|
855
963
|
verdictHash: verdictFingerprint(report),
|
|
856
964
|
};
|
|
857
|
-
if (!opts.noRecord && cfg.record !== false) recordOutcome(repo, report);
|
|
858
965
|
return report;
|
|
859
966
|
}
|
|
860
967
|
|
|
@@ -866,115 +973,18 @@ function run(argv) {
|
|
|
866
973
|
config = autoConfig(repo);
|
|
867
974
|
console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd} (promptwheel init to customize; --no-auto to require a config)`);
|
|
868
975
|
}
|
|
869
|
-
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat,
|
|
976
|
+
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, detectGaming: args.detectGaming, allow: args.allow, config });
|
|
870
977
|
if (args.json) console.log(JSON.stringify(report, null, 2));
|
|
871
978
|
else if (args.markdown) console.log(renderMarkdown(report));
|
|
872
979
|
else printHuman(report);
|
|
873
980
|
process.exit(verdictExitCode(report.verdict));
|
|
874
981
|
}
|
|
875
982
|
|
|
876
|
-
// persisted record: append every gated run to a per-repo outcome record (best-effort, never fails the gate)
|
|
877
|
-
function recordOutcome(repo, report) {
|
|
878
|
-
try {
|
|
879
|
-
const dir = join(repo, '.promptwheel');
|
|
880
|
-
mkdirSync(dir, { recursive: true });
|
|
881
|
-
appendFileSync(join(dir, 'outcomes.jsonl'), JSON.stringify({ ts: new Date().toISOString(), ...report }) + '\n');
|
|
882
|
-
} catch { /* recording must never break the gate */ }
|
|
883
|
-
}
|
|
884
|
-
|
|
885
|
-
// keep-if-improved: run any agent/script, gate the result, keep the change ONLY if a metric improved
|
|
886
|
-
function improve(argv) {
|
|
887
|
-
const args = parseArgs(argv);
|
|
888
|
-
if (!args.attempt) die('improve requires --attempt "<command that changes the repo>"');
|
|
889
|
-
const repo = repoRoot(process.cwd());
|
|
890
|
-
const dirty = git(['status', '--porcelain'], repo).split('\n').filter((l) => l.trim() && !l.includes('.promptwheel'));
|
|
891
|
-
if (dirty.length) die('working tree not clean, commit or stash first (improve needs a clean base to revert to)');
|
|
892
|
-
|
|
893
|
-
console.error(`▶ attempt: ${args.attempt}`);
|
|
894
|
-
try { execSync(args.attempt, { cwd: repo, stdio: 'inherit' }); }
|
|
895
|
-
catch (e) { console.error(` (attempt exited ${e.status ?? 1}, gating whatever it changed)`); }
|
|
896
|
-
|
|
897
|
-
const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
|
|
898
|
-
const noChange = report.metrics.every((m) => m.delta === 0 || m.delta == null);
|
|
899
|
-
const improvedNames = report.metrics.filter((m) => m.status === 'improved').map((m) => m.name);
|
|
900
|
-
|
|
901
|
-
// result + exit code express loop progress so `while improve; do :; done` converges, with the
|
|
902
|
-
// same codes as `run`: 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed
|
|
903
|
-
// (reverted) · 3 = plateau/no-op OR inconclusive (reverted)
|
|
904
|
-
let result, exit, note;
|
|
905
|
-
if (report.verdict === 'fail') { result = 'regression'; exit = 1; revert(repo); note = '✗ guarded regression, reverted'; }
|
|
906
|
-
else if (report.verdict === 'gamed') { result = 'gamed'; exit = 2; revert(repo); note = '🚩 gamed, a metric "improved" by editing tests/config, not source, reverted'; }
|
|
907
|
-
else if (report.verdict === 'inconclusive') { result = 'inconclusive'; exit = 3; revert(repo); note = '≈ inconclusive, a guard measured nothing (inert); reverted'; }
|
|
908
|
-
else if (noChange || improvedNames.length === 0) {
|
|
909
|
-
result = 'plateau'; exit = 3; revert(repo);
|
|
910
|
-
note = noChange ? '= no metric moved, reverted' : '= nothing improved beyond noise, reverted';
|
|
911
|
-
} else {
|
|
912
|
-
git(['add', '-A'], repo);
|
|
913
|
-
execFileSync('git', ['commit', '-qm', `promptwheel: ${args.attempt} [improved ${improvedNames.join(', ')}]`], { cwd: repo });
|
|
914
|
-
result = 'kept'; exit = 0;
|
|
915
|
-
note = `✓ kept, committed ${git(['rev-parse', '--short', 'HEAD'], repo)} (improved ${improvedNames.join(', ')})`;
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
// stdout carries the value (JSON report or the human table); the decision line goes to
|
|
919
|
-
// stderr so a loop driver can consume clean stdout.
|
|
920
|
-
if (args.json) console.log(JSON.stringify({ result, ...report }, null, 2));
|
|
921
|
-
else printHuman(report);
|
|
922
|
-
console.error(` ${note}`);
|
|
923
|
-
process.exit(exit);
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
// discard the attempt's changes; keep the .promptwheel outcome record
|
|
927
|
-
function revert(repo) {
|
|
928
|
-
git(['reset', '--hard', 'HEAD'], repo);
|
|
929
|
-
try { git(['clean', '-fd', '-e', '.promptwheel'], repo); } catch { /* best effort */ }
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
// insights: honest aggregation of the per-repo outcome record (.promptwheel/outcomes.jsonl)
|
|
933
|
-
// into per-metric counts. No model, no decay, no claims: just which metrics actually respond.
|
|
934
|
-
// The reader is tolerant of older records that carry extra fields (cohort, label, subsystems).
|
|
935
|
-
function insights(argv) {
|
|
936
|
-
const args = parseArgs(argv);
|
|
937
|
-
const repo = repoRoot(process.cwd());
|
|
938
|
-
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
939
|
-
if (!existsSync(f)) die('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)');
|
|
940
|
-
const runs = readFileSync(f, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
941
|
-
const agg = {};
|
|
942
|
-
for (const r of runs) for (const m of (r.metrics || [])) {
|
|
943
|
-
const a = agg[m.name] ??= { runs: 0, improved: 0, regressed: 0, inconclusive: 0, unchanged: 0, net: 0, last: null };
|
|
944
|
-
a.runs++; if (a[m.status] != null) a[m.status]++;
|
|
945
|
-
if (typeof m.delta === 'number') a.net = +(a.net + m.delta).toFixed(6);
|
|
946
|
-
a.last = m.after;
|
|
947
|
-
}
|
|
948
|
-
// "lever score" = how reliably acting on this metric yields a real improvement
|
|
949
|
-
const rows = Object.entries(agg).map(([name, a]) => ({ name, ...a, lever: a.runs ? a.improved / a.runs : 0 }))
|
|
950
|
-
.sort((x, y) => y.lever - x.lever);
|
|
951
|
-
if (args.json) { console.log(JSON.stringify({ runs: runs.length, metrics: rows }, null, 2)); return; }
|
|
952
|
-
console.log(`\nPromptWheel insights, ${runs.length} gated runs recorded\n`);
|
|
953
|
-
console.log(` ${'metric'.padEnd(18)} ${'runs'.padStart(5)} ${'impr'.padStart(5)} ${'regr'.padStart(5)} ${'inconc'.padStart(6)} ${'net Δ'.padStart(9)} lever`);
|
|
954
|
-
for (const r of rows) {
|
|
955
|
-
console.log(` ${r.name.padEnd(18)} ${String(r.runs).padStart(5)} ${String(r.improved).padStart(5)} ${String(r.regressed).padStart(5)} ${String(r.inconclusive).padStart(6)} ${String(r.net).padStart(9)} ${(r.lever * 100).toFixed(0)}%`);
|
|
956
|
-
}
|
|
957
|
-
console.log('\n lever = improved/runs, how reliably this metric actually responds. The');
|
|
958
|
-
console.log(' highest-lever metrics are where an agent loop should spend its attempts.\n');
|
|
959
|
-
}
|
|
960
|
-
|
|
961
983
|
// ---------------------------------------------------------------------------
|
|
962
984
|
// init, write a starter config so a newcomer isn't staring at a blank page
|
|
963
985
|
// ---------------------------------------------------------------------------
|
|
964
986
|
const LINT_CMD = 'npx eslint . -f unix 2>/dev/null | grep -c " error " || true';
|
|
965
987
|
const PRESETS = {
|
|
966
|
-
'tests-pass': { desc: 'gate: your test suite still passes (command auto-detected)', metrics: null },
|
|
967
|
-
'lint': { desc: 'track: lint error count does not climb',
|
|
968
|
-
metrics: [{ name: 'lint_errors', cmd: LINT_CMD, extract: 'number', direction: 'down', guard: false }] },
|
|
969
|
-
'bundle-size': { desc: 'track: build output size in kB',
|
|
970
|
-
metrics: [{ name: 'bundle_kb', cmd: 'du -sk dist 2>/dev/null | cut -f1 || echo 0', extract: 'number', direction: 'down', guard: false }] },
|
|
971
|
-
'security': { desc: 'gate: security-invariant violations must not climb (CORS, secrets, cookies, payment; see packs/security)',
|
|
972
|
-
metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }] },
|
|
973
|
-
'llm-eval': { desc: 'gate: AI-feature eval pass-rate + est $/run (see examples/reliability-sprint)',
|
|
974
|
-
metrics: [
|
|
975
|
-
{ name: 'eval_pass_rate', cmd: 'node eval.mjs', extract: 'number', direction: 'up', guard: true },
|
|
976
|
-
{ name: 'cost_per_run_usd', cmd: 'node estimate-cost.mjs', extract: 'number', direction: 'down', guard: true },
|
|
977
|
-
] },
|
|
978
988
|
'antihack': { desc: 'catch reward-hacking: a target + tripwires; pairs with `run --detect-gaming` (source-only re-run)',
|
|
979
989
|
metrics: [
|
|
980
990
|
{ name: 'tests_pass', cmd: '__TESTCMD__', extract: 'exit', direction: 'pass', guard: true },
|
|
@@ -1004,7 +1014,7 @@ const PRESETS = {
|
|
|
1004
1014
|
{ name: 'codeowners-frozen', paths: ['CODEOWNERS'], reason: 'review ownership is set by people' },
|
|
1005
1015
|
{ name: 'scanner-frozen', paths: ['.promptwheel/packs/**'], reason: 'the scanners that read the contract are not edited by the change they read' },
|
|
1006
1016
|
{ name: 'lock-frozen', paths: ['promptwheel.lock.json'], allowable: 'flag', reason: 'the architecture lock changes with a reason from a person (--allow), never as a side effect of a change' },
|
|
1007
|
-
{ name: 'hooks-frozen', paths: ['.claude/settings.json', '.claude/settings.local.json', '.sawbuck/**'], reason: 'the hooks that enforce the contract
|
|
1017
|
+
{ name: 'hooks-frozen', paths: ['.claude/settings.json', '.claude/settings.local.json', '.sawbuck/**'], allowable: 'flag', reason: 'the hooks that enforce the contract change with a reason from a person (--allow), and are never detached by the agent' },
|
|
1008
1018
|
],
|
|
1009
1019
|
metrics: [
|
|
1010
1020
|
{ name: 'lifecycle_scripts', cmd: 'node packs/supply-chain/scan.mjs lifecycle .', extract: 'number', direction: 'down', guard: true, gamingCheck: false, allowable: 'flag' },
|
|
@@ -1017,32 +1027,17 @@ const PRESETS = {
|
|
|
1017
1027
|
] },
|
|
1018
1028
|
};
|
|
1019
1029
|
|
|
1020
|
-
|
|
1021
|
-
// (so an agent that suppresses a security rule, # noqa / # nosec / eslint-disable, or
|
|
1022
|
-
// skips a security test is caught), PLUS a starter `evaluatorPaths` that protects common
|
|
1023
|
-
// security-gate config files (a ruleset or a budget the agent could otherwise loosen to
|
|
1024
|
-
// pass). The scanner pack installs into `.promptwheel/` (already evaluator-protected). It
|
|
1025
|
-
// is the "guard the guards" preset: pairs with --detect-gaming so a win that only moved by
|
|
1026
|
-
// loosening the security gate, not by fixing the code, is GAMED. Blind spot (by design,
|
|
1027
|
-
// not this tool's job): a source-resident backdoor that behaves safely only under test;
|
|
1028
|
-
// that needs held-out/behavioral analysis, not a source-only re-run.
|
|
1029
|
-
PRESETS['security-antihack'] = {
|
|
1030
|
-
desc: 'gate security-invariant findings + antihack tripwires + evaluatorPaths so an agent cannot loosen the security gate to pass it (pairs with --detect-gaming)',
|
|
1031
|
-
evaluatorPaths: ['.semgrep.yml', '.semgrep.yaml', '.bandit', 'bandit.yaml', '.ruff.toml', 'ruff.toml', '.snyk', 'security-budget.json'],
|
|
1032
|
-
metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }, ...PRESETS.antihack.metrics.slice(1)],
|
|
1033
|
-
};
|
|
1034
|
-
|
|
1035
|
-
function detectTestCmd(repo) {
|
|
1036
|
-
const has = (f) => existsSync(join(repo, f));
|
|
1030
|
+
function detectTestCmd(repo, { has, read } = stackAt(repo)) {
|
|
1037
1031
|
if (has('go.mod')) return 'go test ./...';
|
|
1038
1032
|
if (has('Cargo.toml')) return 'cargo test';
|
|
1039
|
-
|
|
1033
|
+
// the venv is not tracked: it is looked for on disk whichever ref the stack is read from
|
|
1034
|
+
if (has('pyproject.toml') || has('setup.py') || has('pytest.ini')) return existsSync(join(repo, '.venv/bin/pytest')) ? '.venv/bin/pytest -q' : 'pytest -q';
|
|
1040
1035
|
// only trust `npm test` when a test script actually exists, otherwise the metric can
|
|
1041
1036
|
// never pass and the gate would "protect" nothing (very common in Next.js app repos)
|
|
1042
1037
|
try {
|
|
1043
|
-
if (JSON.parse(
|
|
1038
|
+
if (JSON.parse(read('package.json')).scripts?.test) return 'npm test --silent';
|
|
1044
1039
|
} catch { /* no or unparsable package.json, fall through */ }
|
|
1045
|
-
return
|
|
1040
|
+
return PLACEHOLDER_TEST;
|
|
1046
1041
|
}
|
|
1047
1042
|
|
|
1048
1043
|
// deps that live outside git + the env a test needs, defaulted per stack. init writes these into
|
|
@@ -1051,18 +1046,16 @@ function detectTestCmd(repo) {
|
|
|
1051
1046
|
// where the detected test command cannot run without a build first (a 10-repo replay put this at
|
|
1052
1047
|
// ~2/3 of gates). Written by `init` into `setup`; a failed setup stays honest (inert -> inconclusive,
|
|
1053
1048
|
// never a fake green). Kept out of the zero-config default so the fast per-turn path stays fast.
|
|
1054
|
-
function detectSetup(repo) {
|
|
1055
|
-
const has = (f) => existsSync(join(repo, f));
|
|
1049
|
+
function detectSetup(repo, { has, read } = stackAt(repo)) {
|
|
1056
1050
|
try {
|
|
1057
|
-
const pkg = JSON.parse(
|
|
1051
|
+
const pkg = JSON.parse(read('package.json'));
|
|
1058
1052
|
if (pkg.scripts && pkg.scripts.build) return 'npm run build --if-present'; // TS/bundler tests that run against a built dir
|
|
1059
1053
|
} catch { /* no or unparsable package.json */ }
|
|
1060
1054
|
if (has('pyproject.toml') || has('setup.py')) return 'pip install -e . --quiet 2>/dev/null || true'; // editable install so imports resolve the measured ref
|
|
1061
1055
|
return undefined;
|
|
1062
1056
|
}
|
|
1063
1057
|
|
|
1064
|
-
function detectEnv(repo) {
|
|
1065
|
-
const has = (f) => existsSync(join(repo, f));
|
|
1058
|
+
function detectEnv(repo, { has } = stackAt(repo)) {
|
|
1066
1059
|
// Python: link the venv (3rd-party deps) and put the worktree src FIRST on PYTHONPATH, so imports
|
|
1067
1060
|
// resolve the MEASURED ref instead of an editable install pointing back at your original checkout.
|
|
1068
1061
|
if (has('pyproject.toml') || has('setup.py') || has('pytest.ini')) return { linkDirs: ['.venv'], env: { PYTHONPATH: '{wt}/src:{wt}' } }; // src/ AND flat layouts; worktree src wins over an editable install
|
|
@@ -1104,10 +1097,9 @@ function init(argv) {
|
|
|
1104
1097
|
if (presetName) {
|
|
1105
1098
|
const p = PRESETS[presetName];
|
|
1106
1099
|
if (!p) die(`unknown preset "${presetName}", try: promptwheel init --list`);
|
|
1107
|
-
evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (
|
|
1100
|
+
evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (supply-chain: the contract)
|
|
1108
1101
|
mandates = p.mandates; allowTrailer = p.allowTrailer; // a preset may ship hard rules (supply-chain: workflows, registries, publish scripts, hooks)
|
|
1109
|
-
metrics = p.metrics
|
|
1110
|
-
metrics = metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
|
|
1102
|
+
metrics = p.metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
|
|
1111
1103
|
// pack-backed presets: the referenced scripts live in THIS package, not
|
|
1112
1104
|
// the user's repo, copy them in so the generated config is
|
|
1113
1105
|
// self-contained and committable (a bare reference silently 404'd)
|
|
@@ -1123,8 +1115,7 @@ function init(argv) {
|
|
|
1123
1115
|
const files = readdirSync(src);
|
|
1124
1116
|
for (const f of files) copyFileSync(join(src, f), join(dst, f));
|
|
1125
1117
|
console.log(` ✓ installed ${pk[2]} → .promptwheel/${pk[2]} (commit it with the config)`);
|
|
1126
|
-
// the gate measures in clean worktrees, so a pack that git ignores
|
|
1127
|
-
// record lives in .promptwheel/ too) is absent at both refs and the metric reads
|
|
1118
|
+
// the gate measures in clean worktrees, so a pack that git ignores is absent at both refs and the metric reads
|
|
1128
1119
|
// unmeasurable, failing the guard with no hint why. Say so now, while it is one edit.
|
|
1129
1120
|
if (files.length && gitIgnores(repo, `.promptwheel/${pk[2]}/${files[0]}`))
|
|
1130
1121
|
console.log(` ⚠ .promptwheel/${pk[2]} is gitignored: the gate measures in clean worktrees, so an ignored pack is absent at both refs and ${m.name} reads unmeasurable (the guard fails). Replace ".promptwheel/" in .gitignore with ".promptwheel/*" and "!.promptwheel/packs/", then commit the pack`);
|
|
@@ -1154,7 +1145,7 @@ function init(argv) {
|
|
|
1154
1145
|
if (wrote.length) console.log(`✓ wrote the agent guide (when to run the gate) to ${wrote.join(', ')}`);
|
|
1155
1146
|
}
|
|
1156
1147
|
console.log('\n next: promptwheel run --working # gate your uncommitted changes');
|
|
1157
|
-
console.log(' promptwheel init --list # other presets (
|
|
1148
|
+
console.log(' promptwheel init --list # other presets (antihack, no-ai-tells, supply-chain)');
|
|
1158
1149
|
}
|
|
1159
1150
|
|
|
1160
1151
|
// Write (or refresh) a short agent-orienting block so the coding AGENT runs the gate itself, not just
|
|
@@ -1177,36 +1168,9 @@ function writeAgentBlock(repo) {
|
|
|
1177
1168
|
return done;
|
|
1178
1169
|
}
|
|
1179
1170
|
|
|
1180
|
-
// observability: list the EFFECTIVE guardrails (incl. inherited) + each one's flag record
|
|
1181
|
-
function guards(argv) {
|
|
1182
|
-
const args = parseArgs(argv);
|
|
1183
|
-
const repo = repoRoot(process.cwd());
|
|
1184
|
-
const cfg = loadConfig(repo);
|
|
1185
|
-
const hist = {};
|
|
1186
|
-
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
1187
|
-
if (existsSync(f)) for (const line of readFileSync(f, 'utf8').split('\n').filter(Boolean)) {
|
|
1188
|
-
let r; try { r = JSON.parse(line); } catch { continue; }
|
|
1189
|
-
for (const m of (r.metrics || [])) {
|
|
1190
|
-
const h = hist[m.name] ??= { runs: 0, flagged: 0, last: null };
|
|
1191
|
-
h.runs++; if (m.guard && m.ok === false) h.flagged++; h.last = m.status;
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
|
-
const rows = cfg.metrics.map((m) => ({
|
|
1195
|
-
name: m.name, direction: m.direction || 'up', guard: !!m.guard,
|
|
1196
|
-
source: m.__src, override: !!m.__override, ...(hist[m.name] || { runs: 0, flagged: 0, last: null }),
|
|
1197
|
-
}));
|
|
1198
|
-
if (args.json) { console.log(JSON.stringify({ guards: rows }, null, 2)); return; }
|
|
1199
|
-
console.log('\nPromptWheel guardrails, effective set for this repo\n');
|
|
1200
|
-
for (const r of rows) {
|
|
1201
|
-
const prov = r.override ? 'local override' : (r.source === 'local' ? 'local' : `inherited ← ${r.source}`);
|
|
1202
|
-
const rec = r.guard ? (r.runs ? `· ${r.flagged} flagged / ${r.runs} runs` : '· no runs yet') : '';
|
|
1203
|
-
console.log(` ${r.guard ? '🛡️ GUARD' : '· info '} ${r.name.padEnd(18)} better=${r.direction.padEnd(5)} [${prov}] ${rec}`);
|
|
1204
|
-
}
|
|
1205
|
-
console.log('\n 🛡️ = enforced (a trusted regression fails the gate) · info = tracked only\n');
|
|
1206
|
-
}
|
|
1207
|
-
|
|
1208
1171
|
const short = (repo, ref) => { try { return git(['rev-parse', '--short', ref], repo); } catch { return ref; } };
|
|
1209
1172
|
|
|
1173
|
+
const wouldHave = (status) => (status === 'unmeasurable' ? 'been unmeasurable' : status);
|
|
1210
1174
|
function printHuman(r) {
|
|
1211
1175
|
const arrowFor = (m) => (m.delta == null ? '?' : m.delta > 0 ? '▲' : m.delta < 0 ? '▼' : '=');
|
|
1212
1176
|
console.log(`\nPromptWheel ${r.base} → ${r.head}${r.repeat > 1 ? ` (×${r.repeat})` : ''}\n`);
|
|
@@ -1215,16 +1179,22 @@ function printHuman(r) {
|
|
|
1215
1179
|
const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
|
|
1216
1180
|
const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
|
|
1217
1181
|
console.log(` ${arrowFor(m)} ${m.name.padEnd(18)} ${String(m.before).padStart(8)} → ${String(m.after).padStart(8)} (${d}, ${m.status}) [${tag}, ${m.confidence}]`);
|
|
1218
|
-
if (m.inert) console.log(` ⚠ "${m.name}"
|
|
1182
|
+
if (m.inert && m.ran === 'both') console.log(` ⚠ "${m.name}" ran and failed before and after this change, so it guards nothing here. Fix the failing tests, or the command, before trusting a pass`);
|
|
1183
|
+
else if (m.inert) console.log(` ⚠ "${m.name}" ${m.ran ? `ran and failed at the ${m.ran} ref and did not run at the other` : 'never ran (same value at both refs)'}: its command needs a build/deps first, or is not your real test command. Fix: promptwheel init, or set setup/linkDirs in promptwheel.config.json`);
|
|
1219
1184
|
if (m.gamed === true) console.log(` 🚩 GAMED, ${m.gamingReason}`);
|
|
1220
1185
|
}
|
|
1221
|
-
if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(
|
|
1222
|
-
|
|
1186
|
+
if (r.gate && r.gate.weakened) for (const w of r.gate.weakened) console.log(w.change === 'measurement-edited'
|
|
1187
|
+
? ` 🚩 GATE WEAKENED, how the guard "${w.name}" is measured (setup, env or linked dirs) was edited in this change, and measured the way it was before it would have ${wouldHave(w.status)} at head (goalpost moved, not earned)`
|
|
1188
|
+
: ` 🚩 GATE WEAKENED, the guard "${w.name}" was ${w.change === 'guard-removed' ? 'removed' : w.change === 'guard-edited' ? 'edited' : 'disabled'} in this change and it would have ${wouldHave(w.status)} at head (goalpost moved, not earned)`);
|
|
1189
|
+
if (r.gate && r.gate.changed) console.log(` ⚠ gate definition changed in this change (fingerprint ${r.gate.fingerprint.base} -> ${r.gate.fingerprint.head}): a metric, guard, evaluatorPath, mandate, or the setup, env or linked dirs they are measured with was added, removed or edited; review promptwheel.config.json as part of the diff`);
|
|
1223
1190
|
const failReason = r.mandates ? ', a mandate was violated (a protected path changed)' : ', a guarded metric regressed (beyond noise)';
|
|
1224
1191
|
console.log(`\n VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? failReason : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
|
|
1225
|
-
if (r.verdict === 'fail' && r.mandates)
|
|
1226
|
-
|
|
1227
|
-
|
|
1192
|
+
if (r.verdict === 'fail' && r.mandates) {
|
|
1193
|
+
const takes = r.mandates.filter((md) => md.allowable).map((md) => `"${md.name}"`);
|
|
1194
|
+
if (takes.length) console.log(` intentional? ${takes.join(', ')} ${takes.length > 1 ? 'accept' : 'accepts'} a person's declared reason: promptwheel run --allow "<reason>" passes with the reason on the record`);
|
|
1195
|
+
if (takes.length < r.mandates.length) console.log(` ${takes.length ? 'any other' : 'a'} mandate is a hard rule; if the change is truly intended, retire the mandate in promptwheel.config.json in a change of its own first (a tracked, tamper-evident change), do not work around it`);
|
|
1196
|
+
} else if (r.verdict === 'fail' && r.metrics.some((m) => m.allowable)) console.log(` intentional? this guard accepts a person's declared reason: promptwheel run --allow "<reason>" passes with the reason on the record (${r.metrics.filter((m) => m.allowable).map((m) => m.name).join(', ')})`);
|
|
1197
|
+
else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name)');
|
|
1228
1198
|
if (r.allowed) console.log(` ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Promptwheel-Allow: ${r.allowed.reason})`);
|
|
1229
1199
|
console.log('');
|
|
1230
1200
|
}
|
|
@@ -1277,9 +1247,10 @@ function renderMarkdown(r) {
|
|
|
1277
1247
|
'|---|--:|--:|--:|---|---|',
|
|
1278
1248
|
rows,
|
|
1279
1249
|
'',
|
|
1280
|
-
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A protected path changed. This is a hard rule
|
|
1281
|
-
...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : 'disabled'} (would have ${w.status} at head)`).join('; ')}. The pass was bought by
|
|
1282
|
-
...(r.
|
|
1250
|
+
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A protected path changed. This is a hard rule; no metric result clears it.' : '> ❌ A 🛡️ guarded metric regressed beyond the noise band.') : r.verdict === 'inconclusive' ? (r.metrics.some((m) => m.inert) && r.metrics.every((m) => !m.inert || m.ran === 'both') ? '> 🟡 A 🛡️ guarded check is **inert**, it fails before and after this change, so nothing was actually verified. Fix it before trusting a pass.' : '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.') : '> ✅ No guarded metric regressed.',
|
|
1251
|
+
...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : w.change === 'guard-edited' ? 'edited' : w.change === 'measurement-edited' ? 'measured another way (setup, env or linked dirs)' : 'disabled'} (would have ${wouldHave(w.status)} at head)`).join('; ')}. The pass was bought by changing the gate, not by earning it.`] : []),
|
|
1252
|
+
...(r.allowed ? ['', `> ✓ **Allowed:** \`${r.allowed.cleared.join('`, `')}\` cleared by a declared reason: ${r.allowed.reason}`] : []),
|
|
1253
|
+
...(r.gate && r.gate.changed ? ['', `> ⚠ **Gate definition changed** in this diff (fingerprint \`${r.gate.fingerprint.base}\` -> \`${r.gate.fingerprint.head}\`): a metric, guard, evaluatorPath, mandate, or the setup, env or linked dirs they are measured with was added, removed or edited. Review the config change with the code.`] : []),
|
|
1283
1254
|
'',
|
|
1284
1255
|
'<sub>🛡️ = guard · _prove every change moved a metric_ · [PromptWheel](https://github.com/promptwheel-ai/promptwheel)</sub>',
|
|
1285
1256
|
].join('\n');
|
|
@@ -1292,12 +1263,10 @@ function parseArgs(argv) {
|
|
|
1292
1263
|
else if (argv[i] === '--head') a.head = argv[++i];
|
|
1293
1264
|
else if (argv[i] === '--repeat') a.repeat = parseInt(argv[++i], 10);
|
|
1294
1265
|
else if (argv[i] === '--working') a.working = true;
|
|
1295
|
-
else if (argv[i] === '--no-record') a.noRecord = true;
|
|
1296
1266
|
else if (argv[i] === '--no-auto') a.noAuto = true;
|
|
1297
1267
|
else if (argv[i] === '--allow') a.allow = argv[++i];
|
|
1298
1268
|
else if (argv[i] === '--detect-gaming') a.detectGaming = true;
|
|
1299
1269
|
else if (argv[i] === '--no-detect-gaming') a.detectGaming = false;
|
|
1300
|
-
else if (argv[i] === '--attempt') a.attempt = argv[++i];
|
|
1301
1270
|
else if (argv[i] === '--json') a.json = true;
|
|
1302
1271
|
else if (argv[i] === '--markdown') a.markdown = true;
|
|
1303
1272
|
}
|
|
@@ -1314,7 +1283,7 @@ export { extract, evaluate, median, spread, renderMarkdown, isNonSource, evaluat
|
|
|
1314
1283
|
// Importing this file never runs the CLI (see the isMain guard at the bottom).
|
|
1315
1284
|
// Stable across 0.x minor versions. The schema check() returns and the exit codes
|
|
1316
1285
|
// verdictExitCode() maps are specified in CONTRACT.md; integration paths (conform,
|
|
1317
|
-
// shell, import
|
|
1286
|
+
// shell, import) are in INTEGRATE.md.
|
|
1318
1287
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
1319
1288
|
|
|
1320
1289
|
// The contract revision this build implements (CONTRACT.md "Versioning"). Bumps only
|
|
@@ -1351,10 +1320,9 @@ export function verdictExitCode(verdict) {
|
|
|
1351
1320
|
// repeat number measurement repeats for the noise band (default: config.repeat ?? 1)
|
|
1352
1321
|
// detectGaming boolean run the Section 3 source-only re-proof (default: true)
|
|
1353
1322
|
// allow string a declared override reason (same as --allow / a Promptwheel-Allow: trailer)
|
|
1354
|
-
// record boolean append the run to .promptwheel/outcomes.jsonl (default: false)
|
|
1355
1323
|
export function check(options = {}) {
|
|
1356
|
-
// `repo` may name a subdirectory: the gate runs at the toplevel (that is where the config
|
|
1357
|
-
//
|
|
1324
|
+
// `repo` may name a subdirectory: the gate runs at the toplevel (that is where the config
|
|
1325
|
+
// lives), exactly as the CLI does from the same directory
|
|
1358
1326
|
let start = process.cwd();
|
|
1359
1327
|
if (options.repo) { try { start = realpathSync(options.repo); } catch { bad(`promptwheel: not inside a git repository (${options.repo})`); } }
|
|
1360
1328
|
const repo = repoRoot(start);
|
|
@@ -1376,7 +1344,6 @@ export function check(options = {}) {
|
|
|
1376
1344
|
repeat: options.repeat,
|
|
1377
1345
|
detectGaming: options.detectGaming ?? true,
|
|
1378
1346
|
allow: options.allow,
|
|
1379
|
-
noRecord: options.record !== true,
|
|
1380
1347
|
});
|
|
1381
1348
|
}
|
|
1382
1349
|
|
|
@@ -1466,72 +1433,44 @@ export function conformsToContract(report) {
|
|
|
1466
1433
|
return { ok: errors.length === 0, errors };
|
|
1467
1434
|
}
|
|
1468
1435
|
|
|
1469
|
-
// `promptwheel conformance [file.json | -] [--cmd "<command>"]`
|
|
1470
|
-
// Self-check a Contract implementation. With a file or stdin, validate that verdict JSON
|
|
1471
|
-
// against the Section 2 schema. With --cmd, run the command, then validate its --json
|
|
1472
|
-
// stdout (Section 2) AND that its exit code matches the verdict per Section 1. Exit 0 if
|
|
1473
|
-
// it conforms, 1 otherwise, listing every problem.
|
|
1474
|
-
function conformance(argv) {
|
|
1475
|
-
let cmd = null, src = null;
|
|
1476
|
-
for (let i = 0; i < argv.length; i++) {
|
|
1477
|
-
if (argv[i] === '--cmd') cmd = argv[++i];
|
|
1478
|
-
else if (argv[i] === '-' || !argv[i].startsWith('-')) src = argv[i];
|
|
1479
|
-
}
|
|
1480
|
-
|
|
1481
|
-
let text, exitCode = null;
|
|
1482
|
-
if (cmd) {
|
|
1483
|
-
try { text = execFileSync('sh', ['-c', cmd], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); exitCode = 0; }
|
|
1484
|
-
catch (e) {
|
|
1485
|
-
text = (e.stdout || '').toString();
|
|
1486
|
-
exitCode = typeof e.status === 'number' ? e.status : null;
|
|
1487
|
-
if (!text) { console.error(`conformance: command produced no stdout (exit ${exitCode}): ${e.message}`); process.exit(1); }
|
|
1488
|
-
}
|
|
1489
|
-
} else {
|
|
1490
|
-
const file = src && src !== '-' ? src : 0; // fd 0 = stdin
|
|
1491
|
-
try { text = readFileSync(file, 'utf8'); }
|
|
1492
|
-
catch (e) { console.error(`conformance: cannot read ${!src || src === '-' ? 'stdin' : src}: ${e.message}`); process.exit(1); }
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
let report;
|
|
1496
|
-
try { report = JSON.parse(text); }
|
|
1497
|
-
catch (e) { console.error(`conformance: not valid JSON: ${e.message}`); process.exit(1); }
|
|
1498
|
-
|
|
1499
|
-
const { errors } = conformsToContract(report);
|
|
1500
|
-
const problems = [...errors];
|
|
1501
|
-
if (cmd && exitCode !== null && VERDICTS.includes(report.verdict)) {
|
|
1502
|
-
const want = verdictExitCode(report.verdict);
|
|
1503
|
-
if (exitCode !== want) problems.push(`exit code ${exitCode} does not match verdict '${report.verdict}' (Section 1 requires ${want})`);
|
|
1504
|
-
}
|
|
1505
|
-
|
|
1506
|
-
if (problems.length === 0) {
|
|
1507
|
-
console.log(`ok, conforms to the Verified-Outcome Contract v${CONTRACT_VERSION} (verdict: ${report.verdict}${cmd ? `, exit ${exitCode}` : ''})`);
|
|
1508
|
-
process.exit(0);
|
|
1509
|
-
}
|
|
1510
|
-
console.error(`does NOT conform to the Contract (${problems.length} problem${problems.length > 1 ? 's' : ''}):`);
|
|
1511
|
-
for (const p of problems) console.error(` - ${p}`);
|
|
1512
|
-
process.exit(1);
|
|
1513
|
-
}
|
|
1514
|
-
|
|
1515
1436
|
// the flags each command accepts. An unknown --flag is a usage error (exit 64), never silently
|
|
1516
1437
|
// ignored: a typo in a CI step must not turn `--no-detect-gaming` into the default gate or `--allow`
|
|
1517
1438
|
// into nothing, and INTEGRATE.md promises a bad flag exits 64. A flag that takes a value consumes
|
|
1518
|
-
// the next argument; anything not starting with `-`
|
|
1439
|
+
// the next argument; anything not starting with `-` is positional. --no-record does nothing now that
|
|
1440
|
+
// no record is written, and stays accepted because Sawbuck and existing CI steps pass it.
|
|
1519
1441
|
const GATE_FLAGS = ['--base', '--head', '--repeat', '--working', '--no-record', '--no-auto', '--allow', '--detect-gaming', '--no-detect-gaming', '--json', '--markdown'];
|
|
1520
1442
|
const COMMAND_FLAGS = {
|
|
1521
|
-
run: GATE_FLAGS,
|
|
1522
|
-
init: ['--preset', '--list', '--force', '--no-agent-block'], conformance: ['--cmd'], lock: [],
|
|
1443
|
+
run: GATE_FLAGS, init: ['--preset', '--list', '--force', '--no-agent-block'], lock: [],
|
|
1523
1444
|
};
|
|
1524
|
-
|
|
1445
|
+
// what each valued flag needs. A missing or unusable value is a usage error too: `--repeat abc` must not
|
|
1446
|
+
// measure nothing and read FAIL, and `--allow` with nothing after it must not run as if no reason was asked for.
|
|
1447
|
+
const VALUED_FLAGS = { '--base': 'a ref', '--head': 'a ref', '--repeat': 'a whole number, 1 or more', '--allow': 'a reason, e.g. --allow "the spec changed"', '--preset': 'a preset name (see promptwheel init --list)' };
|
|
1525
1448
|
function checkFlags(cmd, argv) {
|
|
1526
1449
|
const known = new Set(COMMAND_FLAGS[cmd]);
|
|
1527
1450
|
for (let i = 0; i < argv.length; i++) {
|
|
1528
1451
|
const a = argv[i];
|
|
1529
|
-
if (
|
|
1452
|
+
if (!a.startsWith('-')) continue;
|
|
1530
1453
|
if (!known.has(a)) bad(`promptwheel ${cmd}: unknown flag ${a} (see promptwheel --help)`);
|
|
1531
|
-
if (
|
|
1454
|
+
if (!(a in VALUED_FLAGS)) continue;
|
|
1455
|
+
const v = argv[++i];
|
|
1456
|
+
const gone = v === undefined || known.has(v); // nothing after the flag, or the next flag
|
|
1457
|
+
// an empty --base or --head is the default ref: the Action passes an empty base outside a pull request
|
|
1458
|
+
const blank = !gone && !v.trim() && !(v === '' && (a === '--base' || a === '--head'));
|
|
1459
|
+
if (gone || blank) bad(`promptwheel ${cmd}: ${a} needs ${VALUED_FLAGS[a]}`);
|
|
1460
|
+
if (a === '--repeat' && !/^[1-9]\d*$/.test(v)) bad(`promptwheel ${cmd}: --repeat needs ${VALUED_FLAGS[a]} (got "${v}")`);
|
|
1532
1461
|
}
|
|
1533
1462
|
}
|
|
1534
1463
|
|
|
1464
|
+
// read PR comments on stdin: a JSON array, or one comment per line as `gh api --paginate --jq '.[]'` prints every
|
|
1465
|
+
// page. Streamed: readFileSync(0) corrupted the heap on Node 24.22 reading a live paginated gh pipe.
|
|
1466
|
+
async function prAllow() {
|
|
1467
|
+
let t = ''; process.stdin.setEncoding('utf8');
|
|
1468
|
+
for await (const chunk of process.stdin) t += chunk;
|
|
1469
|
+
t = t.trim(); let c = [];
|
|
1470
|
+
try { c = [].concat(JSON.parse(t || '[]')); } catch { try { c = t.split('\n').filter(Boolean).map((l) => JSON.parse(l)); } catch { /* unreadable, no approval */ } }
|
|
1471
|
+
process.stdout.write(allowFromComments(c));
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1535
1474
|
// write or refresh the architecture lock from the tree as it stands. A person runs this: the lock file is
|
|
1536
1475
|
// a mandate, so the refreshed file passes the gate only with --allow "<reason>".
|
|
1537
1476
|
async function lock() {
|
|
@@ -1546,13 +1485,9 @@ function dispatch() {
|
|
|
1546
1485
|
const cmd = (rest.includes('--help') || rest.includes('-h')) ? 'help' : first; // `promptwheel run --help` is a help request, not a gate run
|
|
1547
1486
|
if (cmd in COMMAND_FLAGS) checkFlags(cmd, rest);
|
|
1548
1487
|
if (cmd === 'run') run(rest);
|
|
1549
|
-
else if (cmd === '
|
|
1550
|
-
else if (cmd === 'pr-allow') { let c = []; try { c = JSON.parse(readFileSync(0, 'utf8') || '[]'); } catch { /* no comments, no approval */ } process.stdout.write(allowFromComments(c)); }
|
|
1551
|
-
else if (cmd === 'insights') insights(rest);
|
|
1488
|
+
else if (cmd === 'pr-allow') return prAllow();
|
|
1552
1489
|
else if (cmd === 'init') init(rest);
|
|
1553
|
-
else if (cmd === '
|
|
1554
|
-
else if (cmd === 'conformance') conformance(rest);
|
|
1555
|
-
else if (cmd === 'lock') lock(rest);
|
|
1490
|
+
else if (cmd === 'lock') return lock(rest);
|
|
1556
1491
|
else {
|
|
1557
1492
|
const help = !cmd || cmd === '--help' || cmd === '-h' || cmd === 'help';
|
|
1558
1493
|
if (!help) console.error(`unknown command: ${cmd}`);
|
|
@@ -1567,16 +1502,11 @@ function dispatch() {
|
|
|
1567
1502
|
' catches reward-hacking BY DEFAULT, re-proves each win from SOURCE edits alone (verdict GAMED,',
|
|
1568
1503
|
' exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
|
|
1569
1504
|
' promptwheel run --working gate uncommitted changes (incl. newly added files)',
|
|
1505
|
+
' promptwheel pr-allow < comments.json print the newest `sawbuck-allow: <reason>` from a PR comment by someone with write access (the Action uses it)',
|
|
1570
1506
|
' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `Promptwheel-Allow: <reason>` commit trailer; `Sawbuck-Allow:` is an alias): recorded, not flagged GAMED',
|
|
1571
|
-
' promptwheel improve --attempt "<cmd>" run an agent/script; keep only if a metric improved',
|
|
1572
|
-
' exit 0=kept · 1=regression · 2=gamed · 3=plateau/inconclusive (all but 0 reverted) · add --json',
|
|
1573
|
-
' promptwheel insights [--json] which metrics actually respond (raw counts from .promptwheel/outcomes.jsonl)',
|
|
1574
|
-
' promptwheel guards [--json] show the effective guardrails (incl. inherited) + flag record',
|
|
1575
|
-
' promptwheel conformance [file|-] [--cmd C] check a verdict JSON (or a command) against the Contract (exit 0 = conforms)',
|
|
1576
1507
|
'',
|
|
1577
1508
|
'Exit codes: 0 pass · 1 fail (a guarded metric regressed, or a mandate was violated) · 2 gamed · 3 inconclusive · 64 usage or config error (never a verdict).',
|
|
1578
|
-
'Integrate (library / conform / shell
|
|
1579
|
-
'Loop it: while promptwheel improve --attempt "$AGENT"; do :; done # stops on plateau/regression',
|
|
1509
|
+
'Integrate (library / conform / shell): see INTEGRATE.md. Import: `import { check } from \'promptwheel\'`.',
|
|
1580
1510
|
'Config: promptwheel.config.json → { extends?, metrics:[{ name, cmd, direction, extract?, guard? }] } (or: promptwheel init)',
|
|
1581
1511
|
' extends: a path (or array) to a shared base config, repos INHERIT its guardrails; local metrics override by name.',
|
|
1582
1512
|
].join('\n'));
|
|
@@ -1587,8 +1517,8 @@ function dispatch() {
|
|
|
1587
1517
|
// run the CLI only when invoked directly (resolves symlinks so the npm bin still works)
|
|
1588
1518
|
let isMain = false;
|
|
1589
1519
|
try { isMain = !!process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { /* not main */ }
|
|
1590
|
-
function main() {
|
|
1591
|
-
try { dispatch(); }
|
|
1520
|
+
async function main() {
|
|
1521
|
+
try { await dispatch(); }
|
|
1592
1522
|
catch (e) { if (e && e.name === 'ConfigError') die(e.message); throw e; }
|
|
1593
1523
|
}
|
|
1594
1524
|
if (isMain) main();
|