promptwheel 0.7.5 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INTEGRATE.md +3 -1
- package/README.md +1 -1
- package/bin/promptwheel.d.ts +314 -0
- package/bin/promptwheel.mjs +49 -15
- package/package.json +6 -2
- package/packs/security/README.md +14 -4
package/INTEGRATE.md
CHANGED
|
@@ -103,7 +103,9 @@ measurement runs in a throwaway git worktree.
|
|
|
103
103
|
|
|
104
104
|
Zero dependencies, no build step, the whole engine is one file. `check()` runs the gate in
|
|
105
105
|
process and returns the Section 2 object. It never prints and never calls `process.exit`; it
|
|
106
|
-
throws a plain `Error` on a precondition failure.
|
|
106
|
+
throws a plain `Error` on a precondition failure. Types ship in the package
|
|
107
|
+
(`bin/promptwheel.d.ts`, hand-written, no build step): `import type { Report, CheckOptions, Config } from 'promptwheel'`
|
|
108
|
+
works in TypeScript, and editors complete the options and the report in plain JavaScript.
|
|
107
109
|
|
|
108
110
|
```js
|
|
109
111
|
import { check, verdictExitCode, conformsToContract, VERDICTS, CONTRACT_VERSION } from 'promptwheel';
|
package/README.md
CHANGED
|
@@ -59,7 +59,7 @@ npx promptwheel init --list # presets: tests-pass · lint · bundl
|
|
|
59
59
|
# the loop: run any agent/script, keep the change ONLY if a metric improved, else revert
|
|
60
60
|
npx promptwheel improve --attempt "claude -p 'reduce lint errors'"
|
|
61
61
|
# exit 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed (reverted) · 3 = plateau / inconclusive (reverted) · add --json
|
|
62
|
-
while npx promptwheel improve --attempt "
|
|
62
|
+
while npx promptwheel improve --attempt "<your agent command>"; do :; done # converges: stops on the first non-zero
|
|
63
63
|
|
|
64
64
|
npx promptwheel insights # which metrics actually respond in this repo (counts from .promptwheel/outcomes.jsonl)
|
|
65
65
|
npx promptwheel guards # the effective guardrails, including ones inherited through extends
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
// Type declarations for bin/promptwheel.mjs. Hand-written: there is no build step and no
|
|
2
|
+
// TypeScript dependency in this package. Every runtime export of the engine is declared here
|
|
3
|
+
// (test/promptwheel.test.mjs checks the list stays complete). The report shape is the
|
|
4
|
+
// Verified-Outcome Contract's Section 2 verdict object; see INTEGRATE.md for the integration
|
|
5
|
+
// surface and CONTRACT.md for the protocol.
|
|
6
|
+
|
|
7
|
+
// ---------------------------------------------------------------------------
|
|
8
|
+
// Section 2 vocabularies
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
|
|
11
|
+
/** The verdict of a check. */
|
|
12
|
+
export type Verdict = 'pass' | 'fail' | 'gamed' | 'inconclusive';
|
|
13
|
+
|
|
14
|
+
/** The status of one metric. A delta inside the measured noise band is `inconclusive`, never a regression. */
|
|
15
|
+
export type MetricStatus = 'improved' | 'regressed' | 'unchanged' | 'inconclusive' | 'unmeasurable';
|
|
16
|
+
|
|
17
|
+
/** The direction that counts as better. `pass` is a 0/1 gate (1 = the command exited 0). */
|
|
18
|
+
export type Direction = 'up' | 'down' | 'pass';
|
|
19
|
+
|
|
20
|
+
/** How much a delta can be trusted. `none` only on an unmeasurable metric; `unverified` is a single read with no noise sample. */
|
|
21
|
+
export type Confidence = 'none' | 'unverified' | 'low' | 'medium' | 'high';
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* How a metric's value is read from its command: the last number in stdout (`number`, the
|
|
25
|
+
* default), the count of non-empty lines (`lines`), the exit code as 1/0 (`exit`), or the
|
|
26
|
+
* first capture group of a regex. A command that exits nonzero reads `null` (unmeasurable)
|
|
27
|
+
* unless the metric sets `allowNonzero`; `exit` mode consumes the exit code by design.
|
|
28
|
+
*/
|
|
29
|
+
export type ExtractMode = 'number' | 'lines' | 'exit' | { regex: string };
|
|
30
|
+
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
// Config (promptwheel.config.json, or the `config` option of check())
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
export interface MetricConfig {
|
|
36
|
+
name: string;
|
|
37
|
+
/** Shell command run in a throwaway worktree at each ref. */
|
|
38
|
+
cmd: string;
|
|
39
|
+
/** Default `'number'`. */
|
|
40
|
+
extract?: ExtractMode;
|
|
41
|
+
/** Default `'up'`. */
|
|
42
|
+
direction?: Direction;
|
|
43
|
+
/** A guarded metric fails the gate on a trusted regression; an unguarded one is tracked only. */
|
|
44
|
+
guard?: boolean;
|
|
45
|
+
/** Read a value even when `cmd` exits nonzero (never needed for `extract: 'exit'`). */
|
|
46
|
+
allowNonzero?: boolean;
|
|
47
|
+
/** Per-command timeout, default 300. */
|
|
48
|
+
timeoutSec?: number;
|
|
49
|
+
/** `false` skips the source-only re-proof for this metric (tripwires whose gain legitimately lives in test files). */
|
|
50
|
+
gamingCheck?: boolean;
|
|
51
|
+
/** Per-metric override of `Config.gamingThreshold`. */
|
|
52
|
+
gamingThreshold?: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** A named hard rule: any change touching one of `paths` fails the gate before any metric is read. */
|
|
56
|
+
export interface Mandate {
|
|
57
|
+
name: string;
|
|
58
|
+
/** Globs: a bare name matches that basename at any depth, a pattern with `/` is anchored at the repo root, `**` crosses directories. */
|
|
59
|
+
paths: string[];
|
|
60
|
+
reason?: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface Config {
|
|
64
|
+
/** Path (or paths) to a shared base config whose metrics and mandates this repo inherits; resolved relative to the config file. */
|
|
65
|
+
extends?: string | string[];
|
|
66
|
+
metrics: MetricConfig[];
|
|
67
|
+
mandates?: Mandate[];
|
|
68
|
+
/** Measurement repeats per metric, for the noise band. Default 1. */
|
|
69
|
+
repeat?: number;
|
|
70
|
+
/** Back-compat for `linkDirs`: `false` links nothing. */
|
|
71
|
+
linkNodeModules?: boolean;
|
|
72
|
+
/** Directories outside git to symlink into each worktree (default `['node_modules']`). */
|
|
73
|
+
linkDirs?: string[];
|
|
74
|
+
/** Extra environment for the metric commands; `{wt}` expands to the worktree path. */
|
|
75
|
+
env?: Record<string, string>;
|
|
76
|
+
/** A per-ref build or install, run in the worktree before measuring (a failed setup leaves the metric inert, never a false green). */
|
|
77
|
+
setup?: string;
|
|
78
|
+
/** `false` never appends to `.promptwheel/outcomes.jsonl`. */
|
|
79
|
+
record?: boolean;
|
|
80
|
+
/** Fraction of a gain that must survive the source-only re-run; below it the win is `gamed`. Default 0.5. */
|
|
81
|
+
gamingThreshold?: number;
|
|
82
|
+
/** Extra files that define the gate (a threshold, a budget); a change to one alongside a win is gamed by declaration. Same glob rules as `Mandate.paths`. */
|
|
83
|
+
evaluatorPaths?: string[];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
// check()
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
export interface CheckOptions {
|
|
91
|
+
/** Repo path. Default: discovered from `process.cwd()`. */
|
|
92
|
+
repo?: string;
|
|
93
|
+
/** Base ref. Default: the merge-base with `origin/main` (then `origin/master`, `main`, `master`); on the trunk itself, `HEAD~1`. */
|
|
94
|
+
base?: string;
|
|
95
|
+
/** Head ref. Default `HEAD`. */
|
|
96
|
+
head?: string;
|
|
97
|
+
/** Gate uncommitted changes (tracked and untracked) instead of `base..head`. Default `false`. */
|
|
98
|
+
working?: boolean;
|
|
99
|
+
/**
|
|
100
|
+
* A resolved config supplied in process, used as given (an `extends` chain is resolved only
|
|
101
|
+
* when the config is loaded from `promptwheel.config.json`). Omit to load the file, or to
|
|
102
|
+
* auto-detect the test command when there is none.
|
|
103
|
+
*/
|
|
104
|
+
config?: Config;
|
|
105
|
+
/** Allow the zero-config auto-detect fallback when there is no config file and no `config` option. Default `true`; `false` throws instead. */
|
|
106
|
+
auto?: boolean;
|
|
107
|
+
/** Measurement repeats for the noise band. Default `config.repeat ?? 1`. */
|
|
108
|
+
repeat?: number;
|
|
109
|
+
/** Run the Section 3 source-only re-proof. Default `true`. */
|
|
110
|
+
detectGaming?: boolean;
|
|
111
|
+
/** A declared override reason (the same as `--allow` or a `Promptwheel-Allow:` commit trailer): a would-be `gamed` reads `pass`, recorded in `report.allowed`. */
|
|
112
|
+
allow?: string;
|
|
113
|
+
/** Append the run to `.promptwheel/outcomes.jsonl`. Default `false`. */
|
|
114
|
+
record?: boolean;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ---------------------------------------------------------------------------
|
|
118
|
+
// The report (Section 2 verdict object)
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
/** What `evaluate()` returns for one metric: the before/after medians and the trust call on their delta. */
|
|
122
|
+
export interface Evaluation {
|
|
123
|
+
before: number | null;
|
|
124
|
+
after: number | null;
|
|
125
|
+
/** `null` when either side is unmeasurable. */
|
|
126
|
+
delta: number | null;
|
|
127
|
+
status: MetricStatus;
|
|
128
|
+
/** `false` only for a guarded metric with a trusted regression. */
|
|
129
|
+
ok: boolean;
|
|
130
|
+
confidence: Confidence;
|
|
131
|
+
/** The measured jitter band (max spread across repeats); `null` when unmeasurable. */
|
|
132
|
+
noise: number | null;
|
|
133
|
+
/** A guarded pass/fail metric stuck at 0 at both refs: it protects nothing. */
|
|
134
|
+
inert?: true;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface MetricResult extends Evaluation {
|
|
138
|
+
name: string;
|
|
139
|
+
direction: Direction;
|
|
140
|
+
guard: boolean;
|
|
141
|
+
/** `true` when the win did not survive the source-only re-run; `null` when the source-only patch did not apply. Absent without gaming detection or when the metric did not improve. */
|
|
142
|
+
gamed?: boolean | null;
|
|
143
|
+
/** The metric's value with only the source edits applied. */
|
|
144
|
+
sourceOnly?: number | null;
|
|
145
|
+
/** The fraction of the gain that survived source-only (0..1). */
|
|
146
|
+
retained?: number | null;
|
|
147
|
+
gamingReason?: string;
|
|
148
|
+
/** Set when a declared override cleared a `gamed` on this metric. */
|
|
149
|
+
allowed?: true;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export interface MandateViolation {
|
|
153
|
+
name: string;
|
|
154
|
+
reason: string;
|
|
155
|
+
/** The protected paths this change touched (at most 20). */
|
|
156
|
+
files: string[];
|
|
157
|
+
/** The mandate was deleted in this same change; it stays in force for the change that removes it. */
|
|
158
|
+
removed?: true;
|
|
159
|
+
/** The mandate's paths were edited in this same change; the version before the change stays in force for the change that edits it. */
|
|
160
|
+
edited?: true;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export interface GateWeakening {
|
|
164
|
+
name: string;
|
|
165
|
+
change: 'guard-removed' | 'guard-disabled';
|
|
166
|
+
/** What the dropped guard would have read at head under its base definition. */
|
|
167
|
+
status: MetricStatus;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export interface EvaluatorChange {
|
|
171
|
+
path: string;
|
|
172
|
+
/** Truncated SHA-256 of the file at each ref; `null` where the file is absent. */
|
|
173
|
+
base: string | null;
|
|
174
|
+
head: string | null;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The gate-integrity record, present when gaming detection ran. */
|
|
178
|
+
export interface GateRecord {
|
|
179
|
+
/** Truncated SHA-256 of the gate definition (metrics, guards, evaluatorPaths, gamingThreshold, mandates) at each ref; `base` is `null` when there was no config at base. */
|
|
180
|
+
fingerprint: { base: string | null; head: string };
|
|
181
|
+
/** The gate definition differs between base and head. */
|
|
182
|
+
changed?: true;
|
|
183
|
+
/** Guards the change dropped that would have failed at head; non-empty forces `gamed` unless allowed. */
|
|
184
|
+
weakened?: GateWeakening[];
|
|
185
|
+
/** Declared `evaluatorPaths` files whose content changed in this diff. */
|
|
186
|
+
evaluatorChanged?: EvaluatorChange[];
|
|
187
|
+
/** A declared override cleared the weakening. */
|
|
188
|
+
allowed?: true;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
export interface AllowedRecord {
|
|
192
|
+
reason: string;
|
|
193
|
+
/** The metric names (and `'gate'`) the override cleared. */
|
|
194
|
+
cleared: string[];
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Tamper-evident receipt: re-run `(base, head, config)` and `verdictHash` must match. Additive; never changes the verdict. */
|
|
198
|
+
export interface Provenance {
|
|
199
|
+
tool: 'promptwheel';
|
|
200
|
+
toolVersion: string | null;
|
|
201
|
+
baseSha: string | null;
|
|
202
|
+
headSha: string | null;
|
|
203
|
+
configHash: string;
|
|
204
|
+
verdictHash: string;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export interface Report {
|
|
208
|
+
/** Short SHA (or the ref as given when it cannot be resolved). */
|
|
209
|
+
base: string;
|
|
210
|
+
head: string;
|
|
211
|
+
mode: 'refs' | 'working';
|
|
212
|
+
repeat: number;
|
|
213
|
+
verdict: Verdict;
|
|
214
|
+
metrics: MetricResult[];
|
|
215
|
+
/** Present only when a mandate was violated (the verdict is then `fail`). */
|
|
216
|
+
mandates?: MandateViolation[];
|
|
217
|
+
gate?: GateRecord;
|
|
218
|
+
allowed?: AllowedRecord;
|
|
219
|
+
provenance: Provenance;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ---------------------------------------------------------------------------
|
|
223
|
+
// Public library API (stable across 0.x minors)
|
|
224
|
+
// ---------------------------------------------------------------------------
|
|
225
|
+
|
|
226
|
+
/** The Contract revision this build implements; bumps only when the verdict schema or the exit codes change. */
|
|
227
|
+
export declare const CONTRACT_VERSION: '1';
|
|
228
|
+
export declare const VERDICTS: readonly ['pass', 'fail', 'gamed', 'inconclusive'];
|
|
229
|
+
export declare const METRIC_STATUSES: readonly ['improved', 'regressed', 'unchanged', 'inconclusive', 'unmeasurable'];
|
|
230
|
+
export declare const DIRECTIONS: readonly ['up', 'down', 'pass'];
|
|
231
|
+
|
|
232
|
+
/** Section 1: `pass` 0, `fail` 1, `gamed` 2, `inconclusive` 3 (any other string maps to 1). The CLI exits with this. */
|
|
233
|
+
export declare function verdictExitCode(verdict: Verdict | string): 0 | 1 | 2 | 3;
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Run the gate in process and return the Section 2 verdict object (identical to `run --json`).
|
|
237
|
+
* Measures only in throwaway git worktrees, never the caller's working tree. Never prints and
|
|
238
|
+
* never calls `process.exit`; throws a plain `Error` on a precondition failure (not a git
|
|
239
|
+
* repository, `auto: false` with no config, a `config` without a non-empty `metrics` array).
|
|
240
|
+
*/
|
|
241
|
+
/**
|
|
242
|
+
* Runs the gate in process and returns the report. Never prints and never exits the process: a
|
|
243
|
+
* precondition failure (not a git repository, a missing, malformed or cyclic config, a missing
|
|
244
|
+
* `extends` target, no commits yet) throws an Error whose `name` is `'ConfigError'` and whose
|
|
245
|
+
* `exitCode` is 64, the code the CLI exits with for the same condition.
|
|
246
|
+
*/
|
|
247
|
+
export declare function check(options?: CheckOptions): Report;
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Structurally validate any verdict object against Section 2 (shapes, the fixed vocabularies,
|
|
251
|
+
* and the honest-verdict rules). Zero dependency, so a third-party implementation can self-check
|
|
252
|
+
* the artifact it emits. `errors` is empty exactly when `ok` is `true`.
|
|
253
|
+
*/
|
|
254
|
+
export declare function conformsToContract(report: unknown): { ok: boolean; errors: string[] };
|
|
255
|
+
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
// Pure helpers (exported for unit tests and for consumers that want the arithmetic)
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
/** Read a metric value from a command's stdout and exit code. */
|
|
261
|
+
export declare function extract(stdout: string, code: number, mode?: ExtractMode, allowNonzero?: boolean): number | null;
|
|
262
|
+
|
|
263
|
+
/** Median of the non-null samples; `null` when there are none. */
|
|
264
|
+
export declare function median(samples: ReadonlyArray<number | null | undefined>): number | null;
|
|
265
|
+
|
|
266
|
+
/** `max - min` of the non-null samples; `0` when there are none. */
|
|
267
|
+
export declare function spread(samples: ReadonlyArray<number | null | undefined>): number;
|
|
268
|
+
|
|
269
|
+
/** The credibility core: compare before/after samples and trust the delta only if it clears the observed noise band. */
|
|
270
|
+
export declare function evaluate(
|
|
271
|
+
metric: Pick<MetricConfig, 'direction' | 'guard' | 'extract'>,
|
|
272
|
+
beforeSamples: ReadonlyArray<number | null>,
|
|
273
|
+
afterSamples: ReadonlyArray<number | null>,
|
|
274
|
+
repeat: number,
|
|
275
|
+
): Evaluation;
|
|
276
|
+
|
|
277
|
+
/** Render a report as the PR-comment markdown the GitHub Action posts. */
|
|
278
|
+
export declare function renderMarkdown(report: Report): string;
|
|
279
|
+
|
|
280
|
+
/** The built-in classifier: is this repo-relative path test / config / grader / golden (not production source)? */
|
|
281
|
+
export declare function isNonSource(path: string): boolean;
|
|
282
|
+
|
|
283
|
+
/** Compile a config's `evaluatorPaths` globs into a matcher over repo-relative paths (always `false` when none are declared). */
|
|
284
|
+
export declare function evaluatorMatcher(config?: { evaluatorPaths?: string[] } | null): (path: string) => boolean;
|
|
285
|
+
|
|
286
|
+
/** Directional improvement of `value` over `before` (positive = better); `null` when either is missing. */
|
|
287
|
+
export declare function gain(metric: { direction?: Direction }, before: number | null, value: number | null): number | null;
|
|
288
|
+
|
|
289
|
+
/** The samples measured with only the source slice of the change applied. */
|
|
290
|
+
export interface SourceOnlyResult {
|
|
291
|
+
samples: ReadonlyArray<number | null>;
|
|
292
|
+
/** `false` when the change touched zero production-source files. */
|
|
293
|
+
hadSourceChange: boolean;
|
|
294
|
+
/** The source-only patch did not apply cleanly, so the judgement is inconclusive. */
|
|
295
|
+
applyFailed?: boolean;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export interface GamingJudgement {
|
|
299
|
+
/** `null` when the source-only patch did not apply. */
|
|
300
|
+
gamed: boolean | null;
|
|
301
|
+
sourceOnly: number | null;
|
|
302
|
+
retained: number | null;
|
|
303
|
+
reason: string;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** Did a metric's full-diff win survive on the source edits alone? `gamed` when less than `threshold` (default 0.5) of the gain is retained. */
|
|
307
|
+
export declare function judgeGaming(
|
|
308
|
+
metric: { direction?: Direction; before: number | null; after: number | null },
|
|
309
|
+
sourceOnly: SourceOnlyResult,
|
|
310
|
+
threshold?: number,
|
|
311
|
+
): GamingJudgement;
|
|
312
|
+
|
|
313
|
+
/** Truncated SHA-256 of the verdict content (everything but `provenance`), the reproducible half of the receipt. */
|
|
314
|
+
export declare function verdictFingerprint(report: Pick<Report, 'verdict'> & Partial<Pick<Report, 'metrics' | 'gate'>>): string;
|
package/bin/promptwheel.mjs
CHANGED
|
@@ -35,10 +35,13 @@ const spread = (xs) => { const a = xs.filter((x) => x != null); return a.length
|
|
|
35
35
|
// (0 pass, 1 fail, 2 gamed, 3 inconclusive): in CI a config typo must not read as GAMED.
|
|
36
36
|
const EX_USAGE = 64;
|
|
37
37
|
function die(msg) { console.error(msg); process.exit(EX_USAGE); }
|
|
38
|
+
// a config or precondition error: thrown, so the library path (check()) surfaces it to the caller
|
|
39
|
+
// instead of exiting the host process; the CLI catches it in main() and exits EX_USAGE.
|
|
40
|
+
function bad(msg) { const e = new Error(msg); e.name = 'ConfigError'; e.exitCode = EX_USAGE; throw e; }
|
|
38
41
|
|
|
39
42
|
function readConfigFile(p) {
|
|
40
43
|
try { return JSON.parse(readFileSync(p, 'utf8')); }
|
|
41
|
-
catch (e) {
|
|
44
|
+
catch (e) { bad(`invalid config ${p}: ${e.message}`); }
|
|
42
45
|
}
|
|
43
46
|
|
|
44
47
|
const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { return p; } };
|
|
@@ -48,7 +51,7 @@ const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { retu
|
|
|
48
51
|
// a repo INHERITS their guardrails, and a local metric of the same name overrides the inherited one.
|
|
49
52
|
function resolveConfig(p, repo, label, seen) {
|
|
50
53
|
const abs = realpathSync(p);
|
|
51
|
-
if (seen.has(abs))
|
|
54
|
+
if (seen.has(abs)) bad(`config inheritance cycle at ${relOf(repo, p)}`);
|
|
52
55
|
seen.add(abs);
|
|
53
56
|
try {
|
|
54
57
|
const cfg = readConfigFile(p);
|
|
@@ -57,7 +60,7 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
57
60
|
const scalars = {};
|
|
58
61
|
for (const ref of [].concat(cfg.extends || [])) {
|
|
59
62
|
const bp = isAbsolute(ref) ? ref : join(dirname(p), ref);
|
|
60
|
-
if (!existsSync(bp))
|
|
63
|
+
if (!existsSync(bp)) bad(`extends target not found: ${ref} (from ${relOf(repo, p)})`);
|
|
61
64
|
const base = resolveConfig(bp, repo, relOf(repo, bp), seen);
|
|
62
65
|
for (const m of base.metrics) byName.set(m.name, m);
|
|
63
66
|
for (const md of (base.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
@@ -76,9 +79,9 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
76
79
|
|
|
77
80
|
function loadConfig(repo) {
|
|
78
81
|
const p = join(repo, 'promptwheel.config.json');
|
|
79
|
-
if (!existsSync(p))
|
|
82
|
+
if (!existsSync(p)) bad('no promptwheel.config.json, run: promptwheel init (writes one for your stack)');
|
|
80
83
|
const cfg = resolveConfig(p, repo, 'local', new Set());
|
|
81
|
-
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0)
|
|
84
|
+
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0) bad('config.metrics must be a non-empty array');
|
|
82
85
|
return cfg;
|
|
83
86
|
}
|
|
84
87
|
|
|
@@ -285,13 +288,18 @@ function evaluatorMatcher(cfg) {
|
|
|
285
288
|
// so a parent scope's rule holds for every repo beneath it.
|
|
286
289
|
// The mandates in force are the UNION of the head config's and the base ref's (`baseMandates`,
|
|
287
290
|
// resolved by gateDefAtRef): deleting a mandate in the same change that violates it does not
|
|
288
|
-
// clear it. A mandate removed at head but violated in the diff is reported with `removed: true
|
|
291
|
+
// clear it. A mandate removed at head but violated in the diff is reported with `removed: true`; one
|
|
292
|
+
// whose paths were edited at head (same name, different paths) is tested in BOTH versions and reported
|
|
293
|
+
// with `edited: true`, so narrowing a rule in the change that breaks it does not clear it either.
|
|
289
294
|
// The honest way to retire one is a separate change that only edits the config.
|
|
290
295
|
function checkMandates(repo, base, head, cfg, baseMandates) {
|
|
291
296
|
const headList = Array.isArray(cfg.mandates) ? cfg.mandates : [];
|
|
292
297
|
const headNames = new Set(headList.map((m) => m && m.name).filter(Boolean));
|
|
293
|
-
const
|
|
294
|
-
const
|
|
298
|
+
const key = (m) => JSON.stringify([String(m.name), [...(Array.isArray(m.paths) ? m.paths : [])].filter((x) => typeof x === 'string').sort()]);
|
|
299
|
+
const headKeys = new Set(headList.filter((m) => m && m.name).map(key));
|
|
300
|
+
const kept = (Array.isArray(baseMandates) ? baseMandates : []).filter((m) => m && m.name && !headKeys.has(key(m)))
|
|
301
|
+
.map((m) => (headNames.has(m.name) ? { ...m, __edited: true } : { ...m, __removed: true }));
|
|
302
|
+
const mandates = [...headList, ...kept];
|
|
295
303
|
if (!mandates.length) return [];
|
|
296
304
|
let changed = [];
|
|
297
305
|
try { changed = git(['diff', '--name-only', base, head], repo).split('\n').filter(Boolean); } catch { return []; }
|
|
@@ -301,9 +309,18 @@ function checkMandates(repo, base, head, cfg, baseMandates) {
|
|
|
301
309
|
if (!pats.length) continue;
|
|
302
310
|
const res = pats.map(globToRe);
|
|
303
311
|
const hit = changed.filter((f) => res.some((re) => re.test(f)));
|
|
304
|
-
if (hit.length) out.push({ name: String(m.name || 'mandate'), reason: m.reason ? String(m.reason).slice(0, 300) : '', files: hit.slice(0, 20), ...(m.__removed ? { removed: true } : {}) });
|
|
312
|
+
if (hit.length) out.push({ name: String(m.name || 'mandate'), reason: m.reason ? String(m.reason).slice(0, 300) : '', files: hit.slice(0, 20), ...(m.__removed ? { removed: true } : {}), ...(m.__edited ? { edited: true } : {}) });
|
|
305
313
|
}
|
|
306
|
-
|
|
314
|
+
// one entry per name: the head version and the edited base version of a rule merge (files unioned)
|
|
315
|
+
const merged = [];
|
|
316
|
+
for (const v of out) {
|
|
317
|
+
const prev = merged.find((x) => x.name === v.name);
|
|
318
|
+
if (!prev) { merged.push(v); continue; }
|
|
319
|
+
prev.files = [...new Set([...prev.files, ...v.files])].slice(0, 20);
|
|
320
|
+
if (v.edited) prev.edited = true;
|
|
321
|
+
if (!prev.reason && v.reason) prev.reason = v.reason;
|
|
322
|
+
}
|
|
323
|
+
return merged;
|
|
307
324
|
}
|
|
308
325
|
|
|
309
326
|
// split the agent's changed files into production-source vs test/config/grader/golden.
|
|
@@ -626,7 +643,7 @@ function gate(repo, opts) {
|
|
|
626
643
|
// never touches the real index or working tree. (A loop agent's most common action
|
|
627
644
|
// is to ADD a file; `git stash create` omits untracked, which silently reverted them.)
|
|
628
645
|
try { git(['rev-parse', '--verify', 'HEAD'], repo); }
|
|
629
|
-
catch {
|
|
646
|
+
catch { bad('no commits yet, make an initial commit before gating --working changes'); }
|
|
630
647
|
base = 'HEAD';
|
|
631
648
|
head = workingSnapshot(repo);
|
|
632
649
|
} else {
|
|
@@ -884,6 +901,13 @@ function hasEslint(repo) {
|
|
|
884
901
|
} catch { return false; }
|
|
885
902
|
}
|
|
886
903
|
|
|
904
|
+
// does git ignore this repo-relative path? `check-ignore -q` exits 0 when it does; 1 when it does
|
|
905
|
+
// not; anything else (not a git repo) counts as not ignored, so init never fails on the check.
|
|
906
|
+
function gitIgnores(repo, relPath) {
|
|
907
|
+
try { execFileSync('git', ['check-ignore', '-q', relPath], { cwd: repo, stdio: 'ignore' }); return true; }
|
|
908
|
+
catch { return false; }
|
|
909
|
+
}
|
|
910
|
+
|
|
887
911
|
function init(argv) {
|
|
888
912
|
if (argv.includes('--list')) {
|
|
889
913
|
console.log('PromptWheel presets (promptwheel init --preset <name>):\n');
|
|
@@ -911,8 +935,14 @@ function init(argv) {
|
|
|
911
935
|
const src = fileURLToPath(new URL(`../${pk[2]}/`, import.meta.url));
|
|
912
936
|
const dst = join(repo, '.promptwheel', pk[2]);
|
|
913
937
|
mkdirSync(dst, { recursive: true });
|
|
914
|
-
|
|
938
|
+
const files = readdirSync(src);
|
|
939
|
+
for (const f of files) copyFileSync(join(src, f), join(dst, f));
|
|
915
940
|
console.log(` ✓ installed ${pk[2]} → .promptwheel/${pk[2]} (commit it with the config)`);
|
|
941
|
+
// the gate measures in clean worktrees, so a pack that git ignores (common: the outcome
|
|
942
|
+
// record lives in .promptwheel/ too) is absent at both refs and the metric reads
|
|
943
|
+
// unmeasurable, failing the guard with no hint why. Say so now, while it is one edit.
|
|
944
|
+
if (files.length && gitIgnores(repo, `.promptwheel/${pk[2]}/${files[0]}`))
|
|
945
|
+
console.log(` ⚠ .promptwheel/${pk[2]} is gitignored: the gate measures in clean worktrees, so an ignored pack is absent at both refs and ${m.name} reads unmeasurable (the guard fails). Replace ".promptwheel/" in .gitignore with ".promptwheel/*" and "!.promptwheel/packs/", then commit the pack`);
|
|
916
946
|
return { ...m, cmd: m.cmd.replaceAll(pk[2] + '/', `.promptwheel/${pk[2]}/`) };
|
|
917
947
|
});
|
|
918
948
|
note = presetName;
|
|
@@ -993,7 +1023,7 @@ const short = (repo, ref) => { try { return git(['rev-parse', '--short', ref], r
|
|
|
993
1023
|
function printHuman(r) {
|
|
994
1024
|
const arrowFor = (m) => (m.delta == null ? '?' : m.delta > 0 ? '▲' : m.delta < 0 ? '▼' : '=');
|
|
995
1025
|
console.log(`\nPromptWheel ${r.base} → ${r.head}${r.repeat > 1 ? ` (×${r.repeat})` : ''}\n`);
|
|
996
|
-
if (r.mandates) for (const md of r.mandates) console.log(` ⛔ MANDATE "${md.name}" violated${md.reason ? ': ' + md.reason : ''} (touched: ${md.files.join(', ')})${md.removed ? ' [deleted in this same change; a mandate stays in force for the change that removes it]' : ''}`);
|
|
1026
|
+
if (r.mandates) for (const md of r.mandates) console.log(` ⛔ MANDATE "${md.name}" violated${md.reason ? ': ' + md.reason : ''} (touched: ${md.files.join(', ')})${md.removed ? ' [deleted in this same change; a mandate stays in force for the change that removes it]' : md.edited ? ' [edited in this same change; the version before the change stays in force for the change that edits it]' : ''}`);
|
|
997
1027
|
for (const m of r.metrics) {
|
|
998
1028
|
const tag = m.guard ? (m.ok ? 'guard✓' : 'GUARD✗') : 'info';
|
|
999
1029
|
const d = m.delta == null ? 'n/a' : (m.delta > 0 ? `+${m.delta}` : `${m.delta}`);
|
|
@@ -1025,7 +1055,7 @@ function renderMarkdown(r) {
|
|
|
1025
1055
|
'',
|
|
1026
1056
|
`\`${r.base} → ${r.head}\` · ${r.mode} mode${r.repeat > 1 ? ` · ×${r.repeat}` : ''}`,
|
|
1027
1057
|
'',
|
|
1028
|
-
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **Mandate violated:** \`${md.name}\`${md.reason ? ': ' + md.reason : ''} (touched \`${md.files.join('`, `')}\`)${md.removed ? ' (deleted in this same change; it stays in force for the change that removes it)' : ''}`).join('\n'), ''] : []),
|
|
1058
|
+
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **Mandate violated:** \`${md.name}\`${md.reason ? ': ' + md.reason : ''} (touched \`${md.files.join('`, `')}\`)${md.removed ? ' (deleted in this same change; it stays in force for the change that removes it)' : md.edited ? ' (edited in this same change; the version before the change stays in force for the change that edits it)' : ''}`).join('\n'), ''] : []),
|
|
1029
1059
|
'| metric | before | after | Δ | status | confidence |',
|
|
1030
1060
|
'|---|--:|--:|--:|---|---|',
|
|
1031
1061
|
rows,
|
|
@@ -1260,7 +1290,7 @@ function conformance(argv) {
|
|
|
1260
1290
|
process.exit(1);
|
|
1261
1291
|
}
|
|
1262
1292
|
|
|
1263
|
-
function
|
|
1293
|
+
function dispatch() {
|
|
1264
1294
|
const [cmd, ...rest] = process.argv.slice(2);
|
|
1265
1295
|
if (cmd === '--version' || cmd === '-v') { console.log(TOOL_VERSION ?? 'unknown'); process.exit(0); }
|
|
1266
1296
|
if (cmd === 'run') run(rest);
|
|
@@ -1302,4 +1332,8 @@ function main() {
|
|
|
1302
1332
|
// run the CLI only when invoked directly (resolves symlinks so the npm bin still works)
|
|
1303
1333
|
let isMain = false;
|
|
1304
1334
|
try { isMain = !!process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); } catch { /* not main */ }
|
|
1335
|
+
function main() {
|
|
1336
|
+
try { dispatch(); }
|
|
1337
|
+
catch (e) { if (e && e.name === 'ConfigError') die(e.message); throw e; }
|
|
1338
|
+
}
|
|
1305
1339
|
if (isMain) main();
|
package/package.json
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "promptwheel",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.7",
|
|
4
4
|
"description": "A deterministic, zero-LLM check for your coding loop. It proves every change earned its win by re-running the result from the agent's source edits alone, so a green that came from editing the test reads GAMED. Runs in CI or a Claude Code hook, or import { check } in process.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
7
7
|
"promptwheel": "bin/promptwheel.mjs"
|
|
8
8
|
},
|
|
9
|
+
"types": "./bin/promptwheel.d.ts",
|
|
9
10
|
"exports": {
|
|
10
|
-
".":
|
|
11
|
+
".": {
|
|
12
|
+
"types": "./bin/promptwheel.d.ts",
|
|
13
|
+
"default": "./bin/promptwheel.mjs"
|
|
14
|
+
},
|
|
11
15
|
"./bin/promptwheel.mjs": "./bin/promptwheel.mjs",
|
|
12
16
|
"./package.json": "./package.json"
|
|
13
17
|
},
|
package/packs/security/README.md
CHANGED
|
@@ -9,18 +9,28 @@ lower the count by editing the pattern file or a config, only by fixing the code
|
|
|
9
9
|
|
|
10
10
|
## Use it
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
The pack ships inside the `promptwheel` package, not in your repo, so install it with the
|
|
13
|
+
preset:
|
|
13
14
|
|
|
14
|
-
```
|
|
15
|
-
|
|
15
|
+
```bash
|
|
16
|
+
promptwheel init --preset security # or --preset security-antihack, which adds the tripwires
|
|
16
17
|
```
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
`init` copies the pack into `.promptwheel/packs/security/` and writes a `security_findings`
|
|
20
|
+
metric that runs it from there. Commit the pack with the config: the gate measures in clean
|
|
21
|
+
git worktrees, so a pack that is not in git is absent at both refs and the metric reads
|
|
22
|
+
`unmeasurable` (init warns when `.promptwheel/` is gitignored; change that rule to
|
|
23
|
+
`.promptwheel/*` plus `!.promptwheel/packs/`, since git cannot re-include a file under an
|
|
24
|
+
ignored directory). Then gate as usual:
|
|
19
25
|
|
|
20
26
|
```bash
|
|
21
27
|
promptwheel run --working # fails if security_findings increased
|
|
22
28
|
```
|
|
23
29
|
|
|
30
|
+
`pack.json` is the same metric as a config fragment for `extends`
|
|
31
|
+
(`{ "extends": ["packs/security/pack.json"] }`); its command is relative to a repo that
|
|
32
|
+
holds the pack at `packs/security/`, such as a vendored copy or this repository.
|
|
33
|
+
|
|
24
34
|
A finding-count that drops because you fixed the code is a `PASS`; a "fix" that
|
|
25
35
|
only survives with test/config/pattern edits reverted is `GAMED`.
|
|
26
36
|
|