promptwheel 0.7.15 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INTEGRATE.md +8 -16
- package/README.md +10 -17
- package/bin/promptwheel.d.ts +0 -4
- package/bin/promptwheel.mjs +24 -227
- package/package.json +1 -1
- package/packs/security/README.md +0 -62
- package/packs/security/pack.json +0 -12
- package/packs/security/patterns.json +0 -209
- package/packs/security/scan.mjs +0 -57
package/INTEGRATE.md
CHANGED
|
@@ -72,11 +72,7 @@ truncated SHA-256 fingerprint of the gate definition (metrics, guards, `evaluato
|
|
|
72
72
|
and lists any guard the change dropped to hide a regression; a non-empty `gate.weakened`
|
|
73
73
|
forces `gamed`, so an agent cannot pass by disabling the guard it is judged by.
|
|
74
74
|
|
|
75
|
-
Self-check your implementation against this schema (see Section 4 below)
|
|
76
|
-
|
|
77
|
-
```bash
|
|
78
|
-
your-tool --json | promptwheel conformance -
|
|
79
|
-
```
|
|
75
|
+
Self-check your implementation against this schema with `conformsToContract` (see Section 4 below).
|
|
80
76
|
|
|
81
77
|
---
|
|
82
78
|
|
|
@@ -135,7 +131,6 @@ Returns the Section 2 verdict object (identical to `run --json`). Reproducible f
|
|
|
135
131
|
| `repeat` | number | `config.repeat ?? 1` | measurement repeats for the noise band |
|
|
136
132
|
| `detectGaming` | boolean | `true` | run the Section 3 source-only re-proof |
|
|
137
133
|
| `allow` | string | none | a declared override reason (the same as `--allow` / a `Promptwheel-Allow:` trailer): a would-be `gamed`, or an `unprovable` win (its source-only re-run could not be applied), reads `pass` with the reason recorded in `report.allowed` |
|
|
138
|
-
| `record` | boolean | `false` | append the run to `.promptwheel/outcomes.jsonl` |
|
|
139
134
|
|
|
140
135
|
Preconditions throw (they do not exit the host): not inside a git repository; `auto: false`
|
|
141
136
|
with no `promptwheel.config.json` and no `config` option; a `config` with no non-empty
|
|
@@ -173,22 +168,19 @@ of truth instead of hardcoding the strings.
|
|
|
173
168
|
|
|
174
169
|
---
|
|
175
170
|
|
|
176
|
-
## 4. Self-check with `
|
|
171
|
+
## 4. Self-check with `conformsToContract`
|
|
177
172
|
|
|
178
173
|
Prove your own check, loop, or CI step speaks the Contract.
|
|
179
174
|
|
|
180
|
-
```
|
|
181
|
-
|
|
182
|
-
promptwheel conformance report.json
|
|
183
|
-
cat report.json | promptwheel conformance -
|
|
175
|
+
```js
|
|
176
|
+
import { conformsToContract, verdictExitCode } from 'promptwheel';
|
|
184
177
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
promptwheel conformance --cmd "your-tool --json"
|
|
178
|
+
const { ok, errors } = conformsToContract(JSON.parse(reportJson)); // Section 2
|
|
179
|
+
const exitMatches = exitCode === verdictExitCode(report.verdict); // Section 1
|
|
188
180
|
```
|
|
189
181
|
|
|
190
|
-
|
|
191
|
-
implementation earns the right to say "Verified-Outcome Contract compatible."
|
|
182
|
+
`ok` is `true` when the report conforms; otherwise `errors` lists every problem. This is how a
|
|
183
|
+
third-party implementation earns the right to say "Verified-Outcome Contract compatible."
|
|
192
184
|
|
|
193
185
|
---
|
|
194
186
|
|
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|

|
|
8
8
|
|
|
9
|
-
##
|
|
9
|
+
## See it in 60 seconds
|
|
10
10
|
|
|
11
11
|
A hand-built cheat, so you can watch the mechanism fire (illustrative, not evidence):
|
|
12
12
|
|
|
@@ -54,18 +54,11 @@ npx promptwheel run --working # gate UNCOMMITTED changes (incl. newl
|
|
|
54
54
|
npx promptwheel run --repeat 5 --json # measure 5× to establish a noise band, emit JSON
|
|
55
55
|
|
|
56
56
|
npx promptwheel init # optional: write promptwheel.config.json (guarded tests + antihack tripwires, + lint if eslint is set up)
|
|
57
|
-
npx promptwheel init --list # presets:
|
|
58
|
-
|
|
59
|
-
# the loop: run any agent/script, keep the change ONLY if a metric improved, else revert
|
|
60
|
-
npx promptwheel improve --attempt "claude -p 'reduce lint errors'"
|
|
61
|
-
# exit 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed (reverted) · 3 = plateau / inconclusive (reverted) · add --json
|
|
62
|
-
while npx promptwheel improve --attempt "<your agent command>"; do :; done # converges: stops on the first non-zero
|
|
63
|
-
|
|
64
|
-
npx promptwheel insights # which metrics actually respond in this repo (counts from .promptwheel/outcomes.jsonl)
|
|
65
|
-
npx promptwheel guards # the effective guardrails, including ones inherited through extends
|
|
57
|
+
npx promptwheel init --list # presets: antihack · no-ai-tells · supply-chain
|
|
58
|
+
npx promptwheel lock # write promptwheel.lock.json, the shape of this repo; the supply-chain preset holds it
|
|
66
59
|
```
|
|
67
60
|
|
|
68
|
-
Gaming detection is on by default in `run
|
|
61
|
+
Gaming detection is on by default in `run`; `--no-detect-gaming` gives you the bare outcome gate.
|
|
69
62
|
|
|
70
63
|
**Give your agent the gate.** `promptwheel init` writes this into your `AGENTS.md` (and `CLAUDE.md` / `.cursorrules` if present; `--no-agent-block` to skip), so the coding agent runs the gate itself. To paste it yourself:
|
|
71
64
|
|
|
@@ -74,7 +67,7 @@ Gaming detection is on by default in `run` and `improve`; `--no-detect-gaming` g
|
|
|
74
67
|
config, or grader, not the source, so revert and earn it in the source. Exit 1 means a guarded
|
|
75
68
|
metric regressed, revert that too. Never weaken, skip, or delete a test to go green.
|
|
76
69
|
|
|
77
|
-
**Two callers, by design.** Your *agent* consumes the verdict as a per-turn signal (`
|
|
70
|
+
**Two callers, by design.** Your *agent* consumes the verdict as a per-turn signal (`run --working --json`). The *harness* (a Stop-hook, or CI) runs the same check as an audit the agent cannot self-clear, because the check re-proves from the source alone and never asks the code to grade itself. See [docs/ENFORCEMENT.md](docs/ENFORCEMENT.md) for the three places to enforce it.
|
|
78
71
|
|
|
79
72
|
## Config, `promptwheel.config.json`
|
|
80
73
|
|
|
@@ -119,7 +112,7 @@ Teams keep one shared base config and have every repo inherit it via `extends`:
|
|
|
119
112
|
"metrics": [ { "name": "cost_per_run_usd", "guard": false } ] } // loosen one inherited guard, locally
|
|
120
113
|
```
|
|
121
114
|
|
|
122
|
-
`extends` takes a path (or array of paths) to base configs: a repo **inherits** their guardrails, and a local metric of the same name **overrides** the inherited one (tighten, loosen, or disable).
|
|
115
|
+
`extends` takes a path (or array of paths) to base configs: a repo **inherits** their guardrails, and a local metric of the same name **overrides** the inherited one (tighten, loosen, or disable). Read this way, `extends` is the set of shared invariants your agents are held to, enforced as measured guards rather than advisory text. Natural-language conventions belong in `AGENTS.md`; only deterministic, measurable guards belong here.
|
|
123
116
|
|
|
124
117
|
### Mandates (protected paths)
|
|
125
118
|
|
|
@@ -173,7 +166,7 @@ The Action runs straight from its own checkout, no npm install, no build. See [`
|
|
|
173
166
|
/plugin install promptwheel@promptwheel-ai
|
|
174
167
|
```
|
|
175
168
|
|
|
176
|
-
Then `/promptwheel:setup` (write a config) · `/promptwheel:gate` (gate uncommitted changes)
|
|
169
|
+
Then `/promptwheel:setup` (write a config) · `/promptwheel:gate` (gate uncommitted changes). See [`plugins/promptwheel/`](plugins/promptwheel/). A Stop-hook that refuses to let an agent end a turn on a `GAMED` win ships at [`examples/detect-gaming-stop.sh`](examples/detect-gaming-stop.sh).
|
|
177
170
|
|
|
178
171
|
## Trust model
|
|
179
172
|
|
|
@@ -194,7 +187,7 @@ Then `/promptwheel:setup` (write a config) · `/promptwheel:gate` (gate uncommit
|
|
|
194
187
|
|
|
195
188
|
A check runs your metric commands in two throwaway worktrees (before and after) plus a source-only re-run when a metric improved, so one check takes roughly 2 to 3× your metric command's runtime plus about 40ms of worktree overhead. Point the per-turn gate at a *fast* metric (a lint count, a quick test subset, a build-size check) and leave the slow full suite for CI: a 50ms check is invisible in a loop, a two-minute one is not. A deterministic metric (a test exit, a count) is accurate at the default `repeat: 1`; only a noisy metric (latency, timing) needs `repeat: 3` or more, which multiplies the cost.
|
|
196
189
|
|
|
197
|
-
**Footprint.** It never touches your working tree: every measurement runs in a throwaway git worktree in your system temp dir (one at a time; configured `linkDirs` are symlinked, not copied), removed when the run finishes.
|
|
190
|
+
**Footprint.** It never touches your working tree: every measurement runs in a throwaway git worktree in your system temp dir (one at a time; configured `linkDirs` are symlinked, not copied), removed when the run finishes. A run writes nothing to your repo; only `init` and `lock` write files, the ones you commit. A hard-killed run cannot leave clutter behind: the next run self-heals any orphaned worktree.
|
|
198
191
|
|
|
199
192
|
## Portability
|
|
200
193
|
|
|
@@ -202,9 +195,9 @@ The core gamed-win check is git only and runs anywhere git does: it classifies t
|
|
|
202
195
|
|
|
203
196
|
## Docs
|
|
204
197
|
|
|
205
|
-
- [INTEGRATE.md](INTEGRATE.md), the integration surface: the exit-code and verdict-JSON contract, shelling `--json`, `import { check }`, and `
|
|
198
|
+
- [INTEGRATE.md](INTEGRATE.md), the integration surface: the exit-code and verdict-JSON contract, shelling `--json`, `import { check }`, and `conformsToContract` for self-checking a third-party implementation.
|
|
206
199
|
- [docs/DETECTION-LAYERS.md](docs/DETECTION-LAYERS.md), how the source-only re-run fits as the deterministic layer alongside held-out tests, LLM judges, and human review: the coverage matrix and the honest in/out-of-scope boundary.
|
|
207
|
-
- [docs/ENFORCEMENT.md](docs/ENFORCEMENT.md), making "the agent can't skip it" real: the
|
|
200
|
+
- [docs/ENFORCEMENT.md](docs/ENFORCEMENT.md), making "the agent can't skip it" real: the two places to enforce (CI + branch protection, a Claude Code Stop-hook), the exact wiring, and the protect-the-gate's-own-config caveat.
|
|
208
201
|
- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md), how the engine works: schemas, extract modes, the trust/noise model.
|
|
209
202
|
- [CHANGELOG.md](CHANGELOG.md), what shipped in each version.
|
|
210
203
|
- [CLAUDE.md](CLAUDE.md), the working notes for anyone (human or agent) changing this repo.
|
package/bin/promptwheel.d.ts
CHANGED
|
@@ -85,8 +85,6 @@ export interface Config {
|
|
|
85
85
|
env?: Record<string, string>;
|
|
86
86
|
/** A per-ref build or install, run in the worktree before measuring (a failed setup leaves the metric inert, never a false green). */
|
|
87
87
|
setup?: string;
|
|
88
|
-
/** `false` never appends to `.promptwheel/outcomes.jsonl`. */
|
|
89
|
-
record?: boolean;
|
|
90
88
|
/** Fraction of a gain that must survive the source-only re-run; below it the win is `gamed`. Default 0.5. */
|
|
91
89
|
gamingThreshold?: number;
|
|
92
90
|
/** Extra files that define the gate (a threshold, a budget); a change to one alongside a win is gamed by declaration. Same glob rules as `Mandate.paths`. */
|
|
@@ -120,8 +118,6 @@ export interface CheckOptions {
|
|
|
120
118
|
detectGaming?: boolean;
|
|
121
119
|
/** A declared override reason (the same as `--allow` or a `Promptwheel-Allow:` commit trailer): a would-be `gamed`, or an `unprovable` win (its source-only re-run could not be applied), reads `pass`, recorded in `report.allowed`. */
|
|
122
120
|
allow?: string;
|
|
123
|
-
/** Append the run to `.promptwheel/outcomes.jsonl`. Default `false`. */
|
|
124
|
-
record?: boolean;
|
|
125
121
|
}
|
|
126
122
|
|
|
127
123
|
// ---------------------------------------------------------------------------
|
package/bin/promptwheel.mjs
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
import { execSync, execFileSync } from 'node:child_process';
|
|
16
16
|
import { createHash } from 'node:crypto';
|
|
17
|
-
import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync,
|
|
17
|
+
import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, realpathSync, readdirSync, statSync, copyFileSync } from 'node:fs';
|
|
18
18
|
import { tmpdir } from 'node:os';
|
|
19
19
|
import { join, dirname, isAbsolute, relative } from 'node:path';
|
|
20
20
|
import { fileURLToPath } from 'node:url';
|
|
@@ -68,7 +68,7 @@ const relOf = (repo, p) => { try { return relative(repo, p) || p; } catch { retu
|
|
|
68
68
|
// resolve a config + its `extends` chain into a flat, provenance-tagged metric list.
|
|
69
69
|
// `extends` is a path (or array of paths) to base configs, relative to the config file:
|
|
70
70
|
// a repo INHERITS their guardrails, and a local metric of the same name overrides the inherited one.
|
|
71
|
-
function resolveConfig(p, repo,
|
|
71
|
+
function resolveConfig(p, repo, seen) {
|
|
72
72
|
const abs = realpathSync(p);
|
|
73
73
|
if (seen.has(abs)) bad(`config inheritance cycle at ${relOf(repo, p)}`);
|
|
74
74
|
seen.add(abs);
|
|
@@ -80,16 +80,16 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
80
80
|
for (const ref of [].concat(cfg.extends || [])) {
|
|
81
81
|
const bp = isAbsolute(ref) ? ref : join(dirname(p), ref);
|
|
82
82
|
if (!existsSync(bp)) bad(`extends target not found: ${ref} (from ${relOf(repo, p)})`);
|
|
83
|
-
const base = resolveConfig(bp, repo,
|
|
83
|
+
const base = resolveConfig(bp, repo, seen);
|
|
84
84
|
for (const m of base.metrics) byName.set(m.name, m);
|
|
85
85
|
for (const md of (base.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
86
|
-
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', '
|
|
86
|
+
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (base[k] !== undefined) scalars[k] = base[k];
|
|
87
87
|
}
|
|
88
88
|
for (const m of (cfg.metrics || [])) {
|
|
89
|
-
byName.set(m.name,
|
|
89
|
+
byName.set(m.name, m);
|
|
90
90
|
}
|
|
91
|
-
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name,
|
|
92
|
-
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', '
|
|
91
|
+
for (const md of (cfg.mandates || [])) if (md && md.name) mandatesByName.set(md.name, md);
|
|
92
|
+
for (const k of ['repeat', 'linkNodeModules', 'linkDirs', 'env', 'setup', 'gamingThreshold', 'evaluatorPaths', 'allowTrailer']) if (cfg[k] !== undefined) scalars[k] = cfg[k];
|
|
93
93
|
return { ...scalars, metrics: [...byName.values()], mandates: [...mandatesByName.values()] };
|
|
94
94
|
} finally {
|
|
95
95
|
seen.delete(abs); // only ACTIVE ancestors are a cycle, a diamond (a base reached two ways) is fine
|
|
@@ -99,7 +99,7 @@ function resolveConfig(p, repo, label, seen) {
|
|
|
99
99
|
function loadConfig(repo) {
|
|
100
100
|
const p = join(repo, 'promptwheel.config.json');
|
|
101
101
|
if (!existsSync(p)) bad('no promptwheel.config.json, run: promptwheel init (writes one for your stack)');
|
|
102
|
-
const cfg = resolveConfig(p, repo,
|
|
102
|
+
const cfg = resolveConfig(p, repo, new Set());
|
|
103
103
|
if (!Array.isArray(cfg.metrics) || cfg.metrics.length === 0) bad('config.metrics must be a non-empty array');
|
|
104
104
|
return cfg;
|
|
105
105
|
}
|
|
@@ -279,7 +279,7 @@ const NON_SOURCE = [
|
|
|
279
279
|
// promptwheel's OWN evaluator surface: the config that defines the gate
|
|
280
280
|
// and any pack copied into the repo by `init --preset`. Editing these is
|
|
281
281
|
// moving the goalposts, never a production improvement, an agent that
|
|
282
|
-
// "improved"
|
|
282
|
+
// "improved" a scanner count by deleting its rules must show 0%
|
|
283
283
|
// retained in the source-only re-run, not 100%.
|
|
284
284
|
/(^|\/)promptwheel\.config\.json$/i,
|
|
285
285
|
/(^|\/)\.promptwheel\//i,
|
|
@@ -457,7 +457,7 @@ function judgeGaming(m, srcResult, threshold = 0.5) {
|
|
|
457
457
|
// main
|
|
458
458
|
// ---------------------------------------------------------------------------
|
|
459
459
|
// snapshot the working tree (tracked + untracked) into a dangling commit WITHOUT
|
|
460
|
-
// touching the real index or files, so a gate
|
|
460
|
+
// touching the real index or files, so a gate sees a file an agent just added.
|
|
461
461
|
// Returns 'HEAD' when nothing changed.
|
|
462
462
|
function workingSnapshot(repo) {
|
|
463
463
|
const idxDir = mkdtempSync(join(tmpdir(), tmpPrefix(repo, 'pw-idx')));
|
|
@@ -731,7 +731,7 @@ function detectGamingPass(repo, base, head, cfg, metrics, repeat) {
|
|
|
731
731
|
// (or a weakened gate) is GAMED; INCONCLUSIVE when nothing could be certified. A guard that never
|
|
732
732
|
// runs (inert: a pass/fail metric stuck at 0 across both refs) verifies nothing, so a bare no-op
|
|
733
733
|
// run is not laundered into a green PASS, BUT an inert guard must not bury a genuine, measured win
|
|
734
|
-
// elsewhere (
|
|
734
|
+
// elsewhere (a loop reading the verdict would throw away a good change). And a
|
|
735
735
|
// win whose source-only re-run could not be applied (`unprovable`) is never a PASS either: a green
|
|
736
736
|
// the engine could not re-prove is exactly the green it exists to doubt.
|
|
737
737
|
export function deriveVerdict(metrics, gateInfo = null, mandated = false) {
|
|
@@ -854,7 +854,6 @@ function gate(repo, opts) {
|
|
|
854
854
|
configHash: gateFingerprint(cfg.metrics, cfg),
|
|
855
855
|
verdictHash: verdictFingerprint(report),
|
|
856
856
|
};
|
|
857
|
-
if (!opts.noRecord && cfg.record !== false) recordOutcome(repo, report);
|
|
858
857
|
return report;
|
|
859
858
|
}
|
|
860
859
|
|
|
@@ -866,115 +865,18 @@ function run(argv) {
|
|
|
866
865
|
config = autoConfig(repo);
|
|
867
866
|
console.error(`no promptwheel.config.json, auto-gating the detected test command: ${config.metrics[0].cmd} (promptwheel init to customize; --no-auto to require a config)`);
|
|
868
867
|
}
|
|
869
|
-
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat,
|
|
868
|
+
const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, detectGaming: args.detectGaming, allow: args.allow, config });
|
|
870
869
|
if (args.json) console.log(JSON.stringify(report, null, 2));
|
|
871
870
|
else if (args.markdown) console.log(renderMarkdown(report));
|
|
872
871
|
else printHuman(report);
|
|
873
872
|
process.exit(verdictExitCode(report.verdict));
|
|
874
873
|
}
|
|
875
874
|
|
|
876
|
-
// persisted record: append every gated run to a per-repo outcome record (best-effort, never fails the gate)
|
|
877
|
-
function recordOutcome(repo, report) {
|
|
878
|
-
try {
|
|
879
|
-
const dir = join(repo, '.promptwheel');
|
|
880
|
-
mkdirSync(dir, { recursive: true });
|
|
881
|
-
appendFileSync(join(dir, 'outcomes.jsonl'), JSON.stringify({ ts: new Date().toISOString(), ...report }) + '\n');
|
|
882
|
-
} catch { /* recording must never break the gate */ }
|
|
883
|
-
}
|
|
884
|
-
|
|
885
|
-
// keep-if-improved: run any agent/script, gate the result, keep the change ONLY if a metric improved
|
|
886
|
-
function improve(argv) {
|
|
887
|
-
const args = parseArgs(argv);
|
|
888
|
-
if (!args.attempt) die('improve requires --attempt "<command that changes the repo>"');
|
|
889
|
-
const repo = repoRoot(process.cwd());
|
|
890
|
-
const dirty = git(['status', '--porcelain'], repo).split('\n').filter((l) => l.trim() && !l.includes('.promptwheel'));
|
|
891
|
-
if (dirty.length) die('working tree not clean, commit or stash first (improve needs a clean base to revert to)');
|
|
892
|
-
|
|
893
|
-
console.error(`▶ attempt: ${args.attempt}`);
|
|
894
|
-
try { execSync(args.attempt, { cwd: repo, stdio: 'inherit' }); }
|
|
895
|
-
catch (e) { console.error(` (attempt exited ${e.status ?? 1}, gating whatever it changed)`); }
|
|
896
|
-
|
|
897
|
-
const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
|
|
898
|
-
const noChange = report.metrics.every((m) => m.delta === 0 || m.delta == null);
|
|
899
|
-
const improvedNames = report.metrics.filter((m) => m.status === 'improved').map((m) => m.name);
|
|
900
|
-
|
|
901
|
-
// result + exit code express loop progress so `while improve; do :; done` converges, with the
|
|
902
|
-
// same codes as `run`: 0 = kept a real win · 1 = guarded regression (reverted) · 2 = gamed
|
|
903
|
-
// (reverted) · 3 = plateau/no-op OR inconclusive (reverted)
|
|
904
|
-
let result, exit, note;
|
|
905
|
-
if (report.verdict === 'fail') { result = 'regression'; exit = 1; revert(repo); note = '✗ guarded regression, reverted'; }
|
|
906
|
-
else if (report.verdict === 'gamed') { result = 'gamed'; exit = 2; revert(repo); note = '🚩 gamed, a metric "improved" by editing tests/config, not source, reverted'; }
|
|
907
|
-
else if (report.verdict === 'inconclusive') { result = 'inconclusive'; exit = 3; revert(repo); note = '≈ inconclusive, a guard measured nothing (inert); reverted'; }
|
|
908
|
-
else if (noChange || improvedNames.length === 0) {
|
|
909
|
-
result = 'plateau'; exit = 3; revert(repo);
|
|
910
|
-
note = noChange ? '= no metric moved, reverted' : '= nothing improved beyond noise, reverted';
|
|
911
|
-
} else {
|
|
912
|
-
git(['add', '-A'], repo);
|
|
913
|
-
execFileSync('git', ['commit', '-qm', `promptwheel: ${args.attempt} [improved ${improvedNames.join(', ')}]`], { cwd: repo });
|
|
914
|
-
result = 'kept'; exit = 0;
|
|
915
|
-
note = `✓ kept, committed ${git(['rev-parse', '--short', 'HEAD'], repo)} (improved ${improvedNames.join(', ')})`;
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
// stdout carries the value (JSON report or the human table); the decision line goes to
|
|
919
|
-
// stderr so a loop driver can consume clean stdout.
|
|
920
|
-
if (args.json) console.log(JSON.stringify({ result, ...report }, null, 2));
|
|
921
|
-
else printHuman(report);
|
|
922
|
-
console.error(` ${note}`);
|
|
923
|
-
process.exit(exit);
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
// discard the attempt's changes; keep the .promptwheel outcome record
|
|
927
|
-
function revert(repo) {
|
|
928
|
-
git(['reset', '--hard', 'HEAD'], repo);
|
|
929
|
-
try { git(['clean', '-fd', '-e', '.promptwheel'], repo); } catch { /* best effort */ }
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
// insights: honest aggregation of the per-repo outcome record (.promptwheel/outcomes.jsonl)
|
|
933
|
-
// into per-metric counts. No model, no decay, no claims: just which metrics actually respond.
|
|
934
|
-
// The reader is tolerant of older records that carry extra fields (cohort, label, subsystems).
|
|
935
|
-
function insights(argv) {
|
|
936
|
-
const args = parseArgs(argv);
|
|
937
|
-
const repo = repoRoot(process.cwd());
|
|
938
|
-
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
939
|
-
if (!existsSync(f)) die('no outcome record yet, run the gate a few times first (.promptwheel/outcomes.jsonl)');
|
|
940
|
-
const runs = readFileSync(f, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
941
|
-
const agg = {};
|
|
942
|
-
for (const r of runs) for (const m of (r.metrics || [])) {
|
|
943
|
-
const a = agg[m.name] ??= { runs: 0, improved: 0, regressed: 0, inconclusive: 0, unchanged: 0, net: 0, last: null };
|
|
944
|
-
a.runs++; if (a[m.status] != null) a[m.status]++;
|
|
945
|
-
if (typeof m.delta === 'number') a.net = +(a.net + m.delta).toFixed(6);
|
|
946
|
-
a.last = m.after;
|
|
947
|
-
}
|
|
948
|
-
// "lever score" = how reliably acting on this metric yields a real improvement
|
|
949
|
-
const rows = Object.entries(agg).map(([name, a]) => ({ name, ...a, lever: a.runs ? a.improved / a.runs : 0 }))
|
|
950
|
-
.sort((x, y) => y.lever - x.lever);
|
|
951
|
-
if (args.json) { console.log(JSON.stringify({ runs: runs.length, metrics: rows }, null, 2)); return; }
|
|
952
|
-
console.log(`\nPromptWheel insights, ${runs.length} gated runs recorded\n`);
|
|
953
|
-
console.log(` ${'metric'.padEnd(18)} ${'runs'.padStart(5)} ${'impr'.padStart(5)} ${'regr'.padStart(5)} ${'inconc'.padStart(6)} ${'net Δ'.padStart(9)} lever`);
|
|
954
|
-
for (const r of rows) {
|
|
955
|
-
console.log(` ${r.name.padEnd(18)} ${String(r.runs).padStart(5)} ${String(r.improved).padStart(5)} ${String(r.regressed).padStart(5)} ${String(r.inconclusive).padStart(6)} ${String(r.net).padStart(9)} ${(r.lever * 100).toFixed(0)}%`);
|
|
956
|
-
}
|
|
957
|
-
console.log('\n lever = improved/runs, how reliably this metric actually responds. The');
|
|
958
|
-
console.log(' highest-lever metrics are where an agent loop should spend its attempts.\n');
|
|
959
|
-
}
|
|
960
|
-
|
|
961
875
|
// ---------------------------------------------------------------------------
|
|
962
876
|
// init, write a starter config so a newcomer isn't staring at a blank page
|
|
963
877
|
// ---------------------------------------------------------------------------
|
|
964
878
|
const LINT_CMD = 'npx eslint . -f unix 2>/dev/null | grep -c " error " || true';
|
|
965
879
|
const PRESETS = {
|
|
966
|
-
'tests-pass': { desc: 'gate: your test suite still passes (command auto-detected)', metrics: null },
|
|
967
|
-
'lint': { desc: 'track: lint error count does not climb',
|
|
968
|
-
metrics: [{ name: 'lint_errors', cmd: LINT_CMD, extract: 'number', direction: 'down', guard: false }] },
|
|
969
|
-
'bundle-size': { desc: 'track: build output size in kB',
|
|
970
|
-
metrics: [{ name: 'bundle_kb', cmd: 'du -sk dist 2>/dev/null | cut -f1 || echo 0', extract: 'number', direction: 'down', guard: false }] },
|
|
971
|
-
'security': { desc: 'gate: security-invariant violations must not climb (CORS, secrets, cookies, payment; see packs/security)',
|
|
972
|
-
metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }] },
|
|
973
|
-
'llm-eval': { desc: 'gate: AI-feature eval pass-rate + est $/run (see examples/reliability-sprint)',
|
|
974
|
-
metrics: [
|
|
975
|
-
{ name: 'eval_pass_rate', cmd: 'node eval.mjs', extract: 'number', direction: 'up', guard: true },
|
|
976
|
-
{ name: 'cost_per_run_usd', cmd: 'node estimate-cost.mjs', extract: 'number', direction: 'down', guard: true },
|
|
977
|
-
] },
|
|
978
880
|
'antihack': { desc: 'catch reward-hacking: a target + tripwires; pairs with `run --detect-gaming` (source-only re-run)',
|
|
979
881
|
metrics: [
|
|
980
882
|
{ name: 'tests_pass', cmd: '__TESTCMD__', extract: 'exit', direction: 'pass', guard: true },
|
|
@@ -1017,21 +919,6 @@ const PRESETS = {
|
|
|
1017
919
|
] },
|
|
1018
920
|
};
|
|
1019
921
|
|
|
1020
|
-
// security-antihack = the security-invariant findings gate + the SAME antihack tripwires
|
|
1021
|
-
// (so an agent that suppresses a security rule, # noqa / # nosec / eslint-disable, or
|
|
1022
|
-
// skips a security test is caught), PLUS a starter `evaluatorPaths` that protects common
|
|
1023
|
-
// security-gate config files (a ruleset or a budget the agent could otherwise loosen to
|
|
1024
|
-
// pass). The scanner pack installs into `.promptwheel/` (already evaluator-protected). It
|
|
1025
|
-
// is the "guard the guards" preset: pairs with --detect-gaming so a win that only moved by
|
|
1026
|
-
// loosening the security gate, not by fixing the code, is GAMED. Blind spot (by design,
|
|
1027
|
-
// not this tool's job): a source-resident backdoor that behaves safely only under test;
|
|
1028
|
-
// that needs held-out/behavioral analysis, not a source-only re-run.
|
|
1029
|
-
PRESETS['security-antihack'] = {
|
|
1030
|
-
desc: 'gate security-invariant findings + antihack tripwires + evaluatorPaths so an agent cannot loosen the security gate to pass it (pairs with --detect-gaming)',
|
|
1031
|
-
evaluatorPaths: ['.semgrep.yml', '.semgrep.yaml', '.bandit', 'bandit.yaml', '.ruff.toml', 'ruff.toml', '.snyk', 'security-budget.json'],
|
|
1032
|
-
metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }, ...PRESETS.antihack.metrics.slice(1)],
|
|
1033
|
-
};
|
|
1034
|
-
|
|
1035
922
|
function detectTestCmd(repo) {
|
|
1036
923
|
const has = (f) => existsSync(join(repo, f));
|
|
1037
924
|
if (has('go.mod')) return 'go test ./...';
|
|
@@ -1104,10 +991,9 @@ function init(argv) {
|
|
|
1104
991
|
if (presetName) {
|
|
1105
992
|
const p = PRESETS[presetName];
|
|
1106
993
|
if (!p) die(`unknown preset "${presetName}", try: promptwheel init --list`);
|
|
1107
|
-
evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (
|
|
994
|
+
evaluatorPaths = p.evaluatorPaths; // a preset may ship a starter evaluator allowlist (supply-chain: the contract)
|
|
1108
995
|
mandates = p.mandates; allowTrailer = p.allowTrailer; // a preset may ship hard rules (supply-chain: workflows, registries, publish scripts, hooks)
|
|
1109
|
-
metrics = p.metrics
|
|
1110
|
-
metrics = metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
|
|
996
|
+
metrics = p.metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
|
|
1111
997
|
// pack-backed presets: the referenced scripts live in THIS package, not
|
|
1112
998
|
// the user's repo, copy them in so the generated config is
|
|
1113
999
|
// self-contained and committable (a bare reference silently 404'd)
|
|
@@ -1123,8 +1009,7 @@ function init(argv) {
|
|
|
1123
1009
|
const files = readdirSync(src);
|
|
1124
1010
|
for (const f of files) copyFileSync(join(src, f), join(dst, f));
|
|
1125
1011
|
console.log(` ✓ installed ${pk[2]} → .promptwheel/${pk[2]} (commit it with the config)`);
|
|
1126
|
-
// the gate measures in clean worktrees, so a pack that git ignores
|
|
1127
|
-
// record lives in .promptwheel/ too) is absent at both refs and the metric reads
|
|
1012
|
+
// the gate measures in clean worktrees, so a pack that git ignores is absent at both refs and the metric reads
|
|
1128
1013
|
// unmeasurable, failing the guard with no hint why. Say so now, while it is one edit.
|
|
1129
1014
|
if (files.length && gitIgnores(repo, `.promptwheel/${pk[2]}/${files[0]}`))
|
|
1130
1015
|
console.log(` ⚠ .promptwheel/${pk[2]} is gitignored: the gate measures in clean worktrees, so an ignored pack is absent at both refs and ${m.name} reads unmeasurable (the guard fails). Replace ".promptwheel/" in .gitignore with ".promptwheel/*" and "!.promptwheel/packs/", then commit the pack`);
|
|
@@ -1154,7 +1039,7 @@ function init(argv) {
|
|
|
1154
1039
|
if (wrote.length) console.log(`✓ wrote the agent guide (when to run the gate) to ${wrote.join(', ')}`);
|
|
1155
1040
|
}
|
|
1156
1041
|
console.log('\n next: promptwheel run --working # gate your uncommitted changes');
|
|
1157
|
-
console.log(' promptwheel init --list # other presets (
|
|
1042
|
+
console.log(' promptwheel init --list # other presets (antihack, no-ai-tells, supply-chain)');
|
|
1158
1043
|
}
|
|
1159
1044
|
|
|
1160
1045
|
// Write (or refresh) a short agent-orienting block so the coding AGENT runs the gate itself, not just
|
|
@@ -1177,34 +1062,6 @@ function writeAgentBlock(repo) {
|
|
|
1177
1062
|
return done;
|
|
1178
1063
|
}
|
|
1179
1064
|
|
|
1180
|
-
// observability: list the EFFECTIVE guardrails (incl. inherited) + each one's flag record
|
|
1181
|
-
function guards(argv) {
|
|
1182
|
-
const args = parseArgs(argv);
|
|
1183
|
-
const repo = repoRoot(process.cwd());
|
|
1184
|
-
const cfg = loadConfig(repo);
|
|
1185
|
-
const hist = {};
|
|
1186
|
-
const f = join(repo, '.promptwheel', 'outcomes.jsonl');
|
|
1187
|
-
if (existsSync(f)) for (const line of readFileSync(f, 'utf8').split('\n').filter(Boolean)) {
|
|
1188
|
-
let r; try { r = JSON.parse(line); } catch { continue; }
|
|
1189
|
-
for (const m of (r.metrics || [])) {
|
|
1190
|
-
const h = hist[m.name] ??= { runs: 0, flagged: 0, last: null };
|
|
1191
|
-
h.runs++; if (m.guard && m.ok === false) h.flagged++; h.last = m.status;
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
|
-
const rows = cfg.metrics.map((m) => ({
|
|
1195
|
-
name: m.name, direction: m.direction || 'up', guard: !!m.guard,
|
|
1196
|
-
source: m.__src, override: !!m.__override, ...(hist[m.name] || { runs: 0, flagged: 0, last: null }),
|
|
1197
|
-
}));
|
|
1198
|
-
if (args.json) { console.log(JSON.stringify({ guards: rows }, null, 2)); return; }
|
|
1199
|
-
console.log('\nPromptWheel guardrails, effective set for this repo\n');
|
|
1200
|
-
for (const r of rows) {
|
|
1201
|
-
const prov = r.override ? 'local override' : (r.source === 'local' ? 'local' : `inherited ← ${r.source}`);
|
|
1202
|
-
const rec = r.guard ? (r.runs ? `· ${r.flagged} flagged / ${r.runs} runs` : '· no runs yet') : '';
|
|
1203
|
-
console.log(` ${r.guard ? '🛡️ GUARD' : '· info '} ${r.name.padEnd(18)} better=${r.direction.padEnd(5)} [${prov}] ${rec}`);
|
|
1204
|
-
}
|
|
1205
|
-
console.log('\n 🛡️ = enforced (a trusted regression fails the gate) · info = tracked only\n');
|
|
1206
|
-
}
|
|
1207
|
-
|
|
1208
1065
|
const short = (repo, ref) => { try { return git(['rev-parse', '--short', ref], repo); } catch { return ref; } };
|
|
1209
1066
|
|
|
1210
1067
|
function printHuman(r) {
|
|
@@ -1224,7 +1081,7 @@ function printHuman(r) {
|
|
|
1224
1081
|
console.log(`\n VERDICT: ${r.verdict.toUpperCase()}${r.verdict === 'fail' ? failReason : r.verdict === 'gamed' ? ', a metric "improved" by editing the goalposts, not the source' : r.verdict === 'inconclusive' ? ', a guard is inert (measured nothing); a pass cannot be certified' : ''}`);
|
|
1225
1082
|
if (r.verdict === 'fail' && r.mandates) console.log(' a mandate is a hard rule; if the change is truly intended, edit the mandate in promptwheel.config.json (a tracked, tamper-evident change), do not work around it');
|
|
1226
1083
|
else if (r.verdict === 'fail' && r.metrics.some((m) => m.allowable)) console.log(` intentional? this guard accepts a person's declared reason: promptwheel run --allow "<reason>" passes with the reason on the record (${r.metrics.filter((m) => m.allowable).map((m) => m.name).join(', ')})`);
|
|
1227
|
-
else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name)
|
|
1084
|
+
else if (r.verdict === 'fail') console.log(' intentional? loosen that guard locally in promptwheel.config.json (guard:false, or override the inherited metric by name)');
|
|
1228
1085
|
if (r.allowed) console.log(` ✓ allowed: ${r.allowed.cleared.join(', ')} acknowledged as an intentional change, recorded (Promptwheel-Allow: ${r.allowed.reason})`);
|
|
1229
1086
|
console.log('');
|
|
1230
1087
|
}
|
|
@@ -1271,13 +1128,13 @@ function renderMarkdown(r) {
|
|
|
1271
1128
|
'',
|
|
1272
1129
|
`\`${r.base} → ${r.head}\` · ${r.mode} mode${r.repeat > 1 ? ` · ×${r.repeat}` : ''}`,
|
|
1273
1130
|
'',
|
|
1274
|
-
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **
|
|
1131
|
+
...(r.mandates ? [r.mandates.map((md) => `> ⛔ **Protected path touched:** \`${md.name}\`${md.reason ? ': ' + md.reason : ''} (touched \`${md.files.join('`, `')}\`)${md.removed ? ' (deleted in this same change; it stays in force for the change that removes it)' : md.edited ? ' (edited in this same change; the version before the change stays in force for the change that edits it)' : ''}`).join('\n'), ''] : []),
|
|
1275
1132
|
...(r.explained ? [...r.explained.map((x) => `> ${x.noted ? 'ℹ️ **Note:**' : '✋ **Needs a look:**'} ${x.what}${x.items ? ': `' + x.items.join('`, `') + '`' : ''}`), ...(r.explained.some((x) => !x.noted && x.allowable) ? ['>', '> To accept this, comment `sawbuck-allow: <your reason>` on this PR and re-run the check.'] : []), ''] : []),
|
|
1276
1133
|
'| metric | before | after | Δ | status | confidence |',
|
|
1277
1134
|
'|---|--:|--:|--:|---|---|',
|
|
1278
1135
|
rows,
|
|
1279
1136
|
'',
|
|
1280
|
-
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A
|
|
1137
|
+
r.verdict === 'gamed' ? '> 🚩 A metric "improved" only because the agent edited tests/config/grader, not the source. The win does not survive a source-only re-run.' : r.verdict === 'fail' ? (r.mandates ? '> ⛔ A protected path changed. This is a hard rule, enforced before any metric.' : '> ❌ A 🛡️ guarded metric regressed beyond the noise band.') : r.verdict === 'inconclusive' ? '> 🟡 A 🛡️ guarded check is **inert**, it never runs, so nothing was actually verified. Fix its command before trusting a pass.' : '> ✅ No guarded metric regressed.',
|
|
1281
1138
|
...(r.gate && r.gate.weakened ? ['', `> 🚩 **Gate weakened:** ${r.gate.weakened.map((w) => `\`${w.name}\` ${w.change === 'guard-removed' ? 'removed' : 'disabled'} (would have ${w.status} at head)`).join('; ')}. The pass was bought by dropping a guard, not by earning it.`] : []),
|
|
1282
1139
|
...(r.gate && r.gate.changed ? ['', `> ⚠ **Gate definition changed** in this diff (fingerprint \`${r.gate.fingerprint.base}\` -> \`${r.gate.fingerprint.head}\`): a metric, guard, evaluatorPath or mandate was added, removed or edited. Review the config change with the code.`] : []),
|
|
1283
1140
|
'',
|
|
@@ -1292,12 +1149,10 @@ function parseArgs(argv) {
|
|
|
1292
1149
|
else if (argv[i] === '--head') a.head = argv[++i];
|
|
1293
1150
|
else if (argv[i] === '--repeat') a.repeat = parseInt(argv[++i], 10);
|
|
1294
1151
|
else if (argv[i] === '--working') a.working = true;
|
|
1295
|
-
else if (argv[i] === '--no-record') a.noRecord = true;
|
|
1296
1152
|
else if (argv[i] === '--no-auto') a.noAuto = true;
|
|
1297
1153
|
else if (argv[i] === '--allow') a.allow = argv[++i];
|
|
1298
1154
|
else if (argv[i] === '--detect-gaming') a.detectGaming = true;
|
|
1299
1155
|
else if (argv[i] === '--no-detect-gaming') a.detectGaming = false;
|
|
1300
|
-
else if (argv[i] === '--attempt') a.attempt = argv[++i];
|
|
1301
1156
|
else if (argv[i] === '--json') a.json = true;
|
|
1302
1157
|
else if (argv[i] === '--markdown') a.markdown = true;
|
|
1303
1158
|
}
|
|
@@ -1351,7 +1206,6 @@ export function verdictExitCode(verdict) {
|
|
|
1351
1206
|
// repeat number measurement repeats for the noise band (default: config.repeat ?? 1)
|
|
1352
1207
|
// detectGaming boolean run the Section 3 source-only re-proof (default: true)
|
|
1353
1208
|
// allow string a declared override reason (same as --allow / a Promptwheel-Allow: trailer)
|
|
1354
|
-
// record boolean append the run to .promptwheel/outcomes.jsonl (default: false)
|
|
1355
1209
|
export function check(options = {}) {
|
|
1356
1210
|
// `repo` may name a subdirectory: the gate runs at the toplevel (that is where the config lives
|
|
1357
1211
|
// and where .promptwheel/ is written), exactly as the CLI does from the same directory
|
|
@@ -1376,7 +1230,6 @@ export function check(options = {}) {
|
|
|
1376
1230
|
repeat: options.repeat,
|
|
1377
1231
|
detectGaming: options.detectGaming ?? true,
|
|
1378
1232
|
allow: options.allow,
|
|
1379
|
-
noRecord: options.record !== true,
|
|
1380
1233
|
});
|
|
1381
1234
|
}
|
|
1382
1235
|
|
|
@@ -1466,67 +1319,21 @@ export function conformsToContract(report) {
|
|
|
1466
1319
|
return { ok: errors.length === 0, errors };
|
|
1467
1320
|
}
|
|
1468
1321
|
|
|
1469
|
-
// `promptwheel conformance [file.json | -] [--cmd "<command>"]`
|
|
1470
|
-
// Self-check a Contract implementation. With a file or stdin, validate that verdict JSON
|
|
1471
|
-
// against the Section 2 schema. With --cmd, run the command, then validate its --json
|
|
1472
|
-
// stdout (Section 2) AND that its exit code matches the verdict per Section 1. Exit 0 if
|
|
1473
|
-
// it conforms, 1 otherwise, listing every problem.
|
|
1474
|
-
function conformance(argv) {
|
|
1475
|
-
let cmd = null, src = null;
|
|
1476
|
-
for (let i = 0; i < argv.length; i++) {
|
|
1477
|
-
if (argv[i] === '--cmd') cmd = argv[++i];
|
|
1478
|
-
else if (argv[i] === '-' || !argv[i].startsWith('-')) src = argv[i];
|
|
1479
|
-
}
|
|
1480
|
-
|
|
1481
|
-
let text, exitCode = null;
|
|
1482
|
-
if (cmd) {
|
|
1483
|
-
try { text = execFileSync('sh', ['-c', cmd], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); exitCode = 0; }
|
|
1484
|
-
catch (e) {
|
|
1485
|
-
text = (e.stdout || '').toString();
|
|
1486
|
-
exitCode = typeof e.status === 'number' ? e.status : null;
|
|
1487
|
-
if (!text) { console.error(`conformance: command produced no stdout (exit ${exitCode}): ${e.message}`); process.exit(1); }
|
|
1488
|
-
}
|
|
1489
|
-
} else {
|
|
1490
|
-
const file = src && src !== '-' ? src : 0; // fd 0 = stdin
|
|
1491
|
-
try { text = readFileSync(file, 'utf8'); }
|
|
1492
|
-
catch (e) { console.error(`conformance: cannot read ${!src || src === '-' ? 'stdin' : src}: ${e.message}`); process.exit(1); }
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
let report;
|
|
1496
|
-
try { report = JSON.parse(text); }
|
|
1497
|
-
catch (e) { console.error(`conformance: not valid JSON: ${e.message}`); process.exit(1); }
|
|
1498
|
-
|
|
1499
|
-
const { errors } = conformsToContract(report);
|
|
1500
|
-
const problems = [...errors];
|
|
1501
|
-
if (cmd && exitCode !== null && VERDICTS.includes(report.verdict)) {
|
|
1502
|
-
const want = verdictExitCode(report.verdict);
|
|
1503
|
-
if (exitCode !== want) problems.push(`exit code ${exitCode} does not match verdict '${report.verdict}' (Section 1 requires ${want})`);
|
|
1504
|
-
}
|
|
1505
|
-
|
|
1506
|
-
if (problems.length === 0) {
|
|
1507
|
-
console.log(`ok, conforms to the Verified-Outcome Contract v${CONTRACT_VERSION} (verdict: ${report.verdict}${cmd ? `, exit ${exitCode}` : ''})`);
|
|
1508
|
-
process.exit(0);
|
|
1509
|
-
}
|
|
1510
|
-
console.error(`does NOT conform to the Contract (${problems.length} problem${problems.length > 1 ? 's' : ''}):`);
|
|
1511
|
-
for (const p of problems) console.error(` - ${p}`);
|
|
1512
|
-
process.exit(1);
|
|
1513
|
-
}
|
|
1514
|
-
|
|
1515
1322
|
// the flags each command accepts. An unknown --flag is a usage error (exit 64), never silently
|
|
1516
1323
|
// ignored: a typo in a CI step must not turn `--no-detect-gaming` into the default gate or `--allow`
|
|
1517
1324
|
// into nothing, and INTEGRATE.md promises a bad flag exits 64. A flag that takes a value consumes
|
|
1518
|
-
// the next argument; anything not starting with `-`
|
|
1325
|
+
// the next argument; anything not starting with `-` is positional. --no-record does nothing now that
|
|
1326
|
+
// no record is written, and stays accepted because Sawbuck and existing CI steps pass it.
|
|
1519
1327
|
const GATE_FLAGS = ['--base', '--head', '--repeat', '--working', '--no-record', '--no-auto', '--allow', '--detect-gaming', '--no-detect-gaming', '--json', '--markdown'];
|
|
1520
1328
|
const COMMAND_FLAGS = {
|
|
1521
|
-
run: GATE_FLAGS,
|
|
1522
|
-
init: ['--preset', '--list', '--force', '--no-agent-block'], conformance: ['--cmd'], lock: [],
|
|
1329
|
+
run: GATE_FLAGS, init: ['--preset', '--list', '--force', '--no-agent-block'], lock: [],
|
|
1523
1330
|
};
|
|
1524
|
-
const VALUED_FLAGS = new Set(['--base', '--head', '--repeat', '--allow', '--
|
|
1331
|
+
const VALUED_FLAGS = new Set(['--base', '--head', '--repeat', '--allow', '--preset']);
|
|
1525
1332
|
function checkFlags(cmd, argv) {
|
|
1526
1333
|
const known = new Set(COMMAND_FLAGS[cmd]);
|
|
1527
1334
|
for (let i = 0; i < argv.length; i++) {
|
|
1528
1335
|
const a = argv[i];
|
|
1529
|
-
if (
|
|
1336
|
+
if (!a.startsWith('-')) continue;
|
|
1530
1337
|
if (!known.has(a)) bad(`promptwheel ${cmd}: unknown flag ${a} (see promptwheel --help)`);
|
|
1531
1338
|
if (VALUED_FLAGS.has(a)) i++;
|
|
1532
1339
|
}
|
|
@@ -1546,12 +1353,8 @@ function dispatch() {
|
|
|
1546
1353
|
const cmd = (rest.includes('--help') || rest.includes('-h')) ? 'help' : first; // `promptwheel run --help` is a help request, not a gate run
|
|
1547
1354
|
if (cmd in COMMAND_FLAGS) checkFlags(cmd, rest);
|
|
1548
1355
|
if (cmd === 'run') run(rest);
|
|
1549
|
-
else if (cmd === 'improve') improve(rest);
|
|
1550
1356
|
else if (cmd === 'pr-allow') { let c = []; try { c = JSON.parse(readFileSync(0, 'utf8') || '[]'); } catch { /* no comments, no approval */ } process.stdout.write(allowFromComments(c)); }
|
|
1551
|
-
else if (cmd === 'insights') insights(rest);
|
|
1552
1357
|
else if (cmd === 'init') init(rest);
|
|
1553
|
-
else if (cmd === 'guards') guards(rest);
|
|
1554
|
-
else if (cmd === 'conformance') conformance(rest);
|
|
1555
1358
|
else if (cmd === 'lock') lock(rest);
|
|
1556
1359
|
else {
|
|
1557
1360
|
const help = !cmd || cmd === '--help' || cmd === '-h' || cmd === 'help';
|
|
@@ -1568,15 +1371,9 @@ function dispatch() {
|
|
|
1568
1371
|
' exit 2, when a metric only moved by editing tests/config/grader). Use --no-detect-gaming for the bare gate.',
|
|
1569
1372
|
' promptwheel run --working gate uncommitted changes (incl. newly added files)',
|
|
1570
1373
|
' promptwheel run --allow "<reason>" acknowledge an intentional test/config change (or a `Promptwheel-Allow: <reason>` commit trailer; `Sawbuck-Allow:` is an alias): recorded, not flagged GAMED',
|
|
1571
|
-
' promptwheel improve --attempt "<cmd>" run an agent/script; keep only if a metric improved',
|
|
1572
|
-
' exit 0=kept · 1=regression · 2=gamed · 3=plateau/inconclusive (all but 0 reverted) · add --json',
|
|
1573
|
-
' promptwheel insights [--json] which metrics actually respond (raw counts from .promptwheel/outcomes.jsonl)',
|
|
1574
|
-
' promptwheel guards [--json] show the effective guardrails (incl. inherited) + flag record',
|
|
1575
|
-
' promptwheel conformance [file|-] [--cmd C] check a verdict JSON (or a command) against the Contract (exit 0 = conforms)',
|
|
1576
1374
|
'',
|
|
1577
1375
|
'Exit codes: 0 pass · 1 fail (a guarded metric regressed, or a mandate was violated) · 2 gamed · 3 inconclusive · 64 usage or config error (never a verdict).',
|
|
1578
1376
|
'Integrate (library / conform / shell / MCP): see INTEGRATE.md. Import: `import { check } from \'promptwheel\'`.',
|
|
1579
|
-
'Loop it: while promptwheel improve --attempt "$AGENT"; do :; done # stops on plateau/regression',
|
|
1580
1377
|
'Config: promptwheel.config.json → { extends?, metrics:[{ name, cmd, direction, extract?, guard? }] } (or: promptwheel init)',
|
|
1581
1378
|
' extends: a path (or array) to a shared base config, repos INHERIT its guardrails; local metrics override by name.',
|
|
1582
1379
|
].join('\n'));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "promptwheel",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "A deterministic, zero-LLM check for your coding loop. A win must survive a re-run from the agent's source edits alone, so a test that went green by editing the test reads GAMED. Runs in CI or a Claude Code plugin.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/packs/security/README.md
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
# PromptWheel security guard-pack
|
|
2
|
-
|
|
3
|
-
Gate your AI coding loop on **security invariants**, not just tests. When an agent
|
|
4
|
-
writes backend code, "tests pass" says nothing about whether it shipped a CORS
|
|
5
|
-
wildcard, a client-controlled payment amount, or a hardcoded secret. This pack
|
|
6
|
-
adds a `security_findings` metric that must not climb - and because it's a
|
|
7
|
-
PromptWheel metric, the win has to survive `--detect-gaming`: an agent can't
|
|
8
|
-
lower the count by editing the pattern file or a config, only by fixing the code.
|
|
9
|
-
|
|
10
|
-
## Use it
|
|
11
|
-
|
|
12
|
-
The pack ships inside the `promptwheel` package, not in your repo, so install it with the
|
|
13
|
-
preset:
|
|
14
|
-
|
|
15
|
-
```bash
|
|
16
|
-
promptwheel init --preset security # or --preset security-antihack, which adds the tripwires
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
`init` copies the pack into `.promptwheel/packs/security/` and writes a `security_findings`
|
|
20
|
-
metric that runs it from there. Commit the pack with the config: the gate measures in clean
|
|
21
|
-
git worktrees, so a pack that is not in git is absent at both refs and the metric reads
|
|
22
|
-
`unmeasurable` (init warns when `.promptwheel/` is gitignored; change that rule to
|
|
23
|
-
`.promptwheel/*` plus `!.promptwheel/packs/`, since git cannot re-include a file under an
|
|
24
|
-
ignored directory). Then gate as usual:
|
|
25
|
-
|
|
26
|
-
```bash
|
|
27
|
-
promptwheel run --working # fails if security_findings increased
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
`pack.json` is the same metric as a config fragment for `extends`
|
|
31
|
-
(`{ "extends": ["packs/security/pack.json"] }`); its command is relative to a repo that
|
|
32
|
-
holds the pack at `packs/security/`, such as a vendored copy or this repository.
|
|
33
|
-
|
|
34
|
-
A finding-count that drops because you fixed the code is a `PASS`; a "fix" that
|
|
35
|
-
only survives with test/config/pattern edits reverted is `GAMED`.
|
|
36
|
-
|
|
37
|
-
## What it checks (v0.1 - 8 high-precision invariants)
|
|
38
|
-
|
|
39
|
-
Decanted from [securitychecks](https://securitychecks.ai)' "recommended" tier -
|
|
40
|
-
the regex-portable subset, chosen for near-zero false positives:
|
|
41
|
-
|
|
42
|
-
| Invariant | Severity |
|
|
43
|
-
| --- | --- |
|
|
44
|
-
| CORS wildcard origin | P0 |
|
|
45
|
-
| CORS origin reflection | P0 |
|
|
46
|
-
| Insecure session cookie (`secure:false`) | P0 |
|
|
47
|
-
| Payment amount from client input | P0 |
|
|
48
|
-
| Hardcoded secret in a sensitive variable | P0 |
|
|
49
|
-
| NPM token committed in `.npmrc` | P0 |
|
|
50
|
-
| Template autoescape disabled (XSS) | P0 |
|
|
51
|
-
| S3 presigned upload without content validation | P2 |
|
|
52
|
-
|
|
53
|
-
Each pattern ships with its own false-positive guards (the `exclude` regexes).
|
|
54
|
-
|
|
55
|
-
## Scope, honestly
|
|
56
|
-
|
|
57
|
-
This is the **regex-portable** slice. The other ~75 recommended invariants
|
|
58
|
-
(auth-enforcement, authz-revocation, tenant-isolation, dataflow-tainted sinks)
|
|
59
|
-
need an AST + dataflow engine and are a possible follow-up, not part of this
|
|
60
|
-
pack. The point of v0.1 is the architecture: security invariants are
|
|
61
|
-
now a first-class, un-gameable PromptWheel metric. Zero dependencies; the scanner
|
|
62
|
-
is one file (`scan.mjs`).
|
package/packs/security/pack.json
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"$comment": "PromptWheel security guard-pack. Decanted from securitychecks (recommended tier). Use via { \"extends\": [\"packs/security/pack.json\"] } in your promptwheel.config.json. Gates on the count of security-invariant violations (direction: down = lower is better; guard: true = fail if it climbs). Because it's a promptwheel metric, the win must survive --detect-gaming: an agent can't lower the count by editing the pattern file or a config, only by fixing the code.",
|
|
3
|
-
"metrics": [
|
|
4
|
-
{
|
|
5
|
-
"name": "security_findings",
|
|
6
|
-
"cmd": "node packs/security/scan.mjs .",
|
|
7
|
-
"extract": "number",
|
|
8
|
-
"direction": "down",
|
|
9
|
-
"guard": true
|
|
10
|
-
}
|
|
11
|
-
]
|
|
12
|
-
}
|
|
@@ -1,209 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"pack": "security",
|
|
3
|
-
"version": "0.1.0",
|
|
4
|
-
"source": "securitychecks recommended tier (regex-portable subset)",
|
|
5
|
-
"patterns": [
|
|
6
|
-
{
|
|
7
|
-
"id": "DATAFLOW.UNTRUSTED.XSS",
|
|
8
|
-
"name": "Template Engine Autoescape Disabled",
|
|
9
|
-
"severity": "P0",
|
|
10
|
-
"match": "autoescape\\s*:\\s*false",
|
|
11
|
-
"exclude": [],
|
|
12
|
-
"message": "Template engine autoescape disabled \u2014 enables stored XSS attacks",
|
|
13
|
-
"fix": "Enable autoescape or sanitize all user input before rendering",
|
|
14
|
-
"files": [
|
|
15
|
-
"**/*.ts",
|
|
16
|
-
"**/*.js",
|
|
17
|
-
"!**/__tests__/**",
|
|
18
|
-
"!**/*.test.*",
|
|
19
|
-
"!**/*.spec.*",
|
|
20
|
-
"!**/test/**",
|
|
21
|
-
"!**/tests/**",
|
|
22
|
-
"!**/__mocks__/**",
|
|
23
|
-
"!**/fixtures/**",
|
|
24
|
-
"!**/www/**",
|
|
25
|
-
"!**/docs/**",
|
|
26
|
-
"!**/documentation/**"
|
|
27
|
-
]
|
|
28
|
-
},
|
|
29
|
-
{
|
|
30
|
-
"id": "SESSION.INSECURE_COOKIE",
|
|
31
|
-
"name": "Insecure Session Cookie",
|
|
32
|
-
"severity": "P0",
|
|
33
|
-
"match": "secure\\s*:\\s*false",
|
|
34
|
-
"exclude": [],
|
|
35
|
-
"message": "Session cookies must have secure and httpOnly flags",
|
|
36
|
-
"fix": "Verify this is not a false positive",
|
|
37
|
-
"files": [
|
|
38
|
-
"**/*.ts",
|
|
39
|
-
"**/*.js",
|
|
40
|
-
"**/*.tsx",
|
|
41
|
-
"**/*.jsx",
|
|
42
|
-
"!**/__tests__/**",
|
|
43
|
-
"!**/*.test.*",
|
|
44
|
-
"!**/*.spec.*",
|
|
45
|
-
"!**/test/**",
|
|
46
|
-
"!**/tests/**",
|
|
47
|
-
"!**/__mocks__/**",
|
|
48
|
-
"!**/fixtures/**",
|
|
49
|
-
"!**/e2e/**",
|
|
50
|
-
"!**/playwright/**",
|
|
51
|
-
"!**/cypress/**",
|
|
52
|
-
"!**/test-utils/**",
|
|
53
|
-
"!**/*.e2e.*",
|
|
54
|
-
"!**/e2e-*/**"
|
|
55
|
-
]
|
|
56
|
-
},
|
|
57
|
-
{
|
|
58
|
-
"id": "AUTHZ.CORS_WILDCARD",
|
|
59
|
-
"name": "CORS Wildcard Origin",
|
|
60
|
-
"severity": "P0",
|
|
61
|
-
"match": "Access-Control-Allow-Origin.*\\*|origin\\s*:\\s*\\*|origin:\\s*true",
|
|
62
|
-
"exclude": [
|
|
63
|
-
"!credentials"
|
|
64
|
-
],
|
|
65
|
-
"message": "Wildcard origin with credentials exposes user data",
|
|
66
|
-
"fix": "Verify this is not a false positive",
|
|
67
|
-
"files": [
|
|
68
|
-
"**/*.ts",
|
|
69
|
-
"**/*.js",
|
|
70
|
-
"**/*.tsx",
|
|
71
|
-
"**/*.jsx",
|
|
72
|
-
"!**/__tests__/**",
|
|
73
|
-
"!**/*.test.*",
|
|
74
|
-
"!**/*.spec.*",
|
|
75
|
-
"!**/test/**",
|
|
76
|
-
"!**/tests/**",
|
|
77
|
-
"!**/__mocks__/**",
|
|
78
|
-
"!**/fixtures/**",
|
|
79
|
-
"!**/*.tsx",
|
|
80
|
-
"!**/*.jsx",
|
|
81
|
-
"!**/components/**"
|
|
82
|
-
]
|
|
83
|
-
},
|
|
84
|
-
{
|
|
85
|
-
"id": "AUTHZ.CORS_REFLECT",
|
|
86
|
-
"name": "CORS Origin Reflection",
|
|
87
|
-
"severity": "P0",
|
|
88
|
-
"match": "(?:Access-Control-Allow-Origin|res\\.(?:set|setHeader|header)\\(['\"]Access-Control).*(?:req\\.headers\\.origin|headers\\.get\\(['\"]origin['\"]\\))",
|
|
89
|
-
"exclude": [],
|
|
90
|
-
"message": "Reflecting origin is equivalent to wildcard \u2014 validate against allowlist",
|
|
91
|
-
"fix": "Verify this is not a false positive",
|
|
92
|
-
"files": [
|
|
93
|
-
"**/*.ts",
|
|
94
|
-
"**/*.js",
|
|
95
|
-
"**/*.tsx",
|
|
96
|
-
"**/*.jsx",
|
|
97
|
-
"!**/__tests__/**",
|
|
98
|
-
"!**/*.test.*",
|
|
99
|
-
"!**/*.spec.*",
|
|
100
|
-
"!**/test/**",
|
|
101
|
-
"!**/tests/**",
|
|
102
|
-
"!**/__mocks__/**",
|
|
103
|
-
"!**/fixtures/**"
|
|
104
|
-
]
|
|
105
|
-
},
|
|
106
|
-
{
|
|
107
|
-
"id": "PAY.CLIENT_AMOUNT",
|
|
108
|
-
"name": "Payment Amount From Client",
|
|
109
|
-
"severity": "P0",
|
|
110
|
-
"match": "amount.*req\\.body|price.*req\\.body|total.*body\\.amount",
|
|
111
|
-
"exclude": [
|
|
112
|
-
"server|database|catalog"
|
|
113
|
-
],
|
|
114
|
-
"message": "Calculate payment amounts server-side, not from client input",
|
|
115
|
-
"fix": "Verify this is not a false positive",
|
|
116
|
-
"files": [
|
|
117
|
-
"**/*.ts",
|
|
118
|
-
"**/*.js",
|
|
119
|
-
"**/*.tsx",
|
|
120
|
-
"**/*.jsx",
|
|
121
|
-
"!**/__tests__/**",
|
|
122
|
-
"!**/*.test.*",
|
|
123
|
-
"!**/*.spec.*",
|
|
124
|
-
"!**/test/**",
|
|
125
|
-
"!**/tests/**",
|
|
126
|
-
"!**/__mocks__/**",
|
|
127
|
-
"!**/fixtures/**"
|
|
128
|
-
]
|
|
129
|
-
},
|
|
130
|
-
{
|
|
131
|
-
"id": "CONFIG.NPMRC_TOKEN",
|
|
132
|
-
"name": "NPM Token in .npmrc",
|
|
133
|
-
"severity": "P0",
|
|
134
|
-
"match": "_authToken=|//registry.*:_auth",
|
|
135
|
-
"exclude": [
|
|
136
|
-
"\\$\\{NPM_TOKEN\\}|process\\.env"
|
|
137
|
-
],
|
|
138
|
-
"message": "NPM tokens should come from environment variables",
|
|
139
|
-
"fix": "Verify this is not a false positive",
|
|
140
|
-
"files": [
|
|
141
|
-
"**/*",
|
|
142
|
-
"!**/__tests__/**",
|
|
143
|
-
"!**/*.test.*",
|
|
144
|
-
"!**/*.spec.*",
|
|
145
|
-
"!**/test/**",
|
|
146
|
-
"!**/tests/**",
|
|
147
|
-
"!**/__mocks__/**",
|
|
148
|
-
"!**/fixtures/**"
|
|
149
|
-
]
|
|
150
|
-
},
|
|
151
|
-
{
|
|
152
|
-
"id": "VALID.UPLOAD_NOCHECK",
|
|
153
|
-
"name": "S3 Presigned Upload Without Content Validation",
|
|
154
|
-
"severity": "P2",
|
|
155
|
-
"match": "(?:createPresignedPost|getSignedUrl|PutObjectCommand)\\s*\\(",
|
|
156
|
-
"exclude": [
|
|
157
|
-
"ContentType|content-type|Content-Length|content-length|Conditions|conditions",
|
|
158
|
-
"allowedTypes|fileFilter|mimetype|maxSize|maxFileSize",
|
|
159
|
-
"@example|Example:|```|description:|guidance",
|
|
160
|
-
"GetObjectCommand|getObject|HeadObjectCommand|DeleteObjectCommand|download|Download|read|Read",
|
|
161
|
-
"generateSignedUrl.*get|getSignedUrl.*getObject"
|
|
162
|
-
],
|
|
163
|
-
"message": "S3 presigned upload URL has no content type or size conditions \u2014 any file type/size can be uploaded",
|
|
164
|
-
"fix": "Add Conditions to presigned POST or ContentType/ContentLength constraints",
|
|
165
|
-
"files": [
|
|
166
|
-
"**/*.ts",
|
|
167
|
-
"**/*.js",
|
|
168
|
-
"!**/*.tsx",
|
|
169
|
-
"!**/*.jsx",
|
|
170
|
-
"!**/components/**",
|
|
171
|
-
"!**/frontend/**",
|
|
172
|
-
"!**/public/**",
|
|
173
|
-
"!**/__tests__/**",
|
|
174
|
-
"!**/*.test.*",
|
|
175
|
-
"!**/*.spec.*",
|
|
176
|
-
"!**/test/**",
|
|
177
|
-
"!**/tests/**",
|
|
178
|
-
"!**/__mocks__/**",
|
|
179
|
-
"!**/fixtures/**"
|
|
180
|
-
]
|
|
181
|
-
},
|
|
182
|
-
{
|
|
183
|
-
"id": "SECRETS.HARDCODED",
|
|
184
|
-
"name": "Hardcoded Secret in Sensitive Variable",
|
|
185
|
-
"severity": "P0",
|
|
186
|
-
"match": "(?:const|let|var)\\s+(?:[A-Z_]*(?:SECRET|JWT_SECRET|API_KEY|AUTH_TOKEN|PRIVATE_KEY|SIGNING_KEY|ENCRYPTION_KEY)[A-Z_]*)\\s*=\\s*['\"][^'\"]{8,}['\"]",
|
|
187
|
-
"exclude": [
|
|
188
|
-
"process\\.env|import\\.meta\\.env|getenv|\\$\\{|placeholder|changeme|YOUR_|TODO|FIXME|example|EXAMPLE"
|
|
189
|
-
],
|
|
190
|
-
"message": "Secret value hardcoded in source code \u2014 use environment variables",
|
|
191
|
-
"fix": "Verify this is not a placeholder or example value",
|
|
192
|
-
"files": [
|
|
193
|
-
"**/*.ts",
|
|
194
|
-
"**/*.js",
|
|
195
|
-
"!**/__tests__/**",
|
|
196
|
-
"!**/*.test.*",
|
|
197
|
-
"!**/*.spec.*",
|
|
198
|
-
"!**/test/**",
|
|
199
|
-
"!**/tests/**",
|
|
200
|
-
"!**/__mocks__/**",
|
|
201
|
-
"!**/fixtures/**",
|
|
202
|
-
"!**/sample/**",
|
|
203
|
-
"!**/samples/**",
|
|
204
|
-
"!**/example/**",
|
|
205
|
-
"!**/examples/**"
|
|
206
|
-
]
|
|
207
|
-
}
|
|
208
|
-
]
|
|
209
|
-
}
|
package/packs/security/scan.mjs
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// PromptWheel security guard-pack scanner — zero-dep.
|
|
3
|
-
// Decanted from securitychecks' "recommended" tier (the regex-portable subset).
|
|
4
|
-
// Emits the count of security-invariant violations to stdout (the metric value).
|
|
5
|
-
// The full AST+dataflow engine (the other 75 recommended patterns) is the follow-up.
|
|
6
|
-
//
|
|
7
|
-
// Usage: node scan.mjs [path] → prints the finding count (for `promptwheel` metrics)
|
|
8
|
-
// node scan.mjs [path] --list → prints each finding (file:line · id · message)
|
|
9
|
-
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
|
10
|
-
import { join, extname } from 'node:path';
|
|
11
|
-
import { fileURLToPath } from 'node:url';
|
|
12
|
-
|
|
13
|
-
const ROOT = process.argv[2] && !process.argv[2].startsWith('--') ? process.argv[2] : '.';
|
|
14
|
-
const LIST = process.argv.includes('--list');
|
|
15
|
-
const PACK = JSON.parse(readFileSync(join(fileURLToPath(new URL('.', import.meta.url)), 'patterns.json'), 'utf8'));
|
|
16
|
-
|
|
17
|
-
const SKIP = /(^|\/)(node_modules|\.git|dist|build|\.next|coverage|vendor)(\/|$)/;
|
|
18
|
-
const CODE = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']);
|
|
19
|
-
const CONFIG_FILES = /(^|\/)\.npmrc$/;
|
|
20
|
-
|
|
21
|
-
function* walk(dir) {
|
|
22
|
-
let entries;
|
|
23
|
-
try { entries = readdirSync(dir); } catch { return; }
|
|
24
|
-
for (const e of entries) {
|
|
25
|
-
const p = join(dir, e);
|
|
26
|
-
if (SKIP.test(p)) continue;
|
|
27
|
-
let st;
|
|
28
|
-
try { st = statSync(p); } catch { continue; }
|
|
29
|
-
if (st.isDirectory()) yield* walk(p);
|
|
30
|
-
else if (CODE.has(extname(p)) || CONFIG_FILES.test(p)) yield p;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
const findings = [];
|
|
35
|
-
for (const file of walk(ROOT)) {
|
|
36
|
-
let text;
|
|
37
|
-
try { text = readFileSync(file, 'utf8'); } catch { continue; }
|
|
38
|
-
const lines = text.split('\n');
|
|
39
|
-
for (const pat of PACK.patterns) {
|
|
40
|
-
let re;
|
|
41
|
-
try { re = new RegExp(pat.match, 'i'); } catch { continue; }
|
|
42
|
-
const excludes = (pat.exclude || []).map((x) => { try { return new RegExp(x, 'i'); } catch { return null; } }).filter(Boolean);
|
|
43
|
-
for (let i = 0; i < lines.length; i++) {
|
|
44
|
-
const line = lines[i];
|
|
45
|
-
if (!re.test(line)) continue;
|
|
46
|
-
if (excludes.some((ex) => ex.test(line))) continue; // pattern's own false-positive guards
|
|
47
|
-
findings.push({ file, line: i + 1, id: pat.id, sev: pat.severity, msg: pat.message });
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
if (LIST) {
|
|
53
|
-
for (const f of findings) console.error(` [${f.sev}] ${f.file}:${f.line} · ${f.id} · ${f.msg}`);
|
|
54
|
-
console.error(`\n${findings.length} security-invariant violation(s)`);
|
|
55
|
-
}
|
|
56
|
-
// the metric value: lower is better, 0 is clean
|
|
57
|
-
process.stdout.write(String(findings.length) + '\n');
|