promptwheel 0.4.1 → 0.4.3

package/README.md

@@ -2,7 +2,7 @@
 
 [![CI](https://github.com/promptwheel-ai/promptwheel/actions/workflows/ci.yml/badge.svg)](https://github.com/promptwheel-ai/promptwheel/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) ![Node ≥18](https://img.shields.io/badge/node-%E2%89%A518-brightgreen) ![deps: zero](https://img.shields.io/badge/deps-zero-blue)
 
-**Catch your agent cheating** — the deterministic auditor that flags when your AI coding agent gamed its own success metric.
+**Add a referee to your coding loop.** A green test should mean your agent *fixed the code* — not that it edited the test to match the bug. PromptWheel is the referee that tells the difference: deterministic, no LLM, and un-gameable, because a player can't call its own fouls.
 
 PromptWheel re-proves every "win" using the agent's **source edits alone**. If the gate only went green because the agent edited the test, mocked the grader, suppressed the error (`@ts-ignore` / `eslint-disable`), or deleted the feature, the win evaporates when those edits are reverted → **`VERDICT GAMED`, exit 2**. No LLM in the loop — a diff partition plus a re-run, so every flag is reproducible in seconds with a human-readable reason.
 
@@ -242,7 +242,7 @@ The engine is one importable file; pure helpers are exported for unit tests, the
 - [x] `promptwheel init` + presets — zero-config onboarding
 - [x] `insights` — reward-stream aggregation (Phase-5 seed)
 - [x] `--detect-gaming` — reward-hack detection: re-prove the win from source edits alone + `antihack` preset
-- [x] npm publish — `promptwheel@0.4.1` (the lead magnet)
+- [x] npm publish — `promptwheel@0.4.2` (the lead magnet)
 - [x] outcome-curated learning + UCB work-discovery — `playbook` + `suggest` + the compounding A/B harness (**experimental**; compounding *claims* stay gated on real-data proof — see [docs/LEARNING.md](docs/LEARNING.md))
 
-> Status: **published** (npm `promptwheel`, v0.4.1) — all core phases built. Lineage: CommandLayer → BlockSpool → PromptWheel (orchestrator, archived) → **PromptWheel (outcome gate)**.
+> Status: **published** (npm `promptwheel`, v0.4.2) — all core phases built. Lineage: CommandLayer → BlockSpool → PromptWheel (orchestrator, archived) → **PromptWheel (outcome gate)**.

package/bin/promptwheel.mjs

@@ -11,7 +11,7 @@
 //   promptwheel run [--base R] [--head R] [--repeat N] [--json]
 
 import { execSync, execFileSync } from 'node:child_process';
-import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, appendFileSync, realpathSync, readdirSync, statSync } from 'node:fs';
+import { readFileSync, writeFileSync, existsSync, mkdtempSync, symlinkSync, rmSync, mkdirSync, appendFileSync, realpathSync, readdirSync, statSync, copyFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join, dirname, isAbsolute, relative } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -87,13 +87,19 @@ function runMetric(cwd, m, env) {
   try {
     stdout = execSync(m.cmd, { cwd, env: env || process.env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: (m.timeoutSec ?? 300) * 1000 });
   } catch (e) { code = e.status ?? 1; stdout = `${e.stdout || ''}${e.stderr || ''}`; }
-  return extract(stdout, code, m.extract);
+  return extract(stdout, code, m.extract, m.allowNonzero);
 }
 
 // extract modes: 'number' (last number, default) | 'lines' | 'exit' | {regex}
-function extract(stdout, code, mode) {
+// A FAILED command is unmeasurable (null -> the metric goes inert, guards
+// fail loudly) — error text often contains numbers ("Node.js v22") and
+// silently measuring those produced identical junk on both refs = a false
+// green. 'exit' mode consumes nonzero by design; anything else needs the
+// per-metric opt-in `allowNonzero: true`.
+function extract(stdout, code, mode, allowNonzero) {
   mode = mode || 'number';
   if (mode === 'exit') return code === 0 ? 1 : 0;
+  if (code !== 0 && !allowNonzero) return null;
   if (mode === 'lines') return stdout.split('\n').filter((l) => l.trim()).length;
   if (mode && typeof mode === 'object' && mode.regex) {
     const mm = new RegExp(mode.regex).exec(stdout);
@@ -647,6 +653,8 @@ const PRESETS = {
     metrics: [{ name: 'lint_errors', cmd: LINT_CMD, extract: 'number', direction: 'down', guard: false }] },
   'bundle-size': { desc: 'track: build output size in kB',
     metrics: [{ name: 'bundle_kb', cmd: 'du -sk dist 2>/dev/null | cut -f1 || echo 0', extract: 'number', direction: 'down', guard: false }] },
+  'security': { desc: 'gate: security-invariant violations must not climb (CORS, secrets, cookies, payment; see packs/security)',
+    metrics: [{ name: 'security_findings', cmd: 'node packs/security/scan.mjs .', extract: 'number', direction: 'down', guard: true }] },
   'llm-eval': { desc: 'gate: AI-feature eval pass-rate + est $/run (see examples/reliability-sprint)',
     metrics: [
       { name: 'eval_pass_rate', cmd: 'node eval.mjs', extract: 'number', direction: 'up', guard: true },
@@ -717,6 +725,19 @@ function init(argv) {
     if (!p) { console.error(`unknown preset "${presetName}" — try: promptwheel init --list`); process.exit(2); }
     metrics = p.metrics || [{ name: 'tests_pass', cmd: detectTestCmd(repo), extract: 'exit', direction: 'pass', guard: true }];
     metrics = metrics.map((m) => m.cmd === '__TESTCMD__' ? { ...m, cmd: detectTestCmd(repo) } : m); // fill the preset's target
+    // pack-backed presets: the referenced scripts live in THIS package, not
+    // the user's repo — copy them in so the generated config is
+    // self-contained and committable (a bare reference silently 404'd)
+    metrics = metrics.map((m) => {
+      const pk = /(^|\s)(packs\/[\w-]+)\//.exec(m.cmd);
+      if (!pk) return m;
+      const src = fileURLToPath(new URL(`../${pk[2]}/`, import.meta.url));
+      const dst = join(repo, '.promptwheel', pk[2]);
+      mkdirSync(dst, { recursive: true });
+      for (const f of readdirSync(src)) copyFileSync(join(src, f), join(dst, f));
+      console.log(`  ✓ installed ${pk[2]} → .promptwheel/${pk[2]} (commit it with the config)`);
+      return { ...m, cmd: m.cmd.replaceAll(pk[2] + '/', `.promptwheel/${pk[2]}/`) };
+    });
     note = presetName;
   } else {
     // default = the headline posture: guarded tests + the antihack tripwires, so the FIRST

package/package.json

@@ -1,22 +1,46 @@
 {
   "name": "promptwheel",
-  "version": "0.4.1",
-  "description": "Catch your agent cheating — a deterministic, zero-LLM CLI that flags when an AI coding agent gamed its own metric (re-proves the win from the agent's source edits alone). Built on an outcome gate; runs in CI or a Claude Code hook.",
+  "version": "0.4.3",
+  "description": "Add a referee to your coding loop \u2014 a deterministic, zero-LLM CLI that checks every change earned its win: it re-proves the result from the agent's source edits alone, so a green that came from editing the test reads GAMED. Runs in CI or a Claude Code hook.",
   "type": "module",
   "bin": {
     "promptwheel": "bin/promptwheel.mjs",
     "pw": "bin/promptwheel.mjs"
   },
-  "files": ["bin"],
-  "engines": { "node": ">=18" },
+  "files": [
+    "bin",
+    "packs"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
   "scripts": {
     "gate": "node bin/promptwheel.mjs run",
     "test": "node --test"
   },
-  "repository": { "type": "git", "url": "git+https://github.com/promptwheel-ai/promptwheel.git" },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/promptwheel-ai/promptwheel.git"
+  },
   "homepage": "https://github.com/promptwheel-ai/promptwheel#readme",
-  "bugs": { "url": "https://github.com/promptwheel-ai/promptwheel/issues" },
+  "bugs": {
+    "url": "https://github.com/promptwheel-ai/promptwheel/issues"
+  },
   "author": "Matthew Owens",
   "license": "MIT",
-  "keywords": ["agent", "ci", "regression", "metrics", "verification", "ai-coding", "outcome", "evals", "loop", "promptwheel", "reward-hacking", "detect-gaming", "reward-hack-detection"]
-}
+  "keywords": [
+    "agent",
+    "ci",
+    "regression",
+    "metrics",
+    "verification",
+    "ai-coding",
+    "outcome",
+    "evals",
+    "loop",
+    "promptwheel",
+    "reward-hacking",
+    "detect-gaming",
+    "reward-hack-detection"
+  ]
+}

package/packs/security/README.md

@@ -0,0 +1,52 @@
+# PromptWheel security guard-pack
+
+Gate your AI coding loop on **security invariants**, not just tests. When an agent
+writes backend code, "tests pass" says nothing about whether it shipped a CORS
+wildcard, a client-controlled payment amount, or a hardcoded secret. This pack
+adds a `security_findings` metric that must not climb — and because it's a
+PromptWheel metric, the win has to survive `--detect-gaming`: an agent can't
+lower the count by editing the pattern file or a config, only by fixing the code.
+
+## Use it
+
+In your `promptwheel.config.json`:
+
+```json
+{ "extends": ["packs/security/pack.json"] }
+```
+
+Then gate as usual:
+
+```bash
+promptwheel run --working          # fails if security_findings increased
+```
+
+A finding-count that drops because you fixed the code is a `PASS`; a "fix" that
+only survives with test/config/pattern edits reverted is `GAMED`.
+
+## What it checks (v0.1 — 8 high-precision invariants)
+
+Decanted from [securitychecks](https://securitychecks.ai)' "recommended" tier —
+the regex-portable subset, chosen for near-zero false positives:
+
+| Invariant | Severity |
+| --- | --- |
+| CORS wildcard origin | P0 |
+| CORS origin reflection | P0 |
+| Insecure session cookie (`secure:false`) | P0 |
+| Payment amount from client input | P0 |
+| Hardcoded secret in a sensitive variable | P0 |
+| NPM token committed in `.npmrc` | P0 |
+| Template autoescape disabled (XSS) | P0 |
+| S3 presigned upload without content validation | P2 |
+
+Each pattern ships with its own false-positive guards (the `exclude` regexes).
+
+## Scope, honestly
+
+This is the **regex-portable** slice. The other ~75 recommended invariants
+(auth-enforcement, authz-revocation, tenant-isolation, dataflow-tainted sinks)
+need the AST + dataflow engine and are the follow-up — see the decant note in
+`explorations/`. The point of v0.1 is the architecture: security invariants are
+now a first-class, un-gameable PromptWheel metric. Zero dependencies; the scanner
+is one file (`scan.mjs`).

package/packs/security/pack.json

@@ -0,0 +1,12 @@
+{
+  "$comment": "PromptWheel security guard-pack. Decanted from securitychecks (recommended tier). Use via { \"extends\": [\"packs/security/pack.json\"] } in your promptwheel.config.json. Gates on the count of security-invariant violations (direction: down = lower is better; guard: true = fail if it climbs). Because it's a promptwheel metric, the win must survive --detect-gaming: an agent can't lower the count by editing the pattern file or a config, only by fixing the code.",
+  "metrics": [
+    {
+      "name": "security_findings",
+      "cmd": "node packs/security/scan.mjs .",
+      "extract": "number",
+      "direction": "down",
+      "guard": true
+    }
+  ]
+}

package/packs/security/patterns.json

@@ -0,0 +1,209 @@
+{
+  "pack": "security",
+  "version": "0.1.0",
+  "source": "securitychecks recommended tier (regex-portable subset)",
+  "patterns": [
+    {
+      "id": "DATAFLOW.UNTRUSTED.XSS",
+      "name": "Template Engine Autoescape Disabled",
+      "severity": "P0",
+      "match": "autoescape\\s*:\\s*false",
+      "exclude": [],
+      "message": "Template engine autoescape disabled \u2014 enables stored XSS attacks",
+      "fix": "Enable autoescape or sanitize all user input before rendering",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**",
+        "!**/www/**",
+        "!**/docs/**",
+        "!**/documentation/**"
+      ]
+    },
+    {
+      "id": "SESSION.INSECURE_COOKIE",
+      "name": "Insecure Session Cookie",
+      "severity": "P0",
+      "match": "secure\\s*:\\s*false",
+      "exclude": [],
+      "message": "Session cookies must have secure and httpOnly flags",
+      "fix": "Verify this is not a false positive",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "**/*.tsx",
+        "**/*.jsx",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**",
+        "!**/e2e/**",
+        "!**/playwright/**",
+        "!**/cypress/**",
+        "!**/test-utils/**",
+        "!**/*.e2e.*",
+        "!**/e2e-*/**"
+      ]
+    },
+    {
+      "id": "AUTHZ.CORS_WILDCARD",
+      "name": "CORS Wildcard Origin",
+      "severity": "P0",
+      "match": "Access-Control-Allow-Origin.*\\*|origin\\s*:\\s*\\*|origin:\\s*true",
+      "exclude": [
+        "!credentials"
+      ],
+      "message": "Wildcard origin with credentials exposes user data",
+      "fix": "Verify this is not a false positive",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "**/*.tsx",
+        "**/*.jsx",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**",
+        "!**/*.tsx",
+        "!**/*.jsx",
+        "!**/components/**"
+      ]
+    },
+    {
+      "id": "AUTHZ.CORS_REFLECT",
+      "name": "CORS Origin Reflection",
+      "severity": "P0",
+      "match": "(?:Access-Control-Allow-Origin|res\\.(?:set|setHeader|header)\\(['\"]Access-Control).*(?:req\\.headers\\.origin|headers\\.get\\(['\"]origin['\"]\\))",
+      "exclude": [],
+      "message": "Reflecting origin is equivalent to wildcard \u2014 validate against allowlist",
+      "fix": "Verify this is not a false positive",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "**/*.tsx",
+        "**/*.jsx",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**"
+      ]
+    },
+    {
+      "id": "PAY.CLIENT_AMOUNT",
+      "name": "Payment Amount From Client",
+      "severity": "P0",
+      "match": "amount.*req\\.body|price.*req\\.body|total.*body\\.amount",
+      "exclude": [
+        "server|database|catalog"
+      ],
+      "message": "Calculate payment amounts server-side, not from client input",
+      "fix": "Verify this is not a false positive",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "**/*.tsx",
+        "**/*.jsx",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**"
+      ]
+    },
+    {
+      "id": "CONFIG.NPMRC_TOKEN",
+      "name": "NPM Token in .npmrc",
+      "severity": "P0",
+      "match": "_authToken=|//registry.*:_auth",
+      "exclude": [
+        "\\$\\{NPM_TOKEN\\}|process\\.env"
+      ],
+      "message": "NPM tokens should come from environment variables",
+      "fix": "Verify this is not a false positive",
+      "files": [
+        "**/*",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**"
+      ]
+    },
+    {
+      "id": "VALID.UPLOAD_NOCHECK",
+      "name": "S3 Presigned Upload Without Content Validation",
+      "severity": "P2",
+      "match": "(?:createPresignedPost|getSignedUrl|PutObjectCommand)\\s*\\(",
+      "exclude": [
+        "ContentType|content-type|Content-Length|content-length|Conditions|conditions",
+        "allowedTypes|fileFilter|mimetype|maxSize|maxFileSize",
+        "@example|Example:|```|description:|guidance",
+        "GetObjectCommand|getObject|HeadObjectCommand|DeleteObjectCommand|download|Download|read|Read",
+        "generateSignedUrl.*get|getSignedUrl.*getObject"
+      ],
+      "message": "S3 presigned upload URL has no content type or size conditions \u2014 any file type/size can be uploaded",
+      "fix": "Add Conditions to presigned POST or ContentType/ContentLength constraints",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "!**/*.tsx",
+        "!**/*.jsx",
+        "!**/components/**",
+        "!**/frontend/**",
+        "!**/public/**",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**"
+      ]
+    },
+    {
+      "id": "SECRETS.HARDCODED",
+      "name": "Hardcoded Secret in Sensitive Variable",
+      "severity": "P0",
+      "match": "(?:const|let|var)\\s+(?:[A-Z_]*(?:SECRET|JWT_SECRET|API_KEY|AUTH_TOKEN|PRIVATE_KEY|SIGNING_KEY|ENCRYPTION_KEY)[A-Z_]*)\\s*=\\s*['\"][^'\"]{8,}['\"]",
+      "exclude": [
+        "process\\.env|import\\.meta\\.env|getenv|\\$\\{|placeholder|changeme|YOUR_|TODO|FIXME|example|EXAMPLE"
+      ],
+      "message": "Secret value hardcoded in source code \u2014 use environment variables",
+      "fix": "Verify this is not a placeholder or example value",
+      "files": [
+        "**/*.ts",
+        "**/*.js",
+        "!**/__tests__/**",
+        "!**/*.test.*",
+        "!**/*.spec.*",
+        "!**/test/**",
+        "!**/tests/**",
+        "!**/__mocks__/**",
+        "!**/fixtures/**",
+        "!**/sample/**",
+        "!**/samples/**",
+        "!**/example/**",
+        "!**/examples/**"
+      ]
+    }
+  ]
+}

package/packs/security/scan.mjs

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+// PromptWheel security guard-pack scanner — zero-dep.
+// Decanted from securitychecks' "recommended" tier (the regex-portable subset).
+// Emits the count of security-invariant violations to stdout (the metric value).
+// The full AST+dataflow engine (the other 75 recommended patterns) is the follow-up.
+//
+// Usage: node scan.mjs [path]            → prints the finding count (for `promptwheel` metrics)
+//        node scan.mjs [path] --list     → prints each finding (file:line — id — message)
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, extname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = process.argv[2] && !process.argv[2].startsWith('--') ? process.argv[2] : '.';
+const LIST = process.argv.includes('--list');
+const PACK = JSON.parse(readFileSync(join(fileURLToPath(new URL('.', import.meta.url)), 'patterns.json'), 'utf8'));
+
+const SKIP = /(^|\/)(node_modules|\.git|dist|build|\.next|coverage|vendor)(\/|$)/;
+const CODE = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']);
+const CONFIG_FILES = /(^|\/)\.npmrc$/;
+
+function* walk(dir) {
+  let entries;
+  try { entries = readdirSync(dir); } catch { return; }
+  for (const e of entries) {
+    const p = join(dir, e);
+    if (SKIP.test(p)) continue;
+    let st;
+    try { st = statSync(p); } catch { continue; }
+    if (st.isDirectory()) yield* walk(p);
+    else if (CODE.has(extname(p)) || CONFIG_FILES.test(p)) yield p;
+  }
+}
+
+const findings = [];
+for (const file of walk(ROOT)) {
+  let text;
+  try { text = readFileSync(file, 'utf8'); } catch { continue; }
+  const lines = text.split('\n');
+  for (const pat of PACK.patterns) {
+    let re;
+    try { re = new RegExp(pat.match, 'i'); } catch { continue; }
+    const excludes = (pat.exclude || []).map((x) => { try { return new RegExp(x, 'i'); } catch { return null; } }).filter(Boolean);
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      if (!re.test(line)) continue;
+      if (excludes.some((ex) => ex.test(line))) continue;       // pattern's own false-positive guards
+      findings.push({ file, line: i + 1, id: pat.id, sev: pat.severity, msg: pat.message });
+    }
+  }
+}
+
+if (LIST) {
+  for (const f of findings) console.error(`  [${f.sev}] ${f.file}:${f.line} — ${f.id} — ${f.msg}`);
+  console.error(`\n${findings.length} security-invariant violation(s)`);
+}
+// the metric value: lower is better, 0 is clean
+process.stdout.write(String(findings.length) + '\n');