promptwheel 0.2.3 → 0.4.0

package/README.md

@@ -72,8 +72,15 @@ npx promptwheel improve --attempt "claude -p 'reduce lint errors'"
 
 # what's actually responding in this repo? (aggregates .promptwheel/outcomes.jsonl)
 npx promptwheel insights
+
+# EXPERIMENTAL (Phase 5): the earned playbook + where the next attempt should go
+npx promptwheel playbook                  # decayed, evidence-gated claims distilled from the record
+npx promptwheel suggest                   # UCB over the lever scores — proven levers vs under-explored arms
+npx promptwheel backfill -n 30            # cold start: seed the record from git history (cohort-tagged, commit types → labels)
 ```
 
+**The consequence ledger.** git records *what changed*; PromptWheel records *what the change did* — same trust model (local, deterministic, append-only, no server, no LLM in the verdict). `playbook` and `suggest` are pure re-derivations over that ledger: every rendered line was measured by the gate, decays unless re-earned, and stays hidden below an evidence threshold. No compounding claim is made for them until the A/B acceptance test (`bench/compounding-ab.mjs`) passes on real usage data.
+
 **Footprint:** it never touches your working tree — every measurement runs in a throwaway git worktree **in your system temp dir** (one at a time; `node_modules` is symlinked, not copied), removed when the run finishes. The only thing PromptWheel writes to your repo is the optional `.promptwheel/outcomes.jsonl` record — commit it to build the per-repo "what moves what" history, or `.gitignore` it (`--no-record` to skip entirely). A hard-killed run can't leave clutter behind: the next run **self-heals** any orphaned worktree (stale registry entry + abandoned temp checkout).
 
 ## Loop patterns
@@ -187,7 +194,7 @@ The accumulated record of **which change-types move which metrics** is the asset
 - [docs/VISION.md](docs/VISION.md) — why we pivoted from orchestrator to outcome gate, the thesis, the moat, the open-core model.
 - [docs/ROADMAP.md](docs/ROADMAP.md) — the phased plan and the ship-now/stay-thin guardrails.
 - [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — how the engine works: schemas, extract modes, the trust/noise model.
-- [docs/LEARNING.md](docs/LEARNING.md) — the (research-gated) Phase-5 design: ACE-style playbook + UCB work-discovery.
+- [docs/LEARNING.md](docs/LEARNING.md) — the (research-gated) Phase-5 design: outcome-curated playbook (Agentic Context Engineering, Stanford 2510.04618) + UCB work-discovery.
 - [CLAUDE.md](CLAUDE.md) — the constitution for anyone (human or agent) working in this repo.
 
 ## Develop
@@ -211,6 +218,6 @@ The engine is one importable file; pure helpers are exported for unit tests, the
 - [x] `insights` — reward-stream aggregation (Phase-5 seed)
 - [x] `--detect-gaming` — reward-hack detection: re-prove the win from source edits alone + `antihack` preset
 - [x] npm publish — `promptwheel@0.1.0` (the lead magnet, shipped 2026-06)
-- [ ] ACE-style learning + UCB work-discovery (**frozen** — gated on data + ≥1 paid engagement; see [docs/LEARNING.md](docs/LEARNING.md))
+- [x] outcome-curated learning + UCB work-discovery — `playbook` + `suggest` + the compounding A/B harness (**experimental**; compounding *claims* stay gated on real-data proof — see [docs/LEARNING.md](docs/LEARNING.md))
 
 > Status: **published** (npm `promptwheel`, v0.2.0) — all core phases built. Lineage: CommandLayer → BlockSpool → PromptWheel (orchestrator, archived) → **PromptWheel (outcome gate)**.

package/bin/promptwheel.mjs

@@ -188,6 +188,18 @@ function changedSourcePaths(repo, base, head) {
   return { source: all.filter((p) => !isNonSource(p)), nonSource: all.filter(isNonSource) };
 }
 
+// coarse change-location fingerprint for the outcome record: top source dirs of the diff
+function subsystemsOf(repo, base, head) {
+  try {
+    const counts = {};
+    for (const p of changedSourcePaths(repo, base, head).source) {
+      const d = p.includes('/') ? p.slice(0, p.indexOf('/')) : '(root)';
+      counts[d] = (counts[d] || 0) + 1;
+    }
+    return Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, 3).map(([d]) => d);
+  } catch { return []; }
+}
+
 // measure `metric` in a worktree at `base` with ONLY the source slice of base→head applied.
 function measureSourceOnly(repo, base, head, metric, linkNM, repeat) {
   const { source } = changedSourcePaths(repo, base, head);
@@ -349,7 +361,15 @@ function gate(repo, opts) {
   const failed = metrics.some((m) => m.guard && !m.ok);
   const gamed = metrics.some((m) => m.gamed === true);
   const verdict = failed ? 'fail' : gamed ? 'gamed' : 'pass';
-  const report = { base: short(repo, base), head: short(repo, head), repeat, mode: opts.working ? 'working' : 'refs', verdict, metrics };
+  const report = {
+    base: short(repo, base), head: short(repo, head), repeat, mode: opts.working ? 'working' : 'refs',
+    // learning-substrate fields (Phase 5): cohort segments reliability by environment,
+    // label attributes a change-type, subsystems fingerprint WHERE the change landed.
+    cohort: opts.cohort ?? (process.env.CI ? 'ci' : 'local'),
+    ...(opts.label ? { label: opts.label } : {}),
+    subsystems: subsystemsOf(repo, base, head),
+    verdict, metrics,
+  };
   if (!opts.noRecord && cfg.record !== false) recordOutcome(repo, report);
   return report;
 }
@@ -357,7 +377,7 @@ function gate(repo, opts) {
 function run(argv) {
   const args = parseArgs(argv);
   const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
-  const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
+  const report = gate(repo, { base: args.base, head: args.head, working: args.working, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, label: args.label });
   if (args.json) console.log(JSON.stringify(report, null, 2));
   else if (args.markdown) console.log(renderMarkdown(report));
   else printHuman(report);
@@ -385,7 +405,7 @@ function improve(argv) {
   try { execSync(args.attempt, { cwd: repo, stdio: 'inherit' }); }
   catch (e) { console.error(`  (attempt exited ${e.status ?? 1} — gating whatever it changed)`); }
 
-  const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming });
+  const report = gate(repo, { working: true, repeat: args.repeat, noRecord: args.noRecord, detectGaming: args.detectGaming, label: args.label ?? args.attempt });
   const noChange = report.metrics.every((m) => m.delta === 0 || m.delta == null);
   const improvedNames = report.metrics.filter((m) => m.status === 'improved').map((m) => m.name);
 
@@ -419,7 +439,7 @@ function revert(repo) {
 }
 
 // Phase-5 seed: turn the accumulated reward stream into signal. Thin on purpose —
-// this is the substrate a future ACE playbook / UCB work-discovery loop trains on,
+// this is the substrate a future outcome-curated playbook / UCB work-discovery loop trains on,
 // NOT that loop itself. Just honest aggregation, no model.
 function insights(argv) {
   const args = parseArgs(argv);
@@ -447,6 +467,149 @@ function insights(argv) {
   console.log('  highest-lever metrics are where an agent loop should spend its attempts.\n');
 }
 
+// ---------------------------------------------------------------------------
+// Phase 5 — outcome-curated playbook + UCB work-discovery.
+// Unfrozen 2026-07-02 by explicit founder decision overriding D7 (see docs/LEARNING.md).
+// Design constraints that survive the unfreeze:
+//   - PURE VIEW: the append-only ledger (.promptwheel/outcomes.jsonl) is the only store;
+//     the playbook is re-derived on every read (decay at read time — no curator state to rot).
+//   - EVIDENCE-GATED: a key renders a claim only past MIN_MOVED weighted observations;
+//     everything below that is counted, not asserted.
+//   - CLAIM-GATED: no public compounding claim until bench/compounding-ab.mjs passes on
+//     real usage data. The gate on the CLAIM outlived the gate on the CODE.
+// ---------------------------------------------------------------------------
+const HALF_LIFE = 20;   // runs — an entry's weight halves every HALF_LIFE runs unless re-earned
+const MIN_MOVED = 3;    // weighted moved-observations before a key earns a rendered claim
+const EXPLORE_C = 1.0;  // UCB exploration constant
+
+function readLedger(repo) {
+  const f = join(repo, '.promptwheel', 'outcomes.jsonl');
+  if (!existsSync(f)) return [];
+  return readFileSync(f, 'utf8').split('\n').filter(Boolean)
+    .map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
+}
+
+// fold the ledger into decayed per-key stats. keys: `metric` · `metric @ subsystem` · `metric # label`
+function foldOutcomes(runs) {
+  const keys = new Map();
+  const N = runs.length;
+  runs.forEach((r, idx) => {
+    const w = Math.pow(0.5, (N - 1 - idx) / HALF_LIFE);   // idx N-1 = newest → weight 1
+    for (const m of (r.metrics || [])) {
+      const tags = [m.name];
+      for (const s of (r.subsystems || [])) tags.push(`${m.name} @ ${s}`);
+      if (r.label) tags.push(`${m.name} # ${r.label}`);
+      for (const key of tags) {
+        const k = keys.get(key) ?? { key, metric: m.name, w: 0, n: 0, imp: 0, reg: 0, moved: 0, effects: [], cohorts: {}, last: null };
+        k.n += 1; k.w += w;
+        const coh = (k.cohorts[r.cohort || 'unknown'] ??= { imp: 0, reg: 0 });
+        if (m.status === 'improved') {
+          k.imp += w; k.moved += w; coh.imp += 1;
+          if (typeof m.delta === 'number' && m.delta !== 0) k.effects.push(Math.abs(m.delta));
+        } else if (m.status === 'regressed') { k.reg += w; k.moved += w; coh.reg += 1; }
+        k.last = r.ts || k.last;
+        keys.set(key, k);
+      }
+    }
+  });
+  return [...keys.values()];
+}
+
+const betaMean = (imp, reg) => (1 + imp) / (2 + imp + reg);   // Beta(1,1)-smoothed helpfulness
+
+// composite lever = smoothed helpfulness × responsiveness; UCB adds an exploration bonus
+function scoreKey(k, totalW) {
+  const p = betaMean(k.imp, k.reg);
+  const move = k.w ? k.moved / k.w : 0;
+  const lever = p * move;
+  const ucb = lever + EXPLORE_C * Math.sqrt(Math.log(Math.max(Math.E, totalW)) / Math.max(1e-6, k.w));
+  return { p: +p.toFixed(3), move: +move.toFixed(3), lever: +lever.toFixed(3), ucb: +ucb.toFixed(3), effect: +(median(k.effects) ?? 0).toFixed(6) };
+}
+
+// cohorts that disagree in sign are flagged, never averaged into a lie
+function cohortNote(k) {
+  const cs = Object.entries(k.cohorts).filter(([, c]) => c.imp + c.reg > 0);
+  if (cs.length < 2) return '';
+  const signs = new Set(cs.map(([, c]) => Math.sign(c.imp - c.reg)));
+  return signs.size > 1 ? ' · ⚠ cohort-dependent (ci vs local disagree)' : '';
+}
+
+function loadEntries(repo) {
+  const runs = readLedger(repo);
+  if (!runs.length) { console.error('no outcome record yet — run the gate a few times first (.promptwheel/outcomes.jsonl)'); process.exit(2); }
+  const keys = foldOutcomes(runs);
+  const totalW = keys.filter((k) => k.key === k.metric).reduce((s, k) => s + k.w, 0);
+  const entries = keys.map((k) => ({ ...k, ...scoreKey(k, totalW), sufficient: k.moved >= MIN_MOVED }))
+    .sort((a, b) => b.lever - a.lever || b.w - a.w);
+  return { runs, entries };
+}
+
+// the distilled, execution-earned context an agent (or CLAUDE.md) can consume
+function playbook(argv) {
+  const args = parseArgs(argv);
+  const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
+  const { runs, entries } = loadEntries(repo);
+  if (args.json) { console.log(JSON.stringify({ runs: runs.length, halfLife: HALF_LIFE, minMoved: MIN_MOVED, entries }, null, 2)); return; }
+  const suff = entries.filter((e) => e.sufficient);
+  const pct = (x) => `${Math.round(x * 100)}%`;
+  const out = [];
+  out.push(`## Earned playbook — ${runs.length} gated runs, decay half-life ${HALF_LIFE} runs`);
+  out.push('*(every line below was measured by the gate, not asserted; entries decay unless re-earned)*', '');
+  if (!suff.length) {
+    out.push(`_Nothing has earned a claim yet — a key needs ≥${MIN_MOVED} weighted moved-observations. Keep gating; ${entries.length} keys are accumulating._`);
+  } else {
+    for (const e of suff.slice(0, 20)) {
+      out.push(`- **${e.key}** — helpful ${pct(e.p)} when it moves (${e.imp.toFixed(1)}✓/${e.reg.toFixed(1)}✗ weighted), responds in ${pct(e.move)} of runs, median effect ${e.effect}${cohortNote(e)}`);
+    }
+    const hidden = entries.length - suff.length;
+    if (hidden > 0) out.push('', `_${hidden} more keys below the evidence threshold — counted, not asserted._`);
+  }
+  console.log(out.join('\n'));
+}
+
+// UCB work-discovery: where should the loop spend its NEXT attempt? Proposes measured
+// targets (metric/subsystem arms) — never code advice; the measurement stays the message.
+function suggest(argv) {
+  const args = parseArgs(argv);
+  const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
+  const { runs, entries } = loadEntries(repo);
+  const arms = entries.filter((e) => e.key === e.metric || e.key.includes(' @ ')).sort((a, b) => b.ucb - a.ucb);
+  if (args.json) { console.log(JSON.stringify({ runs: runs.length, thin: runs.length < 10, arms: arms.slice(0, 10) }, null, 2)); return; }
+  console.log(`\nPromptWheel suggest — UCB over ${runs.length} gated runs${runs.length < 10 ? '  (thin record: exploration dominates — treat as a coin with opinions)' : ''}\n`);
+  for (const a of arms.slice(0, 5)) {
+    console.log(`  ${a.ucb.toFixed(3)}  ${a.key.padEnd(28)} lever ${a.lever.toFixed(3)} (helpful ${a.p}, responds ${a.move}) · evidence ${a.moved.toFixed(1)} moved / ${a.n} runs${a.sufficient ? '' : ' · below claim threshold'}`);
+  }
+  console.log('\n  score = lever + exploration bonus — high scores are either proven levers or under-explored arms.\n');
+}
+
+// backfill — seed the ledger from git history: replay past commits through the CURRENT
+// metrics (LEARNING.md harvest path 1). Deterministic, no LLM. Rows are cohort-tagged
+// 'backfill' — historical human commits are NOT live agent-loop evidence; the cohort
+// machinery segments them and flags disagreement rather than averaging it away. The
+// conventional-commit type (fix/feat/refactor/…) becomes the change-type label for free.
+function backfill(argv) {
+  const args = parseArgs(argv);
+  const repo = git(['rev-parse', '--show-toplevel'], process.cwd());
+  const listArgs = ['rev-list', '--no-merges'];
+  if (args.since) listArgs.push(`${args.since}..HEAD`); else listArgs.push('-n', String(args.n ?? 30), 'HEAD');
+  const commits = git(listArgs, repo).split('\n').filter(Boolean).reverse(); // oldest first → ledger stays chronological, decay stays honest
+  if (!commits.length) { console.error('no commits to backfill'); process.exit(2); }
+  const seen = new Set(readLedger(repo).filter((r) => r.cohort === 'backfill').map((r) => r.head));
+  let done = 0, skipped = 0;
+  console.log(`\nbackfilling ${commits.length} commits through the current metrics (gaming detection ${args.dgExplicit && args.detectGaming ? 'ON' : 'off — pass --detect-gaming to audit history too'})\n`);
+  for (const c of commits) {
+    try { git(['rev-parse', `${c}~1`], repo); } catch { skipped++; continue; }        // root commit
+    if (seen.has(short(repo, c))) { skipped++; continue; }                            // already recorded
+    const subj = git(['log', '-1', '--format=%s', c], repo);
+    const type = /^(feat|fix|docs|test|chore|refactor|perf|ci|build|style)\b/i.exec(subj)?.[1]?.toLowerCase();
+    const report = gate(repo, { base: `${c}~1`, head: c, repeat: args.repeat, detectGaming: args.dgExplicit ? args.detectGaming : false, label: type, cohort: 'backfill' });
+    console.log(`  ${report.head} ${report.verdict.toUpperCase().padEnd(6)} ${type ? `#${type}`.padEnd(10) : ''.padEnd(10)} ${subj.slice(0, 56)}`);
+    done++;
+  }
+  console.log(`\n  backfilled ${done}${skipped ? ` (skipped ${skipped}: root or already recorded)` : ''} — old commits that no longer build record as unmeasurable, never faked.`);
+  console.log('  next:  promptwheel playbook   ·   promptwheel suggest\n');
+}
+
 // ---------------------------------------------------------------------------
 // init — write a starter config so a newcomer isn't staring at a blank page
 // ---------------------------------------------------------------------------
@@ -610,24 +773,30 @@ function parseArgs(argv) {
     else if (argv[i] === '--repeat') a.repeat = parseInt(argv[++i], 10);
     else if (argv[i] === '--working') a.working = true;
     else if (argv[i] === '--no-record') a.noRecord = true;
-    else if (argv[i] === '--detect-gaming' || argv[i] === '--antihack') a.detectGaming = true;
-    else if (argv[i] === '--no-detect-gaming' || argv[i] === '--no-antihack') a.detectGaming = false;
+    else if (argv[i] === '--detect-gaming' || argv[i] === '--antihack') { a.detectGaming = true; a.dgExplicit = true; }
+    else if (argv[i] === '--no-detect-gaming' || argv[i] === '--no-antihack') { a.detectGaming = false; a.dgExplicit = true; }
+    else if (argv[i] === '--since') a.since = argv[++i];
+    else if (argv[i] === '-n' || argv[i] === '--limit') a.n = parseInt(argv[++i], 10);
     else if (argv[i] === '--attempt') a.attempt = argv[++i];
+    else if (argv[i] === '--label') a.label = argv[++i];
     else if (argv[i] === '--json') a.json = true;
-    else if (argv[i] === '--markdown') a.markdown = true;
+    else if (argv[i] === '--markdown' || argv[i] === '--md') a.markdown = true;
   }
   return a;
 }
 
 // pure, side-effect-free helpers are exported for unit testing; the CLI below only
 // runs when this file is executed directly (not when imported by the test suite).
-export { extract, evaluate, median, spread, renderMarkdown, isNonSource, gain, judgeGaming };
+export { extract, evaluate, median, spread, renderMarkdown, isNonSource, gain, judgeGaming, foldOutcomes, betaMean, scoreKey };
 
 function main() {
   const [cmd, ...rest] = process.argv.slice(2);
   if (cmd === 'run') run(rest);
   else if (cmd === 'improve') improve(rest);
   else if (cmd === 'insights') insights(rest);
+  else if (cmd === 'playbook') playbook(rest);
+  else if (cmd === 'suggest') suggest(rest);
+  else if (cmd === 'backfill') backfill(rest);
   else if (cmd === 'init') init(rest);
   else if (cmd === 'guards') guards(rest);
   else {
@@ -641,7 +810,10 @@ function main() {
       '  promptwheel run --working                    gate uncommitted changes (incl. newly added files)',
       '  promptwheel improve --attempt "<cmd>"        run an agent/script; keep only if a metric improved',
       '                                               exit 0=kept · 1=regression · 3=plateau · add --json',
-      '  promptwheel insights                         which metrics actually respond (loop memory)',
+      '  promptwheel insights                         which metrics actually respond (raw counts)',
+      '  promptwheel playbook [--json]                the earned playbook: decayed, evidence-gated claims distilled from the record',
+      '  promptwheel suggest [--json]                 UCB work-discovery: where the next attempt should go (experimental)',
+      '  promptwheel backfill [-n N | --since <ref>]  seed the ledger from git history (cohort-tagged; commit types become labels)',
       '  promptwheel guards                           show the effective guardrails (incl. inherited) + flag record',
       '',
       'Loop it:  while promptwheel improve --attempt "$AGENT"; do :; done   # stops on plateau/regression',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "promptwheel",
-  "version": "0.2.3",
+  "version": "0.4.0",
   "description": "Catch your agent cheating — a deterministic, zero-LLM CLI that flags when an AI coding agent gamed its own metric (re-proves the win from the agent's source edits alone). Built on an outcome gate; runs in CI or a Claude Code hook.",
   "type": "module",
   "bin": {