ultracost 0.2.0 → 0.3.0

package/src/rules.js

@@ -59,19 +59,31 @@ e.g. \`agent(task, { model: 'sonnet', effort: 'low' })\` for a mechanical scan.
 
 Before launching a dynamic workflow:
 1. Draft the workflow script with per-stage \`model\` and \`effort\` set.
-2. Write the draft to a temp file and run \`ultracost estimate <file>\` to get the
-   agent count, model mix, and cost versus an all-\`${policy.tiers[policy.default].model}\` baseline.
+2. Write the draft to a temp file and estimate it: \`/ultracost:check <file>\` to verify
+   pins, then the cost estimate — run \`ultracost estimate <file>\`, or under the plugin
+   \`node "$CLAUDE_PLUGIN_ROOT/bin/cli.js" estimate <file>\` (no global \`ultracost\` bin
+   is required). It reports the agent count, model mix, and cost versus an
+   all-\`${policy.tiers[policy.default].model}\` baseline.
 3. Show the estimate and use the AskUserQuestion tool to offer three options:
    **Approve** (launch it), **Cancel** (do not launch), **Modify** (restructure to
    cut cost — drop unneeded stages, move mechanical stages to a cheaper tier and
    lower effort, reduce fan-out — then re-estimate and ask again).
-4. Launch the workflow only after Approve.
+4. Launch the workflow only after Approve. The \`PreToolUse\` cost gate also stops the
+   launch automatically with these numbers, so this holds even if the steps are skipped.
 
-Verify any script with \`/ultracost:check\` or \`ultracost check <script>\` — it flags
-stages missing a model pin.
+Verify any script with \`/ultracost:check\` (the plugin command) or \`ultracost check
+<script>\` on the CLI — it flags stages missing a model pin, a pin that mismatches the
+work the prompt describes, and effort over the model's cap.
 ${MARKER_END}`;
 }
 
+// The routing block without the HTML markers — the single source for the SessionStart
+// hook injection (reinject.mjs) and the routing skill (skills/ultracost/SKILL.md), so
+// neither can drift from policy.json.
+export function routingGuidance(policy) {
+  return compileRules(policy).split('\n').slice(1, -1).join('\n').trim();
+}
+
 export function replaceBlock(content, block) {
   const re = new RegExp(`${MARKER_START}[\\s\\S]*?${MARKER_END}`);
   if (!re.test(content)) return null;

package/src/transcript.js

@@ -0,0 +1,186 @@
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, sep, basename } from 'node:path';
+import { homedir } from 'node:os';
+import { sumUsage } from './cost.js';
+
+// Read Claude Code's local session transcripts (offline) so ultracost can reconcile
+// its estimate against real token usage and learn from it. Clean-room reimplementation
+// of the well-known parse+dedup contract: assistant lines carry message.usage; the
+// same message can recur across files (resumed sessions, sidechain replays) so we dedup
+// on message.id + requestId; dynamic-workflow agent() stages live in their own
+// subagents/workflows/wf_<id>/agent-<aid>.jsonl files next to a journal.jsonl.
+
+const expandTilde = (p) => (p === '~' || p.startsWith('~/') ? join(homedir(), p.slice(1)) : p);
+
+// All Claude Code `projects/` directories: CLAUDE_CONFIG_DIR (comma-separated, each
+// entry a config dir OR a projects dir), else ~/.config/claude and ~/.claude.
+export function projectsDirs(env = process.env) {
+  const out = [];
+  const add = (dir) => {
+    if (existsSync(join(dir, 'projects'))) out.push(join(dir, 'projects'));
+    else if (basename(dir) === 'projects' && existsSync(dir)) out.push(dir);
+  };
+  if (env.CLAUDE_CONFIG_DIR) {
+    env.CLAUDE_CONFIG_DIR.split(',').map((s) => s.trim()).filter(Boolean).forEach((p) => add(expandTilde(p)));
+  } else {
+    add(env.XDG_CONFIG_HOME ? join(env.XDG_CONFIG_HOME, 'claude') : join(homedir(), '.config', 'claude'));
+    add(join(homedir(), '.claude'));
+  }
+  return [...new Set(out)];
+}
+
+function walk(dir, test, out = []) {
+  let names;
+  try { names = readdirSync(dir); } catch { return out; }
+  for (const name of names) {
+    if (name.startsWith('.')) continue;
+    const full = join(dir, name);
+    let st;
+    try { st = statSync(full); } catch { continue; }
+    if (st.isDirectory()) walk(full, test, out);
+    else if (test(full)) out.push(full);
+  }
+  return out;
+}
+
+// One transcript line -> a normalized usage record, or null if it isn't an assistant
+// message that reports usage.
+export function parseUsageLine(line) {
+  let obj;
+  try { obj = JSON.parse(line); } catch { return null; }
+  if (!obj || obj.isApiErrorMessage) return null;
+  const m = obj.message;
+  if (!m || !m.usage) return null;
+  if (m.role && m.role !== 'assistant' && obj.type !== 'assistant') return null;
+  return {
+    id: m.id || null,
+    requestId: obj.requestId || null,
+    model: m.model || null,
+    usage: m.usage,
+    ts: obj.timestamp || null,
+    isSidechain: !!obj.isSidechain
+  };
+}
+
+function readUsage(file) {
+  let text;
+  try { text = readFileSync(file, 'utf8'); } catch { return []; }
+  const out = [];
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    const e = parseUsageLine(line);
+    if (e) out.push(e);
+  }
+  return out;
+}
+
+// Dedup on message.id + requestId; lines without an id are always kept; on a collision
+// keep the copy with the most output tokens (a sidechain/replay tie-break).
+export function dedupe(entries) {
+  const seen = new Map();
+  const kept = [];
+  for (const e of entries) {
+    if (!e.id) { kept.push(e); continue; }
+    const key = `${e.id}:${e.requestId || ''}`;
+    const idx = seen.get(key);
+    if (idx === undefined) { seen.set(key, kept.length); kept.push(e); }
+    else if ((e.usage.output_tokens || 0) > (kept[idx].usage.output_tokens || 0)) kept[idx] = e;
+  }
+  return kept;
+}
+
+// Classify a transcript file by its path: 'main', 'subagent', or 'workflow-stage'
+// (the ultracode dynamic-workflow agent() stage). Separation is by PATH, never by
+// sessionId (subagent files inherit the parent's sessionId).
+export function classifyTranscriptFile(file, projectsDir) {
+  const rel = projectsDir && file.startsWith(projectsDir) ? file.slice(projectsDir.length + 1) : file;
+  const parts = rel.split(sep);
+  const project = parts[0];
+  const sub = parts.indexOf('subagents');
+  if (sub !== -1) {
+    const parentSessionId = parts[sub - 1];
+    const agentId = basename(file, '.jsonl').replace(/^agent-/, '');
+    if (parts[sub + 1] === 'workflows' && (parts[sub + 2] || '').startsWith('wf_')) {
+      return { kind: 'workflow-stage', project, parentSessionId, wfId: parts[sub + 2], agentId, file };
+    }
+    return { kind: 'subagent', project, parentSessionId, agentId, file };
+  }
+  return { kind: 'main', project, sessionId: basename(file, '.jsonl'), file };
+}
+
+// All usage records across every transcript, classified and globally deduped.
+export function readTranscripts({ env = process.env, root = null } = {}) {
+  const dirs = root ? [root] : projectsDirs(env);
+  const all = [];
+  for (const dir of dirs) {
+    for (const file of walk(dir, (f) => f.endsWith('.jsonl'))) {
+      const cls = classifyTranscriptFile(file, dir);
+      for (const e of readUsage(file)) all.push({ ...e, ...cls });
+    }
+  }
+  return dedupe(all);
+}
+
+function readJournal(file) {
+  const map = {};
+  if (!existsSync(file)) return map;
+  let text;
+  try { text = readFileSync(file, 'utf8'); } catch { return map; }
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    let j;
+    try { j = JSON.parse(line); } catch { continue; }
+    if (j && j.agentId && (j.key || !(j.agentId in map))) map[j.agentId] = j.key || map[j.agentId] || null;
+  }
+  return map;
+}
+
+// Every dynamic-workflow run on disk, newest first, with per-stage token sums. This is
+// what `reconcile` / the savings ledger compare against the estimate.
+export function locateWorkflowRuns({ env = process.env, root = null } = {}) {
+  const dirs = root ? [root] : projectsDirs(env);
+  const runs = [];
+  for (const dir of dirs) {
+    const wfDirs = new Set();
+    walk(dir, (f) => {
+      const p = f.split(sep);
+      const sub = p.indexOf('subagents');
+      if (sub !== -1 && p[sub + 1] === 'workflows' && (p[sub + 2] || '').startsWith('wf_')) {
+        wfDirs.add(p.slice(0, sub + 3).join(sep));
+      }
+      return false;
+    });
+    for (const wfDir of wfDirs) {
+      let names;
+      try { names = readdirSync(wfDir); } catch { continue; }
+      const journal = readJournal(join(wfDir, 'journal.jsonl'));
+      const stages = names
+        .filter((f) => /^agent-.*\.jsonl$/.test(f))
+        .map((f) => {
+          const agentId = f.slice('agent-'.length, -'.jsonl'.length);
+          const entries = dedupe(readUsage(join(wfDir, f)));
+          return {
+            agentId,
+            stageKey: journal[agentId] || null,
+            model: entries.length ? entries[entries.length - 1].model : null,
+            usage: sumUsage(entries.map((e) => e.usage)),
+            lines: entries.length
+          };
+        })
+        .filter((s) => s.lines > 0);
+      if (!stages.length) continue;
+      const parts = wfDir.split(sep);
+      let mtime = 0;
+      try { mtime = statSync(wfDir).mtimeMs; } catch { /* ignore */ }
+      runs.push({
+        wfId: basename(wfDir),
+        dir: wfDir,
+        project: parts[parts.indexOf('projects') + 1],
+        parentSessionId: parts[parts.indexOf('subagents') - 1],
+        stages,
+        mtime
+      });
+    }
+  }
+  return runs.sort((a, b) => b.mtime - a.mtime);
+}

package/templates/hooks/reinject.mjs

@@ -1,21 +1,15 @@
 #!/usr/bin/env node
 // ultracost SessionStart hook. Injects the model-routing policy as context at the
 // start of every session (and after compaction), so workflow authoring sees it
-// without relying on the model choosing to open a skill. Pure node, reads the hook
-// JSON from stdin, emits SessionStart additionalContext. No bash or jq dependency.
+// without relying on the model choosing to open a skill.
+//
+// The injected text is COMPILED from the active policy via src/rules.js — the single
+// source of truth. It is no longer a hand-maintained copy, so it cannot drift from
+// policy.json (or from the CLAUDE.md block and the routing skill). Pure node, reads
+// the hook JSON from stdin, emits SessionStart additionalContext. No npm dependency.
 
-const POLICY = `This project follows the ultracost model-routing and cost policy for Claude Code dynamic workflows (ultracode).
-
-Per-stage model: every agent() stage sets an explicit \`model\` in its options rather than inheriting the session model; haiku is not used.
-- opus for coding and reasoning: writing/editing/refactoring/deleting code; debugging; designing APIs, schemas, or architecture; non-trivial tests; code review; security and performance analysis; planning; synthesis. The orchestrator/planner and the final consolidation stage are always opus.
-- sonnet for pre-planned mechanical and support work: applying an already-decided edit; search, grep, and file discovery; collecting/listing/extracting; running tests and reporting; gathering or summarizing context for an opus stage.
-When a stage is ambiguous, opus is the default.
-
-Per-stage effort: also set \`effort\` per stage, choosing the lowest level that fits, bounded by model (sonnet up to high, opus up to xhigh): low = trivial deterministic work (listing/globbing, simple extraction, formatting); medium = light judgment on a small surface; high = standard coding/analysis; xhigh = hard cross-file reasoning, adversarial review, planning, final synthesis.
-
-Pre-flight cost gate: before launching a workflow, draft the script with per-stage model and effort, write it to a temp file, run \`ultracost estimate <file>\` to get the agent count, model mix, and cost vs an all-opus baseline, then use the AskUserQuestion tool to offer three options — Approve (launch), Cancel (do not launch), or Modify (restructure to cut cost: drop unneeded stages, move mechanical stages to sonnet and lower effort, reduce fan-out; then re-estimate and ask again). Launch only after Approve.
-
-Verify scripts with the /ultracost:check command or \`ultracost check <script>\`, which flags any agent() stage missing a model.`;
+import { loadPolicy } from '../../src/policy.js';
+import { routingGuidance } from '../../src/rules.js';
 
 async function readStdin() {
   if (process.stdin.isTTY) return '';
@@ -25,17 +19,26 @@ async function readStdin() {
   return data;
 }
 
-// Only wired to SessionStart (all sources), so emit the policy unconditionally.
 // Parsing stdin is best-effort; a missing/invalid payload still injects the policy.
+try { await readStdin(); } catch {}
+
+let context;
 try {
-  await readStdin();
-} catch {}
+  const { policy } = loadPolicy();
+  context = routingGuidance(policy);
+} catch {
+  // Fail open with a minimal reminder rather than injecting nothing.
+  context =
+    'ultracost: route every agent() stage explicitly — pin a per-stage model (opus for ' +
+    'coding/reasoning, sonnet for pre-planned mechanical and search work; never haiku) and ' +
+    'an effort level. Verify with /ultracost:check before launching a dynamic workflow.';
+}
 
 process.stdout.write(
   JSON.stringify({
     hookSpecificOutput: {
       hookEventName: 'SessionStart',
-      additionalContext: POLICY
+      additionalContext: context
     }
   })
 );

package/templates/hooks/workflow-gate.mjs

@@ -2,44 +2,39 @@
 // ultracost deterministic cost gate — ON BY DEFAULT (PreToolUse, matcher "Workflow").
 // The plugin registers this in hooks/hooks.json so EVERY dynamic-workflow launch
 // pauses before it runs — it does not depend on the model choosing to ask. It reads
-// the drafted script from tool_input.script, runs the static guard + cost estimate,
-// and returns a permission decision with the numbers AND any unpinned-stage warning
-// up front, so an accidental all-Opus fan-out can't slip through.
+// the drafted script from tool_input.script, runs the static guard + cost estimate
+// (calibrated from your real usage when available), enforces the policy budget caps,
+// and returns a permission decision with an aligned mini cost table up front, so an
+// accidental all-Opus fan-out (or an over-budget launch) can't slip through.
 //
 // A PreToolUse hook runs in EVERY permission mode (bypass only auto-approves the
 // "ask" path; a "deny" is honored regardless of mode). So the gate is mode-aware:
-// it reads `permission_mode` from the event and hard-denies a problem workflow in
-// the modes where an "ask" can't pause.
+// it hard-denies a problem workflow in the modes where an "ask" can't pause.
 //
 // Modes (env ULTRACOST_GATE):
-//   (unset)  mode-aware default. Clean (all pinned) -> ask + estimate, every mode.
-//            Problem (unpinned/banned/inherit) -> ask + ⚠ warning in default /
-//            acceptEdits / auto (an ask surfaces there); DENY in bypassPermissions /
-//            dontAsk (an ask is auto-approved/won't pause there, so we block instead).
-//   strict   deny on ANY problem, in every mode; ask (with estimate) when all pinned.
-//   ask      never escalate to deny — always ask (opt out of the mode-aware deny).
-//   off      disable entirely — for non-interactive runs (headless `claude -p`,
-//            Auto Mode, CI), where an unanswered "ask" is denied (the gate fails closed).
+//   (unset)  mode-aware default. Clean (all pinned, within budget) -> ask + estimate,
+//            every mode. Problem (unpinned/banned/inherit) -> ask + warning in default
+//            /acceptEdits/auto; DENY in bypassPermissions/dontAsk. Budget exceeded ->
+//            DENY in every mode (a hard cap).
+//   strict   deny on ANY problem, in every mode; ask (with estimate) when all clean.
+//   ask      never escalate to deny — always ask (opts out of budget + mode denies).
+//   off      disable entirely (headless `claude -p`, Auto Mode, CI).
 //
 // Residual limitation: Claude Code currently skips PreToolUse hooks for subagents
-// dispatched under bypassPermissions (anthropics/claude-code#43772), so a nested
-// agent there can evade the gate. The top-level Workflow launch is still gated.
+// dispatched under bypassPermissions (anthropics/claude-code#43772).
 
 import { loadPolicy } from '../../src/policy.js';
 import { estimateText } from '../../src/estimate.js';
 import { analyze, CODES } from '../../src/guard.js';
+import { applyCalibration, spentToday } from '../../src/loop.js';
 
 const money = (x) => '$' + Number(x).toFixed(4);
 const MODE = process.env.ULTRACOST_GATE;
-// Modes where an "ask" decision won't actually pause the user, so a problem
-// workflow must be denied instead to be enforced.
 const ESCALATE_MODES = new Set(['bypassPermissions', 'dontAsk']);
 
-// `systemMessage` is the documented channel for surfacing text to the USER from a
-// hook (hooks have no TTY). We send it alongside permissionDecisionReason because
-// Claude Code does NOT render the reason for an "ask" decision in the TUI
-// (anthropics/claude-code#24059) — without systemMessage the estimate would be
-// computed but invisible. For "deny" the reason renders too; we set both regardless.
+// systemMessage is the documented channel for surfacing text to the USER from a hook
+// (hooks have no TTY); Claude Code does NOT render permissionDecisionReason for an
+// "ask" (anthropics/claude-code#24059), so we send both.
 function decide(decision, message) {
   process.stdout.write(JSON.stringify({
     systemMessage: message,
@@ -62,65 +57,76 @@ async function readStdin() {
   return d;
 }
 
-// Explicit opt-out for automation / headless / CI.
 if (MODE === 'off') process.exit(0);
 
 let evt = {};
 try {
   evt = JSON.parse(await readStdin());
 } catch {
-  process.exit(0); // can't parse the event -> stay out of the way
+  process.exit(0);
 }
 
-// Only govern the Workflow tool; every other tool passes untouched.
 if (evt?.tool_name !== 'Workflow') process.exit(0);
-
 const permMode = evt?.permission_mode;
 
-// A workflow IS launching: always pause. Show numbers when the script is readable.
 const script = evt?.tool_input?.script;
 if (typeof script !== 'string') {
   ask('ultracost cost gate: a dynamic workflow is about to launch, but its script could not be read to estimate cost. Approve to launch, or deny and review.');
 }
 
+// An aligned, multi-line cost table — far more scannable than one dense line.
+function costTable(e) {
+  const a = e.agents;
+  const agents = a.fanoutGroups
+    ? `~${a.assumedTotal} (${a.known} fixed + ${a.fanoutGroups} fan-out x ~${a.assumedPerFanout})`
+    : `${a.known}`;
+  const mix = Object.entries(e.modelMix).map(([k, v]) => `${v}x ${k}`).join(', ') || 'none';
+  return [
+    `  agents     ${agents}`,
+    `  model mix  ${mix}`,
+    `  tiered     ${money(e.cost.tiered)}   vs all-${e.assumptions.sessionModel} ${money(e.cost.baseline)}   (save ${money(e.cost.savings)}, ${e.cost.savingsPct}%)`
+  ].join('\n');
+}
+
 try {
   const { policy } = loadPolicy();
-  const e = estimateText(script, policy);
+  const e = estimateText(script, applyCalibration(policy));
   const { stages, findings } = analyze(script, policy);
 
   const unpinned = findings.filter((f) => f.code === CODES.NOOPTS || f.code === CODES.MISSING).length;
   const banned = findings.filter((f) => f.code === CODES.BANNED).length;
   const inherit = findings.filter((f) => f.code === CODES.INHERIT).length;
+  const table = costTable(e);
 
-  const a = e.agents;
-  const agents = a.fanoutGroups
-    ? `~${a.assumedTotal} (${a.known} fixed + ${a.fanoutGroups} fan-out x ~${a.assumedPerFanout})`
-    : `${a.known}`;
-  const mix = Object.entries(e.modelMix).map(([k, v]) => `${v}x ${k}`).join(', ') || 'none';
-  const estLine =
-    `${agents} agents; model mix ${mix}; ` +
-    `est. ${money(e.cost.tiered)} vs all-${e.assumptions.sessionModel} baseline ${money(e.cost.baseline)} ` +
-    `(save ${money(e.cost.savings)}, ${e.cost.savingsPct}%).`;
+  // 1) Budget caps — a hard pre-flight stop in every mode (unless =ask opts out).
+  const budget = policy.budget || {};
+  const today = spentToday();
+  const overRun = budget.perRun != null && e.cost.tiered > budget.perRun;
+  const overDay = budget.perDay != null && today + e.cost.tiered > budget.perDay;
+  if ((overRun || overDay) && MODE !== 'ask') {
+    const why = overRun
+      ? `est. ${money(e.cost.tiered)} exceeds budget.perRun ${money(budget.perRun)}`
+      : `today's spend ${money(today)} + est. ${money(e.cost.tiered)} exceeds budget.perDay ${money(budget.perDay)}`;
+    deny(`\u26a0 ultracost budget: ${why}.\nultracost estimate:\n${table}\nReduce the workflow (cheaper tiers, fewer stages, less fan-out) and relaunch, or raise the cap in policy.json.`);
+  }
 
+  // 2) Pinning problems.
   const problems = [];
   if (unpinned) problems.push(`${unpinned}/${stages} stage(s) NOT pinned -> will inherit ${e.assumptions.sessionModel}`);
   if (banned) problems.push(`${banned} stage(s) pin a banned model`);
   if (inherit) problems.push(`${inherit} stage(s) use model:'inherit'`);
 
   if (problems.length) {
-    const head = `\u26a0 ultracost: ${problems.join('; ')}. `;
-    // Hard-deny when forced (strict) or when the current mode wouldn't surface an
-    // ask anyway (bypassPermissions/dontAsk). ULTRACOST_GATE=ask opts out of the
-    // mode-aware escalation and always asks.
+    const head = `\u26a0 ultracost: ${problems.join('; ')}.`;
     const hard = MODE === 'strict' || (MODE !== 'ask' && ESCALATE_MODES.has(permMode));
     if (hard) {
-      deny(`${head}estimate: ${estLine} Pin every stage (opus for reasoning, sonnet for mechanical work) and relaunch.`);
+      deny(`${head}\nultracost estimate:\n${table}\nPin every stage (opus for reasoning, sonnet for mechanical work) and relaunch.`);
     }
-    ask(`${head}estimate: ${estLine} Deny and ask me to pin every stage, or approve to run as-is.`);
+    ask(`${head}\nultracost estimate:\n${table}\nDeny and ask me to pin every stage, or approve to run as-is.`);
   }
 
-  ask(`ultracost estimate: ${estLine} Approve to launch, or deny and ask me to make it cheaper.`);
+  // 3) Clean.
+  ask(`ultracost estimate:\n${table}\nApprove to launch, or deny and ask me to make it cheaper.`);
 } catch {
-  // Estimator/policy failure must not silently let an unpriced fan-out through.
   ask('ultracost cost gate: a dynamic workflow is about to launch (cost estimate unavailable). Approve to launch, or deny and review.');
 }

package/templates/policy.default.json

@@ -1,5 +1,5 @@
 {
-  "version": 1,
+  "version": 2,
   "neverUse": ["haiku"],
   "allowInherit": false,
   "default": "opus",
@@ -32,6 +32,18 @@
     },
     "maxByModel": { "sonnet": "high", "opus": "xhigh" }
   },
+  "classify": {
+    "_note": "Extra keyword signals (merged with the built-in rubric) for the UC006 wrong-tier check and `ultracost explain`. The opening imperative verb of a prompt is weighted most.",
+    "keywords": {
+      "opus": ["architecture", "threat-model", "migrate", "redesign", "tradeoff"],
+      "sonnet": ["lint", "stub", "boilerplate", "transcribe", "tally"]
+    }
+  },
+  "budget": {
+    "_note": "Pre-flight caps enforced by the cost gate. null = no cap. perRun is per workflow launch; perDay sums the savings-ledger spend for the current day.",
+    "perRun": null,
+    "perDay": null
+  },
   "pricing": {
     "_unit": "USD per million tokens",
     "_source": "https://platform.claude.com/docs/en/about-claude/pricing.md",
@@ -44,6 +56,7 @@
   "estimation": {
     "tokensPerStage": { "input": 2000, "output": 1200 },
     "effortOutputMultiplier": { "low": 0.4, "medium": 1, "high": 1.8, "xhigh": 3, "max": 4 },
-    "assumedFanout": 5
+    "assumedFanout": 5,
+    "cacheMultipliers": { "cacheRead": 0.1, "cacheWrite": 1.25 }
   }
 }