pi-warden 0.5.2 → 0.5.4

package/README.md

@@ -96,6 +96,8 @@ Written code gets a `security_risk` question in the existing action request: har
 
 For a single text block of at least `context.tailMinChars`, the same output request asks Jev to choose `all`, `errors_and_summary`, or `summary_only`. The full result stays when P(keep all) exceeds `1 - context.confidence` (default: keep when Jev gives `all` more than 0.2), when the probability is missing, or on any request error. Code—not Jev—keeps bounded diagnostic/head/tail excerpts. An exact copy is written to an owner-only temporary directory before replacement; the excerpt links to it. If storage fails, the original stays. Multiple text blocks keep their layout and are not compressed. Trace and widget show the actual bytes saved. No extra steer is persisted for compression alone.
 
+**Measuring it.** `/warden status` keeps a session ledger so the saver's value is a number, not a claim: how many outputs were large enough to consider, how many Jev compressed, the bytes removed (≈ tokens at 4 bytes each), the *token-turns spared* (removed tokens × every later turn, which is what a prompt cache would otherwise have re-read), and **recalls** — how often the agent went back and read the stored full output. A recall means the excerpt was not enough; a recall rate above about 10% says `context.confidence` is too low for your work, near 0% says it could go lower. Every compression and recall also appears in the trace with the running ledger. The quality effect (less context rot) does not show in a counter; compare a scripted task with `context.enabled` on and off if you want that number.
+
 This MVP is always tail-only: set `context.enabled: false` to stop compression. It never prunes a warm prefix, changes the system prompt, or edits historical session entries. Full-output files can contain secrets; they remain in the OS temporary directory until you or the OS remove them. Excerpts can omit relevant evidence; read the linked file before making a decision that needs it.
 
 ### Task continuity

package/dist/done.d.ts

@@ -5,7 +5,12 @@ export type ToolOutcome = "read" | "mutation" | "check-pass" | "check-fail" | "u
  * What a finished tool call contributes to the run's evidence. Only write/edit count as code changes: shell side effects
  * (deleting a temp dir, installing a package) are too varied to demand a test run for. Custom tools are unknown.
  */
-export declare function classifyToolResult(tool: string, input: Record<string, unknown>, failed: boolean): ToolOutcome;
+export declare function classifyToolResult(tool: string, input: Record<string, unknown>, failed: boolean, output?: string): ToolOutcome;
+/**
+ * Recognise a test/type-check runner's own summary in tool output: node:test, jest/vitest, pytest, cargo, go test,
+ * tsc. Returns the outcome the summary reports, or undefined when no runner summary is present.
+ */
+export declare function checkSummary(output: string): "pass" | "fail" | undefined;
 export interface RunEvidence {
     mutations: number;
     checks: Array<{

package/dist/done.js

@@ -9,7 +9,7 @@ const CHECK_COMMAND = /\b(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|check|lin
  * What a finished tool call contributes to the run's evidence. Only write/edit count as code changes: shell side effects
  * (deleting a temp dir, installing a package) are too varied to demand a test run for. Custom tools are unknown.
  */
-export function classifyToolResult(tool, input, failed) {
+export function classifyToolResult(tool, input, failed, output) {
     if (tool === "write" || tool === "edit")
         return "mutation";
     if (tool === "read" || tool === "grep" || tool === "find" || tool === "ls")
@@ -19,8 +19,35 @@ export function classifyToolResult(tool, input, failed) {
         return "unknown";
     if (CHECK_COMMAND.test(view.command))
         return failed ? "check-fail" : "check-pass";
+    // A test runner launched from inside a script (ctx_execute JavaScript, a Python wrapper) leaves no runner name in the
+    // command text, but its output still carries the runner's summary. Judge that summary instead.
+    const summary = output === undefined ? undefined : checkSummary(output);
+    if (summary)
+        return summary === "fail" || failed ? "check-fail" : "check-pass";
     return view.shell && isReadOnlyCommand(view.command) ? "read" : "unknown";
 }
+/**
+ * Recognise a test/type-check runner's own summary in tool output: node:test, jest/vitest, pytest, cargo, go test,
+ * tsc. Returns the outcome the summary reports, or undefined when no runner summary is present.
+ */
+export function checkSummary(output) {
+    const tail = output.slice(-6000);
+    const nodeTest = /\u2139 (?:tests|pass|fail) \d+/.test(tail) && /\u2139 fail (\d+)/.exec(tail);
+    if (nodeTest)
+        return Number(nodeTest[1]) > 0 ? "fail" : "pass";
+    const jest = /^Tests:\s+(?:(\d+) failed, )?.*?\d+ total/m.exec(tail);
+    if (jest)
+        return jest[1] && Number(jest[1]) > 0 ? "fail" : "pass";
+    const pytest = /^=+ .*?(?:(\d+) failed|(\d+) error).*?in [\d.]+s/m.exec(tail) ?? /^=+ (\d+) passed.*? in [\d.]+s =+$/m.exec(tail);
+    if (pytest)
+        return /\d+ (?:failed|error)/.test(pytest[0]) ? "fail" : "pass";
+    const cargoOrGo = /^test result: (ok|FAILED)\./m.exec(tail) ?? /^(ok|FAIL)\s+\S+\s+[\d.]+s$/m.exec(tail);
+    if (cargoOrGo)
+        return cargoOrGo[1] === "ok" ? "pass" : "fail";
+    if (/\berror TS\d{4}:/.test(tail))
+        return "fail";
+    return undefined;
+}
 export function emptyEvidence() {
     return { mutations: 0, checks: [] };
 }

package/dist/extension.js

@@ -11,6 +11,7 @@ import { redact } from "./redact.js";
 import { AttemptWindow, evaluateStuck, formatStuck, makeAttempt, resultFailed, stuckNudge } from "./stuck.js";
 import { openTracePanel } from "./panel.js";
 import { completeConfig, shapeWarning } from "./shape.js";
+import { ContextLedger, formatLedger } from "./saver.js";
 import { actionDetails, doneDetails, proseDetails, stuckDetails, Trace } from "./trace.js";
 import { proseTokens, renderTemplate, TOKEN_NAMES } from "./widget.js";
 export const disclosure = "With TypeSafe judgments enabled, pi-warden sends to api.typesafe.ai: your latest request and up to eight redacted prior user/assistant text messages for task context, plus a redacted, truncated summary of each guarded bash, write, or edit call before it runs; the last few tool calls and output tails when the agent keeps failing; the agent's final message when it reports completion without running checks; and redacted tool-output samples for security and tail compression. Compression stores an exact, owner-only copy in a temporary file on this machine. Requests may incur charges. Secret redaction is best-effort. Results are model judgments, not proof or authorization; offline pattern checks stay active either way.";
@@ -75,7 +76,6 @@ export function confirmMessage(verdict) {
     lines.push("Judgments are model output, not authorization. Yes runs the tool; No blocks it and tells the agent.");
     return lines.join("\n");
 }
-const callKey = (tool, input) => `${tool}\0${JSON.stringify(input)}`;
 const SLOP_FIXES = {
     stub: "replace stubs, placeholders, and hard-coded fake data with the working implementation, or state in your reply exactly what is left unimplemented and why",
     comments: "delete comments that restate the code; keep only those that explain intent, constraints, or non-obvious behaviour",
@@ -98,13 +98,19 @@ export default function wardenExtension(pi) {
     const trace = new Trace();
     let panel;
     let lastUi;
-    /** Calls held in steer mode, with the user prompt current at that time; a retry after the user replies asks Jev about approval. */
-    const held = new Map();
+    /**
+     * Steer-mode hold state. After a hold, the next guarded call that runs under a *new* user prompt asks Jev whether that
+     * prompt approves it. The retry rarely repeats the held string byte for byte (a `command -v` dropped, a different
+     * timeout), so approval is judged against the action itself, not matched against the earlier command text.
+     */
+    let lastHoldPrompt;
+    let holdPending = false;
     let attempts = new AttemptWindow(defaultConfig().stuck.window);
     let evidence = emptyEvidence();
     let doneNudged = false;
     const prose = new ProseTrend();
     const slopCounts = { stub: 0, comments: 0, dead: 0, hedging: 0 };
+    const ledger = new ContextLedger();
     // A partially updated module graph can hand this build a config without the sections it expects; see shape.ts.
     let shapeReported = false;
     const configFor = (ctx) => {
@@ -177,11 +183,13 @@ export default function wardenExtension(pi) {
         widget.clear();
         trace.clear();
         panel?.close();
-        held.clear();
+        lastHoldPrompt = undefined;
+        holdPending = false;
         attempts.reset();
         evidence = emptyEvidence();
         doneNudged = false;
         prose.reset();
+        ledger.reset();
         for (const symptom of Object.keys(slopCounts))
             slopCounts[symptom] = 0;
         if (ctx.hasUI)
@@ -196,15 +204,24 @@ export default function wardenExtension(pi) {
     pi.on("agent_start", async () => {
         evidence = emptyEvidence();
     });
+    // Every turn that runs after a compression is a turn that did not carry the removed text.
+    pi.on("turn_end", async () => {
+        ledger.turnEnd();
+    });
     pi.on("tool_call", async (event, ctx) => {
         const config = configFor(ctx);
-        if (!config.enabled || !config.action.enabled || !config.action.tools.includes(event.toolName))
+        if (!config.enabled)
+            return;
+        // A read of a stored full output means the excerpt was not enough; that is the number that tunes context.confidence.
+        const recalled = ledger.noteAccess(JSON.stringify(event.input));
+        if (recalled)
+            record(ctx, config, "context", renderTemplate(config.widget.context, { tool: event.toolName, retention: "full output recalled" }), [`the agent went back to ${recalled}`, formatLedger(ledger.snapshot())]);
+        if (!config.action.enabled || !config.action.tools.includes(event.toolName))
             return;
         stats.inspected++;
         const task = latestUserPrompt(ctx);
-        const key = callKey(event.toolName, event.input);
-        const heldAt = held.get(key);
-        const retryAfterHold = held.has(key) && heldAt !== task;
+        // A hold happened under an earlier prompt and the user has since replied: ask whether the reply approves this action.
+        const retryAfterHold = holdPending && lastHoldPrompt !== task;
         const judge = judgeFor(config);
         const verdict = await evaluateAction({ tool: event.toolName, input: event.input, cwd: ctx.cwd, task, context: recentTaskContext(ctx) }, { config: config.action, judge, signal: ctx.signal, slop: config.slop, security: config.security, retryAfterHold });
         if (verdict.source === "skipped")
@@ -224,7 +241,7 @@ export default function wardenExtension(pi) {
         }
         if (verdict.approvedByUser) {
             stats.approved++;
-            held.delete(key);
+            holdPending = false;
         }
         const mode = activeMode(config, ctx.hasUI);
         const told = verdict.level === "confirm" && mode === "steer" ? steerReason(verdict, { canApprove: judge !== undefined }) : undefined;
@@ -262,7 +279,10 @@ export default function wardenExtension(pi) {
             return { block: true, reason: `pi-warden: the user declined this ${event.toolName} call (${reasons}). Do not retry it unchanged; ask the user how to proceed.` };
         }
         stats.held++;
-        held.set(key, task);
+        // A re-hold under the user's reply keeps the original reference prompt; otherwise the reply could never approve anything.
+        if (!holdPending)
+            lastHoldPrompt = task;
+        holdPending = true;
         if (ctx.hasUI)
             ctx.ui.notify(`warden · held ${event.toolName}: ${reasons}. The agent was told why and asked to re-plan or ask you.`, "warning");
         return { block: true, reason: told ?? steerReason(verdict, { canApprove: judge !== undefined }) };
@@ -283,6 +303,8 @@ export default function wardenExtension(pi) {
             noteError(ctx, output.error, output.errorCode);
         let content = event.content;
         const notice = securityNotice(output);
+        if (config.context.enabled && textBlocks.length === 1 && text.length >= config.context.tailMinChars)
+            ledger.candidate();
         const excerpt = compressOutput(text, output.retention);
         if (excerpt && !ctx.signal?.aborted) {
             try {
@@ -291,9 +313,11 @@ export default function wardenExtension(pi) {
                 const bytesSaved = Buffer.byteLength(text) - Buffer.byteLength(replacement) - (notice ? Buffer.byteLength(notice) * 2 + 4 : 0);
                 if (bytesSaved > 0) {
                     content = content.map(part => part.type === "text" ? { ...part, text: replacement } : part);
+                    ledger.record(path, bytesSaved);
                     record(ctx, config, "context", renderTemplate(config.widget.context, { tool: event.toolName, retention: output.retention, bytesSaved: String(bytesSaved) }), [
                         `retention: ${output.retention}; confidence ${output.confidence?.toFixed(2)}; ${output.model}; ${output.elapsedMs} ms`,
                         `saved ${bytesSaved} bytes; full output: ${path}`,
+                        formatLedger(ledger.snapshot()),
                     ]);
                 }
             }
@@ -322,7 +346,7 @@ export default function wardenExtension(pi) {
         // Checks and repeat detection use the original result, not the excerpts or security banner.
         const failed = resultFailed(event.isError, event.details, event.content);
         if (config.done.enabled)
-            recordOutcome(evidence, classifyToolResult(event.toolName, event.input, failed), event.input, event.toolName);
+            recordOutcome(evidence, classifyToolResult(event.toolName, event.input, failed, text), event.input, event.toolName);
         if (!config.stuck.enabled)
             return patch;
         attempts.push(makeAttempt(event.toolName, event.input, event.content, failed));
@@ -427,6 +451,7 @@ export default function wardenExtension(pi) {
                         `pi-warden: ${config.enabled ? `guarding ${config.action.tools.join(", ")} (${guards})` : "off"}; mode ${activeMode(config, ctx.hasUI)}; TypeSafe judgments ${source ? `enabled via ${source}` : "disabled (run /warden enable)"}; key ${key ? key.source === "stored" ? "stored (shared with pi-typesafe)" : "from TYPESAFE_API_KEY" : "missing (run /warden enable)"}.`,
                         `Session: ${stats.inspected} inspected, ${stats.judged} judged, ${stats.warned} warned, ${stats.held} held, ${stats.approved} approved on retry, ${stats.slop} slop notes, ${stats.stuck}/${stats.stuckChecks} stuck, ${stats.unverified}/${stats.doneChecks} unverified done, ${stats.proseNudges}/${stats.proseChecks} prose nudges, ${stats.errors} TypeSafe errors; ${usage?.requestsStarted ?? 0}/${config.maxRequests} requests. Steers are ${config.steerVisible ? "shown in the transcript" : "hidden from the transcript (trace panel shows them)"}.`,
                         `Thresholds: irreversible warn ${config.action.irreversible.warn} / hold ${config.action.irreversible.confirm}; off-task warn ${config.action.offTask.warn} / hold ${config.action.offTask.confirm}; stuck same-strategy ${config.stuck.sameStrategy} after ${config.stuck.minFailures} failures; done claims ${config.done.claimsDone}; slop ${config.slop.threshold}, prose ${config.slop.prose.threshold} in ${config.slop.prose.trend}/3 replies; failOpen ${config.action.failOpen}.`,
+                        formatLedger(ledger.snapshot()),
                         `Config: ${userConfigPath()}${ctx.isProjectTrusted() ? ` and ${projectConfigPath(ctx.cwd)}` : ""}.`,
                         widget.size ? `Last: ${[...widget.values()].join(" | ")}` : "No guarded activity yet this session.",
                         `Trace: ${trace.entries().length} events (/warden trace${shortcut ? `, ${shortcut}` : ""}, or click the status line in fullscreen mode; each toggles the sidebar). Widget templates in config.widget: action tokens ${TOKEN_NAMES.action.map(name => `{${name}}`).join(" ")}.`,

package/dist/guard.js

@@ -287,7 +287,12 @@ export async function evaluateAction(action, options) {
     const patterns = matchPatterns(action.tool, action.input, action.cwd);
     const reasons = [];
     let level = "allow";
+    // A shell command that merely mentions a secrets file (grep for key names, cat .env.example) is decided after Jev
+    // says whether it can write; write/edit on such a path, and offline runs, keep the immediate warning.
+    const deferSensitive = judge !== undefined && (action.tool !== "write" && action.tool !== "edit");
     for (const hit of patterns) {
+        if (hit.severity === "sensitive" && deferSensitive)
+            continue;
         level = higher(level, hit.severity === "destructive" ? "confirm" : "warn");
         reasons.push(`${hit.severity}: ${hit.label}`);
     }
@@ -323,6 +328,18 @@ export async function evaluateAction(action, options) {
             judgment.approved = answers.approved.noul;
         if (typeof answers.mutates?.noul === "number")
             judgment.mutates = answers.mutates.noul;
+        if (deferSensitive) {
+            for (const hit of patterns) {
+                if (hit.severity !== "sensitive")
+                    continue;
+                if ((judgment.mutates ?? 1) >= 0.5) {
+                    level = higher(level, "warn");
+                    reasons.push(`${hit.severity}: ${hit.label}`);
+                }
+                else
+                    reasons.push(`${hit.label} (read-only, not warned)`);
+            }
+        }
         // write/edit always change something; a command that Jev judges read-only is warned about, never held, for scope alone.
         const canChange = summary.tool === "write" || summary.tool === "edit" || (judgment.mutates ?? 1) >= 0.5;
         if (judgment.irreversible >= config.irreversible.confirm) {

package/dist/index.d.ts

@@ -11,6 +11,8 @@ export { evaluateOutput, buildOutputRequest, outputQuestions, compressOutput, sa
 export type { Retention, OutputVerdict, OutputOptions } from "./output.js";
 export { redact } from "./redact.js";
 export { completeConfig, shapeWarning, EXPECTED_SCHEMA } from "./shape.js";
+export { ContextLedger, formatLedger } from "./saver.js";
+export type { ContextLedgerSnapshot } from "./saver.js";
 export type { ShapeResult } from "./shape.js";
 export { CONFIG_SCHEMA, defaultConfig, loadConfig, applyUserOverrides, applyProjectOverrides, isMode, userConfigPath, projectConfigPath, readUserConfig, writeUserConfig, setUserSetting, PACKAGE_NAME } from "./config.js";
 export type { ActionGuardConfig, StuckGuardConfig, DoneGuardConfig, SlopGuardConfig, ProseConfig, SecurityConfig, ContextConfig, Threshold, WardenConfig, WardenMode, LoadOptions } from "./config.js";

package/dist/index.js

@@ -6,6 +6,7 @@ export { classifyToolResult, recordOutcome, emptyEvidence, needsDoneCheck, final
 export { evaluateOutput, buildOutputRequest, outputQuestions, compressOutput, saveOutput, securityNotice } from "./output.js";
 export { redact } from "./redact.js";
 export { completeConfig, shapeWarning, EXPECTED_SCHEMA } from "./shape.js";
+export { ContextLedger, formatLedger } from "./saver.js";
 export { CONFIG_SCHEMA, defaultConfig, loadConfig, applyUserOverrides, applyProjectOverrides, isMode, userConfigPath, projectConfigPath, readUserConfig, writeUserConfig, setUserSetting, PACKAGE_NAME } from "./config.js";
 export { commandOf, outputReportsFailure, COMMAND_TOOLS } from "./tools.js";
 export { renderTemplate, actionTokens, stuckTokens, doneTokens, proseTokens, defaultWidgetConfig, DEFAULT_TEMPLATES, TOKEN_NAMES } from "./widget.js";

package/dist/saver.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Session accounting for the context saver, so its value can be measured instead of assumed:
+ * how often large outputs appear, how many were compressed, how much was removed, how many turns that removal was
+ * spared from (token-turns), and how often the agent went back for the full text (a recall means the excerpt was
+ * not enough). Deterministic; no requests.
+ */
+export interface ContextLedgerSnapshot {
+    /** Single-text-block outputs at or above tailMinChars, i.e. candidates for compression. */
+    large: number;
+    compressed: number;
+    bytesSaved: number;
+    /** Assistant turns completed since the session started. */
+    turns: number;
+    /** Sum over turns of the bytes that were no longer in context on that turn, in rough tokens (bytes / 4). */
+    tokenTurnsSaved: number;
+    /** Times the agent read a stored full output after compression. */
+    recalls: number;
+}
+export declare class ContextLedger {
+    private large;
+    private compressed;
+    private bytesSaved;
+    private turns;
+    private tokenTurnsSaved;
+    private recalls;
+    private readonly stored;
+    candidate(): void;
+    record(path: string, bytesSaved: number): void;
+    /** One LLM turn finished: everything removed so far was absent from this turn's prompt. */
+    turnEnd(): void;
+    /** True when `text` (a path, command, or serialized input) mentions a stored full output for the first time. */
+    noteAccess(text: string): string | undefined;
+    snapshot(): ContextLedgerSnapshot;
+    reset(): void;
+}
+/** One line for /warden status. Says when there is nothing to report instead of printing zeros. */
+export declare function formatLedger(snapshot: ContextLedgerSnapshot): string;

package/dist/saver.js

@@ -0,0 +1,55 @@
+export class ContextLedger {
+    large = 0;
+    compressed = 0;
+    bytesSaved = 0;
+    turns = 0;
+    tokenTurnsSaved = 0;
+    recalls = 0;
+    stored = new Map();
+    candidate() {
+        this.large++;
+    }
+    record(path, bytesSaved) {
+        this.compressed++;
+        this.bytesSaved += bytesSaved;
+        this.stored.set(path, { recalled: false });
+    }
+    /** One LLM turn finished: everything removed so far was absent from this turn's prompt. */
+    turnEnd() {
+        this.turns++;
+        this.tokenTurnsSaved += Math.round(this.bytesSaved / 4);
+    }
+    /** True when `text` (a path, command, or serialized input) mentions a stored full output for the first time. */
+    noteAccess(text) {
+        for (const [path, state] of this.stored) {
+            if (!text.includes(path))
+                continue;
+            if (!state.recalled) {
+                state.recalled = true;
+                this.recalls++;
+            }
+            return path;
+        }
+        return undefined;
+    }
+    snapshot() {
+        return { large: this.large, compressed: this.compressed, bytesSaved: this.bytesSaved, turns: this.turns, tokenTurnsSaved: this.tokenTurnsSaved, recalls: this.recalls };
+    }
+    reset() {
+        this.large = 0;
+        this.compressed = 0;
+        this.bytesSaved = 0;
+        this.turns = 0;
+        this.tokenTurnsSaved = 0;
+        this.recalls = 0;
+        this.stored.clear();
+    }
+}
+/** One line for /warden status. Says when there is nothing to report instead of printing zeros. */
+export function formatLedger(snapshot) {
+    if (snapshot.large === 0)
+        return "Context saver: no tool output large enough to consider this session.";
+    const kb = (snapshot.bytesSaved / 1024).toFixed(1);
+    const recallRate = snapshot.compressed ? Math.round((snapshot.recalls / snapshot.compressed) * 100) : 0;
+    return `Context saver: ${snapshot.large} large outputs, ${snapshot.compressed} compressed, ${kb} KB removed (~${Math.round(snapshot.bytesSaved / 4)} tokens), ~${snapshot.tokenTurnsSaved} token-turns spared over ${snapshot.turns} turns, ${snapshot.recalls} recall${snapshot.recalls === 1 ? "" : "s"} of the full output (${recallRate}%).`;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-warden",
-  "version": "0.5.2",
+  "version": "0.5.4",
   "description": "Guardrails for Pi built on pi-typesafe that steer the agent instead of interrupting you: Jev judges irreversible and off-task tool calls before they run, detects stuck loops, checks unverified done claims, and flags slop.",
   "type": "module",
   "license": "MIT",