pi-canary 1.0.0

package/.gitattributes

@@ -0,0 +1,4 @@
+* text=auto eol=lf
+*.ts text eol=lf
+*.json text eol=lf
+*.md text eol=lf

package/README.md

@@ -0,0 +1,83 @@
+# pi-canary
+
+A [pi](https://pi.dev) extension that silently verifies the agent's context awareness on every turn using hidden canary tokens.
+
+Before answering your message, the agent must locate and return N canary tokens distributed across the conversation history. The entire verification exchange is invisible: no tokens in the thinking block, no tokens in the visible response, nothing in the history the agent uses to answer you.
+
+## Install
+
+```bash
+pi install git:github.com/sebaxzero/pi-canary.git
+```
+
+Or install project-locally (adds to `.pi/settings.json` only):
+
+```bash
+pi install git:github.com/sebaxzero/pi-canary.git -l
+```
+
+## How it works
+
+Every time you send a message, the extension runs a hidden two-phase exchange before your question is answered:
+
+**Phase 1 — Verify**
+
+- N random 24-character canary tokens are generated (or reused if `VARIANT=fixed`).
+- They are injected at the configured positions across the conversation history. The last token also carries a verification instruction.
+- Your original question is temporarily suppressed.
+- The agent is asked only to return the N tokens by name.
+- The response is captured and checked. The exchange is hidden from the TUI.
+
+**Phase 2 — Respond**
+
+- The tokens and the verification exchange are stripped from context entirely.
+- Your original question is restored.
+- The agent answers normally, with no canary tokens anywhere in its view.
+
+If verification fails, a warning notification appears in the TUI.
+
+## Configuration
+
+Persistent configuration lives in `extensions/canary.json`. You can ask the agent to edit it directly:
+
+```json
+{
+  "COUNT": 3,
+  "POSITION": "end",
+  "VARIANT": "fixed",
+  "FAIL_COMPACT": 0
+}
+```
+
+| Key | Default | Description |
+|-----|---------|-------------|
+| `COUNT` | `3` | Number of canary tokens injected per turn |
+| `POSITION` | `end` | Where tokens are injected: `start`, `equidistant`, or `end` |
+| `VARIANT` | `fixed` | `fixed` = same tokens every turn (preserves KV cache); `variant` = new tokens each turn |
+| `FAIL_COMPACT` | `0` | Compact context after N consecutive failures (`0` = disabled) |
+
+**`POSITION=end` + `VARIANT=fixed`** (the defaults) is the cache-friendly mode for local model servers: the message prefix never changes and the injected suffix is always the same tokens, so the KV cache stays warm after the first turn. Use `POSITION=equidistant` + `VARIANT=variant` for maximum coverage at the cost of cache invalidation every turn.
+
+Changes to the JSON take effect on the next session. For live tuning within a session, use the command below.
+
+## Command
+
+```
+/canary                    — show current phase, failure count, and config
+/canary set KEY=VAL        — override config for the current session only
+/canary set KEY=VAL KEY=VAL ...
+```
+
+Example: `/canary set COUNT=5 POSITION=equidistant VARIANT=variant`
+
+## Compatibility
+
+Works alongside [pi-loop-police](https://github.com/sebaxzero/pi-loop-police). When loop-police aborts a turn, the canary check yields gracefully and does not fire its own recovery.
+
+## License
+
+MIT
+
+---
+
+Built with [Claude](https://claude.ai).

package/extensions/canary.json

@@ -0,0 +1,6 @@
+{
+  "COUNT": 3,
+  "POSITION": "end",
+  "VARIANT": "fixed",
+  "FAIL_COMPACT": 0
+}

package/extensions/canary.ts

@@ -0,0 +1,323 @@
+/**
+ * Canary Extension
+ *
+ * Runs a hidden pre-turn where the agent must return N canary tokens distributed
+ * across the conversation history. The verification is invisible to the user:
+ * tokens never appear in the thinking block, visible responses, or final answer.
+ *
+ * Flow per user turn:
+ *   1. [VERIFY] Hidden LLM call — tokens visible, original question suppressed.
+ *      Agent returns tokens only. Response hidden.
+ *   2. [RESPOND] Real LLM call — clean context, no tokens anywhere, agent answers normally.
+ *
+ * POSITION=end + VARIANT=fixed (defaults) preserves KV cache on local model servers:
+ * the prefix never changes and the injected suffix is always the same tokens.
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+
+// Loaded from sibling JSON at startup; /set overrides for the current session only
+const cfg = (() => {
+  const defaults = {
+    COUNT: 3,
+    POSITION: "end" as "start" | "equidistant" | "end",
+    VARIANT: "fixed" as "fixed" | "variant",
+    FAIL_COMPACT: 0,
+  };
+  try {
+    const path = new URL("canary.json", import.meta.url).pathname;
+    return { ...defaults, ...JSON.parse((globalThis as any).Deno.readTextFileSync(path)) };
+  } catch {
+    return defaults;
+  }
+})();
+
+// --- Token generation ---
+
+const TOKEN_LENGTH = 24;
+const TOKEN_CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+
+function generateToken(): string {
+  const arr = new Uint8Array(TOKEN_LENGTH);
+  crypto.getRandomValues(arr);
+  return Array.from(arr, (b) => TOKEN_CHARSET[b % TOKEN_CHARSET.length]).join("");
+}
+
+// --- Message builders ---
+
+function positionLabel(index: number, total: number): string {
+  if (cfg.POSITION === "start") return "beginning of context";
+  if (cfg.POSITION === "end") return "end of context";
+  // equidistant
+  if (total === 1) return "end of context";
+  const fraction = index / (total - 1);
+  if (fraction === 0) return "beginning of context";
+  if (fraction === 1) return "end of context";
+  return `middle of context (~${Math.round(fraction * 100)}%)`;
+}
+
+function buildAnchorContent(token: string, index: number, total: number): string {
+  return `[CANARY — TOKEN_${index + 1} of ${total} — ${positionLabel(index, total)}]\n${token}`;
+}
+
+function buildVerificationInstruction(count: number): string {
+  const formatLines = Array.from({ length: count }, (_, i) => `TOKEN_${i + 1}: <value>`);
+  return [
+    "═══════════════════════════════════════════════════════════",
+    "⚠️  CANARY CHECK ⚠️",
+    "═══════════════════════════════════════════════════════════",
+    "",
+    `Return ONLY the ${count} canary token${count === 1 ? "" : "s"} below. Do not say anything else.`,
+    ...formatLines,
+    "═══════════════════════════════════════════════════════════",
+  ].join("\n");
+}
+
+// --- Extension ---
+
+type Phase = "idle" | "verifying" | "responding";
+
+export default function (pi: ExtensionAPI) {
+  let phase: Phase = "idle";
+  let currentTokens: string[] | null = null;
+  // Reused across turns in fixed mode; null forces regeneration
+  let fixedTokens: string[] | null = null;
+  let consecutiveFailures = 0;
+  // Guard against double-injection if context fires twice before message_end (retries)
+  let verifyContextSent = false;
+  // Timestamp of the hidden verification assistant message — used to filter it in Phase 2
+  let verifyResponseTimestamp: number | null = null;
+
+  pi.on("before_agent_start", (_event, _ctx) => {
+    if (cfg.VARIANT === "fixed") {
+      if (!fixedTokens || fixedTokens.length !== cfg.COUNT) {
+        fixedTokens = Array.from({ length: cfg.COUNT }, generateToken);
+      }
+      currentTokens = fixedTokens;
+    } else {
+      currentTokens = Array.from({ length: cfg.COUNT }, generateToken);
+    }
+    phase = "verifying";
+    verifyContextSent = false;
+  });
+
+  pi.on("context", (event, _ctx) => {
+    // --- Phase 1: build verification-only context ---
+    if (phase === "verifying" && currentTokens && !verifyContextSent) {
+      verifyContextSent = true;
+      const messages = [...event.messages];
+
+      // Suppress the original user question so the agent focuses only on the canary check.
+      // The question remains in session history and reappears in Phase 2.
+      if (messages.length > 0 && (messages[messages.length - 1] as any).role === "user") {
+        messages.pop();
+      }
+
+      const histLen = messages.length;
+      const count = currentTokens.length;
+
+      const injections = currentTokens
+        .map((token, i) => {
+          let insertAt: number;
+          if (cfg.POSITION === "start") insertAt = 0;
+          else if (cfg.POSITION === "end") insertAt = histLen;
+          else {
+            // equidistant
+            const fraction = count === 1 ? 1 : i / (count - 1);
+            insertAt = Math.round(fraction * histLen);
+          }
+          return { insertAt, token, i, isLast: i === count - 1 };
+        })
+        .reverse(); // reverse so splices don't shift earlier indices
+
+      for (const { insertAt, token, i, isLast } of injections) {
+        const content = isLast
+          ? buildAnchorContent(token, i, count) + "\n\n" + buildVerificationInstruction(count)
+          : buildAnchorContent(token, i, count);
+
+        messages.splice(insertAt, 0, {
+          role: "custom",
+          customType: "canary",
+          content,
+          display: false,
+          timestamp: Date.now(),
+        } as any);
+      }
+
+      return { messages };
+    }
+
+    // --- Phase 2: strip the hidden verification exchange ---
+    if (phase === "responding") {
+      const messages = event.messages.filter(
+        (m: any) => m.customType !== "canary" &&
+                    m.timestamp !== verifyResponseTimestamp
+      );
+      if (messages.length !== event.messages.length) return { messages };
+    }
+  });
+
+  pi.on("message_end", (event, ctx) => {
+    if (event.message.role !== "assistant") return;
+
+    // --- Phase 2 completion: reset and let the response through ---
+    if (phase === "responding") {
+      phase = "idle";
+      return;
+    }
+
+    if (phase !== "verifying") return;
+
+    const tokens = currentTokens;
+    currentTokens = null;
+
+    // Skip aborted or errored turns
+    const stopReason = (event.message as any).stopReason;
+    if (stopReason === "aborted" || stopReason === "error") {
+      phase = "idle";
+      return;
+    }
+
+    // Yield to loop-police
+    const rawContent = event.message.content;
+    if (Array.isArray(rawContent)) {
+      for (const part of rawContent) {
+        if (
+          part && typeof part === "object" && "type" in part &&
+          part.type === "thinking" && "thinking" in part &&
+          typeof part.thinking === "string" &&
+          (part.thinking.includes("[THINKING LOOP") || part.thinking.includes("[SEMANTIC LOOP"))
+        ) {
+          phase = "idle";
+          return;
+        }
+      }
+    }
+
+    if (!tokens) { phase = "idle"; return; }
+
+    // Extract text from the verification response
+    const textParts: string[] = [];
+    if (typeof rawContent === "string") {
+      textParts.push(rawContent);
+    } else if (Array.isArray(rawContent)) {
+      for (const part of rawContent) {
+        if (part && typeof part === "object" && "type" in part && part.type === "text" && "text" in part && typeof part.text === "string") {
+          textParts.push(part.text);
+        }
+      }
+    }
+    const fullText = textParts.join("\n");
+
+    const missing = tokens
+      .map((t, i) => ({ label: `TOKEN_${i + 1}`, value: t, found: fullText.includes(t) }))
+      .filter((x) => !x.found);
+
+    // Replace the verification response with an empty assistant message.
+    // AssistantMessage.role must stay "assistant" — no display flag exists on this type,
+    // but content: [] renders as nothing in TUI. Filtered from Phase 2 context by timestamp.
+    verifyResponseTimestamp = (event.message as any).timestamp ?? null;
+    const emptyResponse = { ...event.message, content: [] } as any;
+
+    if (missing.length === 0) {
+      // Verification passed — proceed to Phase 2
+      consecutiveFailures = 0;
+      phase = "responding";
+      pi.sendMessage(
+        { customType: "canary", content: "✓", display: false },
+        { triggerTurn: true, deliverAs: "steer" }
+      );
+      return { message: emptyResponse };
+    }
+
+    // Verification failed
+    consecutiveFailures++;
+
+    if (cfg.FAIL_COMPACT > 0 && consecutiveFailures >= cfg.FAIL_COMPACT) {
+      consecutiveFailures = 0;
+      phase = "idle";
+      ctx.ui.notify(
+        `Canary: ${cfg.FAIL_COMPACT} consecutive failure(s) — triggering compaction`,
+        "warning"
+      );
+      ctx.compact({
+        customInstructions: `The agent failed to recall canary tokens ${cfg.FAIL_COMPACT} time(s) in a row. After compaction, return all canary tokens before doing anything else.`,
+      });
+      return { message: emptyResponse };
+    }
+
+    // Proceed to Phase 2 despite failure (user still gets a response)
+    ctx.ui.notify("⚠️ Canary check failed — context may be degraded", "warning");
+    phase = "responding";
+    pi.sendMessage(
+      { customType: "canary", content: "⚠️", display: false },
+      { triggerTurn: true, deliverAs: "steer" }
+    );
+    return { message: emptyResponse };
+  });
+
+  pi.registerCommand("canary", {
+    description: "Show status; /canary set KEY=VAL [KEY=VAL ...]",
+    handler: (args, ctx) => {
+      const trimmed = args?.trim() ?? "";
+
+      if (trimmed.startsWith("set ")) {
+        const results: string[] = [];
+        for (const pair of trimmed.slice(4).trim().split(/\s+/)) {
+          const eq = pair.indexOf("=");
+          const key = pair.slice(0, eq).toUpperCase();
+          const val = pair.slice(eq + 1);
+          if (eq > 0 && val !== "") {
+            if (key === "COUNT") {
+              const n = parseInt(val, 10);
+              if (n > 0) { cfg.COUNT = n; fixedTokens = null; results.push(`COUNT=${cfg.COUNT}`); }
+              else results.push(`invalid COUNT: ${val}`);
+            } else if (key === "POSITION") {
+              if (val === "start" || val === "equidistant" || val === "end") {
+                cfg.POSITION = val; results.push(`POSITION=${cfg.POSITION}`);
+              } else results.push(`invalid POSITION: ${val} (start|equidistant|end)`);
+            } else if (key === "VARIANT") {
+              if (val === "fixed" || val === "variant") {
+                cfg.VARIANT = val; if (val === "variant") fixedTokens = null;
+                results.push(`VARIANT=${cfg.VARIANT}`);
+              } else results.push(`invalid VARIANT: ${val} (fixed|variant)`);
+            } else if (key === "FAIL_COMPACT") {
+              const n = parseInt(val, 10);
+              if (n >= 0) { cfg.FAIL_COMPACT = n; results.push(`FAIL_COMPACT=${cfg.FAIL_COMPACT}`); }
+              else results.push(`invalid FAIL_COMPACT: ${val}`);
+            } else {
+              results.push(`unknown: ${key}`);
+            }
+          }
+        }
+        ctx.ui.notify(`Canary: ${results.join(", ")}`, "info");
+        return;
+      }
+
+      ctx.ui.notify(
+        [
+          "Canary status",
+          `  phase:                 ${phase}`,
+          `  tokens per turn:       ${cfg.COUNT}`,
+          `  position:              ${cfg.POSITION}`,
+          `  variant:               ${cfg.VARIANT}`,
+          `  compact after N fails: ${cfg.FAIL_COMPACT === 0 ? "disabled" : cfg.FAIL_COMPACT}`,
+          `  consecutive failures:  ${consecutiveFailures}`,
+          "",
+          "  config (/set = session only; edit canary.json for persistence):",
+          `    COUNT=${cfg.COUNT}`,
+          `    POSITION=${cfg.POSITION}`,
+          `    VARIANT=${cfg.VARIANT}`,
+          `    FAIL_COMPACT=${cfg.FAIL_COMPACT}`,
+        ].join("\n"),
+        "info"
+      );
+    },
+  });
+
+  pi.on("session_start", (_event, ctx) => {
+    if (ctx.hasUI) {
+      ctx.ui.notify(`Canary loaded — ${cfg.COUNT} token(s), position=${cfg.POSITION}, variant=${cfg.VARIANT}`, "info");
+    }
+  });
+}

package/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "pi-canary",
+  "version": "1.0.0",
+  "description": "Pi extension: silently verifies agent context awareness every turn using hidden canary tokens. KV-cache friendly.",
+  "keywords": ["pi-package", "pi", "pi-coding-agent", "extension", "context-awareness", "canary", "safety", "verification", "local-llm"],
+  "license": "MIT",
+  "pi": {
+    "extensions": ["./extensions"]
+  }
+}