talon-agent 3.12.5 → 3.14.0

package/src/backend/shared/cache-telemetry.ts

@@ -0,0 +1,297 @@
+/**
+ * Prompt-cache telemetry — the numbers needed to tell whether Talon's
+ * prompt cache is actually working, and why not when it isn't.
+ *
+ * Motivation (see docs/memory-persona-plan.md §3.6): the obvious lever for a
+ * sparse-traffic chat bot would be a 1-hour cache TTL, but
+ * `@anthropic-ai/claude-agent-sdk` exposes no cache-control surface at all —
+ * it owns `cache_control` placement internally. So the only levers Talon has
+ * are (a) keeping the prompt prefix byte-stable and (b) keeping it small,
+ * and the only way to know which one is costing money is to measure.
+ *
+ * Four things are measured here, each answering a question the aggregate
+ * `cache=NN%` on the accounting line cannot:
+ *
+ *   1. **Cross-turn vs within-turn hits.** An agentic turn makes many model
+ *      requests; every one after the first reads the prefix the first one
+ *      just paid for. So a 97% aggregate hit rate is compatible with *every
+ *      turn* re-writing the whole prefix. The turn's FIRST request is the
+ *      only one that reports whether the previous turn's cache survived —
+ *      that is the number that tracks cost.
+ *   2. **Tool-set churn.** Tool definitions render BEFORE the system prompt,
+ *      so any change to the tool array invalidates the system prompt and the
+ *      whole message history with it. A stable prompt behind an unstable
+ *      tool list buys nothing, and Talon's per-chat MCP servers are exactly
+ *      the shape that can shift mid-session.
+ *   3. **Lookback-window risk.** A cache breakpoint searches back at most
+ *      `CACHE_LOOKBACK_BLOCKS` content blocks for a prior entry. A turn that
+ *      emits more blocks than that can leave the *next* turn's breakpoint
+ *      unable to find anything — a silent miss with no error.
+ *   4. **Sub-minimum prompts.** Below a model-specific token floor nothing
+ *      caches, and the API reports no error. Talon's chat prompts clear every
+ *      floor; its one-shot prompts (dream, heartbeat, cron) may not.
+ *
+ * Everything here is pure except `noteToolFingerprint`, which keeps a small
+ * per-chat map so it can name what changed rather than just that something
+ * did.
+ */
+
+import { logWarn } from "../../util/log.js";
+
+// ── Per-turn cache stats ────────────────────────────────────────────────────
+
+/**
+ * One model request's usage, as the SDK reports it in
+ * `result.usage.iterations`. Fields are optional because a given provider
+ * may omit any of them; absent is treated as zero.
+ */
+export type CacheIteration = {
+  readonly input_tokens?: number;
+  readonly cache_read_input_tokens?: number;
+  readonly cache_creation_input_tokens?: number;
+};
+
+/** Cache behaviour for one user-visible turn. */
+export type TurnCacheStats = {
+  /** Model requests the SDK made inside this turn. */
+  readonly modelRequests: number;
+  /** Cache read on the turn's FIRST request — did the last turn's prefix live? */
+  readonly firstRead: number;
+  /** Cache written by the turn's first request. */
+  readonly firstWrite: number;
+  /** Cache read across every request in the turn. */
+  readonly totalRead: number;
+  /** Cache written across every request in the turn. */
+  readonly totalWrite: number;
+};
+
+/**
+ * What the turn's first request says about the previous turn's cache.
+ *
+ *   - `hit`  — the prefix survived; this turn read it instead of paying for it.
+ *   - `miss` — the prefix was gone and had to be re-written. The expensive case,
+ *              and the one a TTL shorter than the gap between turns produces.
+ *   - `none` — nothing was cached either way: below the model's cacheable
+ *              minimum, or a provider that doesn't cache at all.
+ */
+export type CrossTurnVerdict = "hit" | "miss" | "none";
+
+/**
+ * Fold a turn's per-request usage into cache stats. Returns undefined when
+ * the provider reported no iterations — the caller then logs nothing rather
+ * than inventing a verdict from aggregate totals it can't attribute.
+ */
+export function turnCacheStats(
+  iterations: readonly CacheIteration[] | undefined,
+): TurnCacheStats | undefined {
+  if (!iterations || iterations.length === 0) return undefined;
+  const first = iterations[0]!;
+  let totalRead = 0;
+  let totalWrite = 0;
+  for (const it of iterations) {
+    totalRead += it.cache_read_input_tokens ?? 0;
+    totalWrite += it.cache_creation_input_tokens ?? 0;
+  }
+  return {
+    modelRequests: iterations.length,
+    firstRead: first.cache_read_input_tokens ?? 0,
+    firstWrite: first.cache_creation_input_tokens ?? 0,
+    totalRead,
+    totalWrite,
+  };
+}
+
+/** Classify the turn's first request. See {@link CrossTurnVerdict}. */
+export function crossTurnVerdict(stats: TurnCacheStats): CrossTurnVerdict {
+  if (stats.firstRead > 0) return "hit";
+  if (stats.firstWrite > 0) return "miss";
+  return "none";
+}
+
+/**
+ * Compact suffix for the per-turn accounting line, e.g.
+ * `xturn=miss reqs=11 rw=8.0`. Deliberately terse and space-delimited so a
+ * week of logs can be parsed without a schema:
+ *
+ *   - `xturn` — the cross-turn verdict; the field that tracks cost.
+ *   - `reqs`  — model requests in the turn; explains a high aggregate hit
+ *               rate that isn't saving anything.
+ *   - `rw`    — total read ÷ total write. Above ~1 the turn amortised its
+ *               write; at or below it, the write dominated.
+ */
+export function formatTurnCache(stats: TurnCacheStats): string {
+  const parts = [
+    `xturn=${crossTurnVerdict(stats)}`,
+    `reqs=${stats.modelRequests}`,
+  ];
+  if (stats.totalWrite > 0) {
+    parts.push(`rw=${(stats.totalRead / stats.totalWrite).toFixed(1)}`);
+  }
+  return parts.join(" ");
+}
+
+// ── Lookback window ────────────────────────────────────────────────────────
+
+/**
+ * How far back a cache breakpoint searches for a prior entry, in content
+ * blocks. A turn that emits more than this can prevent the NEXT turn from
+ * finding any cache to read.
+ */
+export const CACHE_LOOKBACK_BLOCKS = 20;
+
+/**
+ * Rough content-block count for a turn from its tool-call count. Each tool
+ * call is a `tool_use` block plus a `tool_result` block, and the assistant
+ * text around them adds at least one more — so `2n + 1` is a floor, not an
+ * estimate. Used only to decide whether to warn, never to report a number.
+ */
+export function estimateTurnBlocks(toolCalls: number): number {
+  return Math.max(0, toolCalls) * 2 + 1;
+}
+
+/** True when a turn plausibly emitted more blocks than a breakpoint looks back. */
+export function exceedsLookbackWindow(toolCalls: number): boolean {
+  return estimateTurnBlocks(toolCalls) > CACHE_LOOKBACK_BLOCKS;
+}
+
+// ── Cacheable minimum ──────────────────────────────────────────────────────
+
+/**
+ * Minimum cacheable prefix, in tokens, by model. Below this nothing is
+ * cached and the API reports no error — `cache_creation_input_tokens` is
+ * simply 0.
+ *
+ * The floor is NOT monotonic across generations (512 on the newest models,
+ * 4096 on Opus 4.6 and Haiku 4.5), so it can't be inferred from a version
+ * number. Longest match wins, so `sonnet-4-6` is checked before `sonnet`.
+ *
+ * Bare aliases resolve only where Talon's catalog leaves no ambiguity:
+ * `haiku` is Haiku 4.5 (and carries the largest floor, making it the most
+ * likely to silently not cache). Ambiguous aliases — `opus`, `sonnet`,
+ * `default` — deliberately return undefined: a warning that fires on the
+ * wrong model teaches people to ignore warnings.
+ */
+const CACHE_MINIMUMS: readonly (readonly [string, number])[] = [
+  ["fable-5", 512],
+  ["mythos-5", 512],
+  ["opus-5", 512],
+  ["opus-4-8", 1024],
+  ["sonnet-5", 1024],
+  ["sonnet-4-6", 1024],
+  ["sonnet-4-5", 1024],
+  ["opus-4-1", 1024],
+  ["opus-4-7", 2048],
+  ["mythos-preview", 2048],
+  ["opus-4-6", 4096],
+  ["opus-4-5", 4096],
+  ["haiku-4-5", 4096],
+  ["haiku", 4096],
+];
+
+/**
+ * The cacheable-prefix floor for a model, or undefined when the model string
+ * doesn't unambiguously identify one.
+ */
+export function cacheMinimumTokens(model: string): number | undefined {
+  const m = model.toLowerCase();
+  let best: { key: string; min: number } | undefined;
+  for (const [key, min] of CACHE_MINIMUMS) {
+    if (!m.includes(key)) continue;
+    if (!best || key.length > best.key.length) best = { key, min };
+  }
+  return best?.min;
+}
+
+/** Cheap tokenizer-free estimate (~4 chars/token), matching soul/projector. */
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Warn when a prompt is too small to be cacheable on its model. No-op when
+ * the model's floor is unknown or the prompt clears it. `label` names the
+ * caller (e.g. `"dream"`) so the warning points somewhere.
+ */
+export function warnIfBelowCacheMinimum(
+  label: string,
+  model: string,
+  prompt: string,
+): void {
+  const min = cacheMinimumTokens(model);
+  if (min === undefined) return;
+  const estimated = estimateTokens(prompt);
+  if (estimated >= min) return;
+  logWarn(
+    "agent",
+    `[${label}] prompt ~${estimated} tokens is below ${model}'s ${min}-token ` +
+      `cacheable minimum — nothing will be cached for this run`,
+  );
+}
+
+// ── Tool-set fingerprint ───────────────────────────────────────────────────
+
+/**
+ * Per-chat fingerprint of the last tool set seen. Bounded so a long-lived
+ * process with many chats can't grow it without limit; eviction is
+ * insertion-ordered, and a false "changed" warning after eviction is
+ * cheaper than unbounded retention.
+ */
+const MAX_TRACKED_CHATS = 256;
+const lastToolSets = new Map<string, readonly string[]>();
+
+/** Stable fingerprint for a tool set: sorted, deduped names. */
+export function toolFingerprint(
+  builtinTools: readonly string[],
+  mcpServerNames: readonly string[],
+): readonly string[] {
+  return [
+    ...new Set([...builtinTools, ...mcpServerNames.map((n) => `mcp:${n}`)]),
+  ].sort();
+}
+
+/**
+ * Record this turn's tool set for a chat and warn when it differs from the
+ * previous turn's. Returns true when a change was detected.
+ *
+ * Tools render before the system prompt, so a mid-session change invalidates
+ * the system prompt and every cached message after it — the most expensive
+ * cache event available, and invisible in aggregate hit-rate numbers.
+ */
+export function noteToolFingerprint(
+  chatId: string,
+  fingerprint: readonly string[],
+): boolean {
+  const previous = lastToolSets.get(chatId);
+
+  if (lastToolSets.size >= MAX_TRACKED_CHATS && !previous) {
+    const oldest = lastToolSets.keys().next().value;
+    if (oldest !== undefined) lastToolSets.delete(oldest);
+  }
+  lastToolSets.set(chatId, fingerprint);
+
+  if (!previous) return false;
+  if (
+    previous.length === fingerprint.length &&
+    previous.every((t, i) => t === fingerprint[i])
+  ) {
+    return false;
+  }
+
+  const before = new Set(previous);
+  const after = new Set(fingerprint);
+  const added = fingerprint.filter((t) => !before.has(t));
+  const removed = previous.filter((t) => !after.has(t));
+  logWarn(
+    "agent",
+    `[${chatId}] tool set changed mid-session — invalidates the whole prompt ` +
+      `cache for this chat` +
+      (added.length ? ` (+${added.join(",")})` : "") +
+      (removed.length ? ` (-${removed.join(",")})` : ""),
+  );
+  return true;
+}
+
+/** Drop all tracked fingerprints (tests / explicit reset). */
+export function resetToolFingerprints(): void {
+  lastToolSets.clear();
+}

package/src/backend/shared/index.ts

@@ -68,6 +68,16 @@ export {
   type TokenUsageSnapshot,
 } from "./usage.js";
 
+// Only what is consumed THROUGH the barrel. Everything else in
+// cache-telemetry.ts is imported from the module directly, matching the
+// barrel discipline the prompt/ barrel was just trimmed to.
+export {
+  formatTurnCache,
+  estimateTurnBlocks,
+  exceedsLookbackWindow,
+  CACHE_LOOKBACK_BLOCKS,
+} from "./cache-telemetry.js";
+
 export {
   prepareSystemPrompt,
   appendBackendSuffix,

package/src/core/background/dream.ts

@@ -235,6 +235,7 @@ If commands fail, log the error and continue — this stage is optional.`
       .replace(/\{\{lastRunIso\}\}/g, lastRunIso)
       .replace(/\{\{memoryFile\}\}/g, memoryFile)
       .replace(/\{\{dailyMemoryDir\}\}/g, dirs.dailyMemory)
+      .replace(/\{\{memoryArchiveDir\}\}/g, dirs.memoryArchive)
       .replace(/\{\{mempalaceSection\}\}/g, mempalaceSection);
   } catch {
     throw new Error("Failed to read dream prompt (dream.md)");

package/src/core/background/heartbeat/agent.ts

@@ -127,14 +127,17 @@ export async function runHeartbeatAgent(
 
   let prompt: string;
   let hadGoalsVar: boolean;
+  let hadStateVar: boolean;
   try {
     const raw = readFileSync(promptPath, "utf-8");
     hadGoalsVar = raw.includes("{{goals}}");
+    hadStateVar = raw.includes("{{stateFile}}");
     prompt = raw
       .replace(/\{\{workspace\}\}/g, workspace)
       .replace(/\{\{logsDir\}\}/g, logsDir)
       .replace(/\{\{lastRunIso\}\}/g, lastRunIso)
       .replace(/\{\{memoryFile\}\}/g, memoryFile)
+      .replace(/\{\{stateFile\}\}/g, pathFiles.state)
       .replace(/\{\{instructionsFile\}\}/g, instructionsFile)
       .replace(/\{\{dailyMemoryFile\}\}/g, dailyMemoryFile)
       .replace(/\{\{runCount\}\}/g, String(runCount))
@@ -155,6 +158,24 @@ export async function runHeartbeatAgent(
     }).trim()}`;
   }
 
+  // Same vintage problem for the memory/state split: a seeded heartbeat.md
+  // from before it still instructs the agent to write memory.md, which is
+  // how status snapshots accreted in the durable store in the first place.
+  // Append the ownership rules so the split holds regardless of template
+  // vintage — the seeded copy is never rewritten once the user owns it.
+  if (!hadStateVar) {
+    logWarn(
+      "heartbeat",
+      `Seeded ${promptPath} predates the memory/state split — appending ` +
+        `file-ownership rules. Delete that file to re-seed the current prompt.`,
+    );
+    prompt += `\n\n${loadSystemTemplate("heartbeat-agent", {
+      mode: "state-fallback",
+      stateFile: pathFiles.state,
+      memoryFile,
+    }).trim()}`;
+  }
+
   const model = config.heartbeatModel ?? config.model ?? getDefaultModel();
 
   const backend = config.getBackend?.() ?? null;

package/src/core/prompt/assemble.ts

@@ -16,8 +16,12 @@
  *   2. Core behaviour                       ~/.talon/prompts/custom.md,
  *                                           else base.md, else fallback
  *   3. Frontend capabilities                ~/.talon/prompts/<frontend>.md
- *   4. Persistent memory (size-capped)      prompts/system/persistent-memory.md
+ *   4. Persistent memory (ranked, capped)   prompts/system/persistent-memory.md
  *                                           wrapping ~/.talon/workspace/memory/memory.md
+ *                                           via memory-view.ts
+ *   4.5 Live state (capped)                 prompts/system/live-state.md
+ *                                           wrapping ~/.talon/workspace/memory/state.md
+ *                                           (heartbeat-owned, rewritten whole)
  *   5. Memory recall + capability docs      prompts/system/{memory-recall,workspace,...}.md
  *   6. Plugin additions                     plugin.systemPrompt() contributions
  *   (7. Delivery contract — appended by the backend as its suffix,
@@ -57,6 +61,7 @@ import { dirs, files as pathFiles } from "../../util/paths.js";
 import { todayAndYesterday } from "../../util/time.js";
 import { log } from "../../util/log.js";
 import { loadSystemTemplate } from "./templates.js";
+import { renderMemoryView } from "./memory-view.js";
 import { renderWorkspaceListing } from "./workspace-listing.js";
 import { renderSkillsPrompt } from "../../storage/skill-store.js";
 import { renderStickerLibraryPrompt } from "../../storage/sticker-store.js";
@@ -95,13 +100,14 @@ export function joinSystemPromptParts(parts: SystemPromptParts): string {
 // ── Tunables ────────────────────────────────────────────────────────────────
 
 /**
- * Cap on how much of `memory.md` is injected into the static prompt.
- * Memory files grow without bound over months of use; injecting all
- * of it bloats EVERY session from its very first turn. 12k chars is
- * roughly 3k tokens — past that, the model gets the head of the file
- * plus a truncation pointer and can Read the rest on demand.
+ * Cap on the injected `state.md` block. Deliberately much tighter than the
+ * memory cap: this is a status snapshot the heartbeat rewrites every run, so
+ * anything past a couple of thousand chars means the heartbeat is
+ * accumulating history in a file that is supposed to be replaced — the
+ * failure the memory/state split exists to prevent. Truncating loudly is the
+ * signal that it is happening.
  */
-export const MEMORY_INJECT_MAX_CHARS = 12_000;
+export const STATE_INJECT_MAX_CHARS = 2_000;
 
 // ── Helpers ─────────────────────────────────────────────────────────────────
 
@@ -114,23 +120,6 @@ function readOptionalFile(path: string): string {
   return "";
 }
 
-/**
- * Truncate memory content at the cap, snapping back to the previous
- * newline so the cut never lands mid-sentence. Returns the (possibly
- * shortened) content and whether truncation happened.
- */
-function capMemory(content: string): { text: string; truncated: boolean } {
-  if (content.length <= MEMORY_INJECT_MAX_CHARS) {
-    return { text: content, truncated: false };
-  }
-  const head = content.slice(0, MEMORY_INJECT_MAX_CHARS);
-  const lastNewline = head.lastIndexOf("\n");
-  return {
-    text: (lastNewline > 0 ? head.slice(0, lastNewline) : head).trimEnd(),
-    truncated: true,
-  };
-}
-
 let lastLoggedPromptKey = "";
 
 // ── Assembly ────────────────────────────────────────────────────────────────
@@ -194,17 +183,41 @@ export function assembleSystemPrompt(
   }
 
   // 4. Persistent memory — size-capped so a memory file that has grown
-  //    for months can't bloat every session from turn 0.
+  //    for months can't bloat every session from turn 0. Over the cap the
+  //    view ranks sections rather than head-slicing, so durable knowledge
+  //    isn't evicted by whatever happens to sit at the top of the file
+  //    (see prompt/memory-view.ts).
   const memory = readOptionalFile(pathFiles.memory);
   if (memory) {
-    const { text, truncated } = capMemory(memory);
+    const { text, truncated, omitted } = renderMemoryView(memory);
     staticParts.push(
       loadSystemTemplate("persistent-memory", {
         content: text,
         truncated: truncated ? "yes" : undefined,
+        omitted: omitted || undefined,
+      }),
+    );
+    loaded.push(truncated ? "memory(ranked)" : "memory");
+  }
+
+  // 4.5. Live state — the heartbeat's rewritten-whole status snapshot, kept
+  //      OUT of memory.md so "as of Run #N" sections can't accrete in the
+  //      durable store and push real knowledge past the cap. Capped hard:
+  //      this is the most volatile content in the static prompt, and a
+  //      status file that grows is the exact failure this split exists to
+  //      prevent.
+  const state = readOptionalFile(pathFiles.state);
+  if (state) {
+    const truncated = state.length > STATE_INJECT_MAX_CHARS;
+    staticParts.push(
+      loadSystemTemplate("live-state", {
+        content: truncated
+          ? state.slice(0, STATE_INJECT_MAX_CHARS).trimEnd()
+          : state,
+        truncated: truncated ? "yes" : undefined,
       }),
     );
-    loaded.push(truncated ? "memory(capped)" : "memory");
+    loaded.push(truncated ? "state(capped)" : "state");
   }
 
   // 5. Package-owned behavioural and capability docs. The memory policy

package/src/core/prompt/embedded-prompts.ts

@@ -26,14 +26,15 @@ import asset12 from "../../../prompts/system/cron.md" with { type: "file" };
 import asset13 from "../../../prompts/system/daily-memory.md" with { type: "file" };
 import asset14 from "../../../prompts/system/goals.md" with { type: "file" };
 import asset15 from "../../../prompts/system/heartbeat-agent.md" with { type: "file" };
-import asset16 from "../../../prompts/system/memory-recall.md" with { type: "file" };
-import asset17 from "../../../prompts/system/persistent-memory.md" with { type: "file" };
-import asset18 from "../../../prompts/system/skills.md" with { type: "file" };
-import asset19 from "../../../prompts/system/triggers.md" with { type: "file" };
-import asset20 from "../../../prompts/system/workspace.md" with { type: "file" };
-import asset21 from "../../../prompts/teams.md" with { type: "file" };
-import asset22 from "../../../prompts/telegram.md" with { type: "file" };
-import asset23 from "../../../prompts/terminal.md" with { type: "file" };
+import asset16 from "../../../prompts/system/live-state.md" with { type: "file" };
+import asset17 from "../../../prompts/system/memory-recall.md" with { type: "file" };
+import asset18 from "../../../prompts/system/persistent-memory.md" with { type: "file" };
+import asset19 from "../../../prompts/system/skills.md" with { type: "file" };
+import asset20 from "../../../prompts/system/triggers.md" with { type: "file" };
+import asset21 from "../../../prompts/system/workspace.md" with { type: "file" };
+import asset22 from "../../../prompts/teams.md" with { type: "file" };
+import asset23 from "../../../prompts/telegram.md" with { type: "file" };
+import asset24 from "../../../prompts/terminal.md" with { type: "file" };
 
 /** rel path (posix, under prompts/) → embedded file path (/$bunfs/… when compiled). */
 const ASSETS: Record<string, string> = {
@@ -53,14 +54,15 @@ const ASSETS: Record<string, string> = {
   "system/daily-memory.md": asset13,
   "system/goals.md": asset14,
   "system/heartbeat-agent.md": asset15,
-  "system/memory-recall.md": asset16,
-  "system/persistent-memory.md": asset17,
-  "system/skills.md": asset18,
-  "system/triggers.md": asset19,
-  "system/workspace.md": asset20,
-  "teams.md": asset21,
-  "telegram.md": asset22,
-  "terminal.md": asset23,
+  "system/live-state.md": asset16,
+  "system/memory-recall.md": asset17,
+  "system/persistent-memory.md": asset18,
+  "system/skills.md": asset19,
+  "system/triggers.md": asset20,
+  "system/workspace.md": asset21,
+  "teams.md": asset22,
+  "telegram.md": asset23,
+  "terminal.md": asset24,
 };
 
 /** Read an embedded prompt by its rel path (e.g. "system/cron.md"). */

package/src/core/prompt/index.ts

@@ -3,6 +3,7 @@
  *
  * One stop for everything system-prompt:
  *   - `assemble`          — section pipeline + static/dynamic split
+ *   - `memory-view`       — ranked selection of `memory.md` under a budget
  *   - `templates`         — package-owned `prompts/system/*.md` loader
  *   - `workspace-listing` — lazy workspace tree for the dynamic tail
  *