talon-agent 4.6.1 → 5.0.1

package/src/backend/{shared → runtime}/index.ts

@@ -1,50 +1,56 @@
 /**
- * Shared backend framework — barrel re-export.
+ * Backend runtime library — barrel re-export.
  *
  * Helpers used by every concrete backend (`claude-sdk`, `codex`,
  * `kilo`, `opencode`, `openai-agents`) to keep behaviour aligned and
- * avoid copy-paste drift.
+ * avoid copy-paste drift. The modules sit in three groups — `turn/`
+ * (what a turn does once the SDK loop is running), `prompt/` (the text
+ * handed to the model) and `cache/` (prompt-cache telemetry) — with the
+ * cross-cutting vocabulary (`usage`, `metrics`, `frontends`, `sleep`)
+ * at the root beside this barrel.
  *
  * What's here:
- *   - `delivered-text` — scratchpad/dedup primitives.
- *   - `delivery-contract` — per-backend response-flow contract
+ *   - `turn/delivered-text` — scratchpad/dedup primitives.
+ *   - `prompt/delivery-contract` — per-backend response-flow contract
  *     (rendered from prompts/system templates), frontend-aware
  *     flow-violation reminder, first-turn nudge.
- *   - `flow-violation` — flow-violation detection + reminder text.
+ *   - `turn/flow-violation` — flow-violation detection + reminder text.
  *   - `metrics` — the shared metric vocabulary (tool calls, per-turn
  *     rollups, flow violations) with backend dimensions.
- *   - `prompt-format` — user-prompt formatter ([time] [Name] [msg_id:N]).
+ *   - `prompt/prompt-format` — user-prompt formatter
+ *     ([time] [Name] [msg_id:N]).
  *   - `frontends` — config `frontend` → messaging-frontend list.
- *   - `extractSessionName` — re-exported from `util/session-name` so the
+ *   - `extractSessionName` — re-exported from `core/weaver/session-name` so the
  *     backends keep one import site; the helper itself is frontend-neutral.
  *   - `usage` — cache-hit % + log summarisers.
- *   - `system-prompt` — per-session prompt snapshots + backend suffix
- *     (assembly itself lives in `core/prompt/`).
- *   - `model-retry` — session-expiry / context-overflow / fallback decisions.
- *   - `stream-state` — backend-agnostic accumulator for stream loops.
- *   - `turn-interrupt` — user-driven mid-turn interrupt registry (the
+ *   - `prompt/system-prompt` — per-session prompt snapshots + backend
+ *     suffix (assembly itself lives in `core/prompt/`).
+ *   - `turn/model-retry` — session-expiry / context-overflow / fallback
+ *     decisions.
+ *   - `turn/stream-state` — backend-agnostic accumulator for stream loops.
+ *   - `turn/turn-interrupt` — user-driven mid-turn interrupt registry (the
  *     shared `ChatBackend.interruptChatTurn` for callback backends).
- *   - `turn-phases` — the post-stream phases (accounting, session name,
+ *   - `turn/turn-phases` — the post-stream phases (accounting, session name,
  *     trailing-prose contract, result tail) every handler runs.
  *
  * What's NOT here (intentionally):
  *   - SDK-specific event types — those live in each backend.
  *   - Session storage — that's `src/storage/sessions.ts`.
  *   - MCP server registration — backend-specific transport details
- *     (the spawn/env contract they share is `core/tools/mcp-env.ts`).
+ *     (the spawn/env contract they share is `core/tools/ops/mcp-env.ts`).
  */
 
-export { captureDeliveredText } from "./delivered-text.js";
+export { captureDeliveredText } from "./turn/delivered-text.js";
 
-export { registerTurnInterrupt } from "./turn-interrupt.js";
+export { registerTurnInterrupt } from "./turn/turn-interrupt.js";
 
-export { formatUserPrompt } from "./prompt-format.js";
+export { formatUserPrompt } from "./prompt/prompt-format.js";
 
 export {
   buildDeliveryContract,
   buildFlowViolationReminder,
   buildFirstTurnReminder,
-} from "./delivery-contract.js";
+} from "./prompt/delivery-contract.js";
 
 export { summarizeUsage } from "./usage.js";
 
@@ -57,9 +63,12 @@ export {
   priorLookbackOverflow,
   noteLookbackRisk,
   CACHE_LOOKBACK_BLOCKS,
-} from "./cache-telemetry.js";
+} from "./cache/cache-telemetry.js";
 
-export { prepareSystemPrompt, appendBackendSuffix } from "./system-prompt.js";
+export {
+  prepareSystemPrompt,
+  appendBackendSuffix,
+} from "./prompt/system-prompt.js";
 
 export {
   createStreamState,
@@ -71,19 +80,19 @@ export {
   pushLiveUsage,
   finalizeResponseText,
   type StreamState,
-} from "./stream-state.js";
+} from "./turn/stream-state.js";
 
 export {
   routeDelivery,
   buildDeliveryFailureReminder,
   TextBlockDeliveryError,
-} from "./delivery.js";
+} from "./turn/delivery.js";
 
 export { sleep } from "./sleep.js";
 
 export { recordToolCall } from "./metrics.js";
 
-export { applyRetryDecision } from "./handle-retry.js";
+export { applyRetryDecision } from "./turn/handle-retry.js";
 
 export {
   accountTurn,
@@ -92,6 +101,6 @@ export {
   enforceTrailingProse,
   finishCallbackTurn,
   turnUsageSnapshot,
-} from "./turn-phases.js";
+} from "./turn/turn-phases.js";
 
-export { buildResultEvents } from "./result-events.js";
+export { buildResultEvents } from "./turn/result-events.js";

package/src/backend/{shared → runtime/prompt}/delivery-contract.ts

@@ -34,7 +34,7 @@
  * logic-adjacent strings, like log messages.
  */
 
-import { loadSystemTemplate } from "../../core/prompt/templates.js";
+import { loadSystemTemplate } from "../../../core/prompt/templates.js";
 
 // ── Types ───────────────────────────────────────────────────────────────────
 

package/src/backend/{shared → runtime/prompt}/prompt-format.ts

@@ -15,7 +15,7 @@
  *   DM (no msg_id): "[2026-05-15 11:01:23] actual text"
  */
 
-import { formatFullDatetime } from "../../util/time.js";
+import { formatFullDatetime } from "../../../util/time.js";
 
 // ── Public API ──────────────────────────────────────────────────────────────
 
@@ -38,8 +38,33 @@ export type PromptFormatInputs = {
   messageId?: number | string;
   /** When true, omit the leading `[YYYY-MM-DD HH:MM:SS]` tag. */
   omitTimeTag?: boolean;
+  /**
+   * Memory retrieved for THIS turn (`core/memory/turn-retrieval.ts`),
+   * already ranked, trust-filtered and budgeted. Appended after the
+   * message text under a verify-first header.
+   *
+   * This is the **only** place retrieved memory enters a prompt, and
+   * every backend reaches it through this one helper — which is what
+   * keeps #639's divergence (two backends reading the field, four
+   * silently dropping it) from coming back. It belongs to the user
+   * turn: it must never reach `prepareSystemPrompt()`, a prompt
+   * addition, or a backend `system` field, or it would break the
+   * per-session frozen prompt (plan §3.6).
+   *
+   * Absent or blank → the returned prompt is BYTE-IDENTICAL to what
+   * this helper produced before the field existed.
+   */
+  retrievedMemory?: string;
 };
 
+/**
+ * Header on the injected block. "Verify before relying on it" is
+ * deliberate: retrieval is a bm25 guess, not a fact, and the model
+ * should treat a recalled line as a lead rather than as truth.
+ */
+export const RECALLED_MEMORY_HEADER =
+  "[Recalled from memory — verify before relying on it]";
+
 /**
  * Format a user prompt for the AI backend.
  *
@@ -49,6 +74,13 @@ export type PromptFormatInputs = {
  * is on the other end.
  */
 export function formatUserPrompt(inputs: PromptFormatInputs): string {
+  return withRecalledMemory(formatMessageLine(inputs), inputs.retrievedMemory);
+}
+
+// ── Helpers ─────────────────────────────────────────────────────────────────
+
+/** The message line itself — time tag, sender label, `msg_id`, text. */
+function formatMessageLine(inputs: PromptFormatInputs): string {
   const timeTag = inputs.omitTimeTag ? "" : `[${formatFullDatetime()}]`;
   const msgIdHint =
     inputs.messageId !== undefined ? ` [msg_id:${inputs.messageId}]` : "";
@@ -68,7 +100,19 @@ export function formatUserPrompt(inputs: PromptFormatInputs): string {
   return joinNonEmpty(timeTag, inputs.text);
 }
 
-// ── Helpers ─────────────────────────────────────────────────────────────────
+/**
+ * Append the retrieved-memory block AFTER the message, never before:
+ * the user's own words stay the first thing the model reads, and the
+ * recalled lines read as an annotation on them.
+ */
+function withRecalledMemory(
+  prompt: string,
+  memory: string | undefined,
+): string {
+  const block = memory?.trim();
+  if (!block) return prompt;
+  return `${prompt}\n\n${RECALLED_MEMORY_HEADER}\n${block}`;
+}
 
 function joinNonEmpty(...parts: string[]): string {
   return parts

package/src/backend/{shared → runtime/prompt}/system-prompt.ts

@@ -39,10 +39,10 @@ import {
   joinSystemPromptParts,
   type SystemPromptParts,
   type TalonConfig,
-} from "../../core/config/index.js";
-import { getPluginPromptAdditions } from "../../core/plugin/index.js";
-import { onPromptInputsChanged } from "../../core/prompt/invalidation.js";
-import { frontendForChatId, nonTerminalFrontends } from "./frontends.js";
+} from "../../../core/config/index.js";
+import { getPluginPromptAdditions } from "../../../core/plugin/index.js";
+import { onPromptInputsChanged } from "../../../core/prompt/invalidation.js";
+import { frontendForChatId, nonTerminalFrontends } from "../frontends.js";
 
 // ── Types ───────────────────────────────────────────────────────────────────
 

package/src/backend/{shared → runtime/turn}/delivered-text.ts

@@ -25,7 +25,7 @@
  * whether to surface a flow violation.
  */
 
-import { stripMcpPrefix } from "../../core/tools/index.js";
+import { stripMcpPrefix } from "../../../core/tools/index.js";
 
 // ── Constants ───────────────────────────────────────────────────────────────
 

package/src/backend/{shared → runtime/turn}/delivery.ts

@@ -9,7 +9,7 @@
  * Routes:
  *
  *   - `tool` — a delivery tool (`end_turn` / `send` / `react`) already
- *     bridged the message to the platform via `core/tools/messaging.ts`.
+ *     bridged the message to the platform via `core/tools/chat/messaging.ts`.
  *     The message is already in the chat; we don't re-emit. We may
  *     have captured `deliveredTextNorms` for dedup, or just have
  *     `hadBridgeDelivery` set (e.g. `send(type="photo")` — non-text
@@ -35,8 +35,8 @@
 
 import type { StreamState } from "./stream-state.js";
 import { undeliveredResponseText } from "./stream-state.js";
-import { logWarn } from "../../util/log.js";
-import { incrementCounter } from "../../storage/metrics.js";
+import { logWarn } from "../../../util/log.js";
+import { incrementCounter } from "../../../storage/metrics.js";
 
 /** Route the delivery decision selected. */
 export type DeliveryRoute =

package/src/backend/{shared → runtime/turn}/handle-retry.ts

@@ -23,12 +23,12 @@
  */
 
 import type { QueryParams, QueryResult } from "./handler-types.js";
-import { classify, type TalonError } from "../../core/errors.js";
-import { logWarn } from "../../util/log.js";
-import { incrementCounter } from "../../storage/metrics.js";
-import { resetSession } from "../../storage/sessions.js";
+import { classify, type TalonError } from "../../../core/errors.js";
+import { logWarn } from "../../../util/log.js";
+import { incrementCounter } from "../../../storage/metrics.js";
+import { resetSession } from "../../../storage/sessions.js";
 import { classifyRetry } from "./model-retry.js";
-import type { AgentEvent } from "../../core/agent-runtime/events.js";
+import type { AgentEvent } from "../../../core/agent-runtime/events.js";
 
 /** Inputs for `applyRetryDecision`. */
 export interface ApplyRetryDecisionInputs {

package/src/backend/{shared → runtime/turn}/handler-to-events.ts

@@ -19,14 +19,34 @@
 import {
   type AgentEvent,
   classifiedToAgentError,
-} from "../../core/agent-runtime/events.js";
-import { classify } from "../../core/errors.js";
-import type { ChatRunParams } from "../../core/agent-runtime/capabilities.js";
+} from "../../../core/agent-runtime/events.js";
+import { classify } from "../../../core/errors.js";
+import type { ChatRunParams } from "../../../core/agent-runtime/capabilities.js";
 import type { QueryParams, QueryResult } from "./handler-types.js";
 import { buildResultEvents } from "./result-events.js";
 
 const SENTINEL = Symbol("handler-to-events:sentinel");
 
+/**
+ * The DATA half of `QueryParams`, carried verbatim from the canonical
+ * `ChatRunParams` (only `ModelRef` flattens to `model.id`). The
+ * callback half is the queue's and is built per run below. Keeping the
+ * carry in one place is what stops a field — `retrievedMemory` being
+ * the one #639 lost — from reaching some handlers and not others.
+ */
+function turnFields(params: ChatRunParams) {
+  return {
+    chatId: params.chatId,
+    model: params.model.id,
+    text: params.text,
+    senderName: params.senderName,
+    senderHandle: params.senderHandle,
+    isGroup: params.isGroup,
+    messageId: params.messageId,
+    retrievedMemory: params.retrievedMemory,
+  };
+}
+
 type QueueEvent = AgentEvent | typeof SENTINEL;
 
 /**
@@ -79,13 +99,7 @@ export async function* handlerToEvents(
   let lastAccumulated = "";
 
   const handlerParams: QueryParams = {
-    chatId: params.chatId,
-    model: params.model.id,
-    text: params.text,
-    senderName: params.senderName,
-    senderHandle: params.senderHandle,
-    isGroup: params.isGroup,
-    messageId: params.messageId,
+    ...turnFields(params),
     onStreamDelta: (accumulated) => {
       if (typeof accumulated !== "string" || accumulated.length === 0) {
         return;

package/src/backend/{shared → runtime/turn}/handler-types.ts

@@ -10,7 +10,7 @@
  * stream loop that `handler-to-events.ts` adapts onto the native
  * `AsyncIterable<AgentEvent>` shape at the factory boundary.
  *
- * Living in `backend/shared/` keeps `core/types.ts` clean of any
+ * Living in `backend/runtime/turn/` keeps `core/types.ts` clean of any
  * implementation-detail shapes, so the dispatcher / cron / triggers /
  * frontends never accidentally couple to the callback contract.
  */
@@ -35,6 +35,12 @@ export type QueryParams = {
    * Provider message ID. Telegram is numeric; Discord snowflakes are strings.
    */
   messageId?: number | string;
+  /**
+   * Per-turn retrieved memory, carried verbatim from `ChatRunParams`.
+   * The handler passes it to `formatUserPrompt` and does nothing else
+   * with it — see `backend/runtime/prompt/prompt-format.ts`.
+   */
+  retrievedMemory?: string;
   onStreamDelta?: (accumulated: string, phase?: "thinking" | "text") => void;
   onTextBlock?: (text: string) => Promise<void>;
   /**

package/src/backend/{shared → runtime/turn}/model-retry.ts

@@ -21,8 +21,8 @@
  * model swap) because those touch backend-specific storage.
  */
 
-import type { TalonError } from "../../core/errors.js";
-import { getFallbackModel } from "../../core/models/catalog.js";
+import type { TalonError } from "../../../core/errors.js";
+import { getFallbackModel } from "../../../core/models/catalog.js";
 
 // ── Public API ──────────────────────────────────────────────────────────────
 

package/src/backend/{shared → runtime/turn}/result-events.ts

@@ -5,8 +5,8 @@
  * logging imports so the adapter stays a pure event translator.
  */
 
-import type { AgentEvent } from "../../core/agent-runtime/events.js";
-import type { TokenUsageSnapshot } from "./usage.js";
+import type { AgentEvent } from "../../../core/agent-runtime/events.js";
+import type { TokenUsageSnapshot } from "../usage.js";
 
 export function buildResultEvents(inputs: {
   text: string;

package/src/backend/{shared → runtime/turn}/stream-state.ts

@@ -20,8 +20,8 @@
  */
 
 import { captureDeliveredText } from "./delivered-text.js";
-import { isTurnTerminator, stripMcpPrefix } from "../../core/tools/index.js";
-import { updateLiveTurn } from "../../storage/sessions.js";
+import { isTurnTerminator, stripMcpPrefix } from "../../../core/tools/index.js";
+import { updateLiveTurn } from "../../../storage/sessions.js";
 
 // ── State shape ─────────────────────────────────────────────────────────────
 

package/src/backend/{shared → runtime/turn}/turn-interrupt.ts

@@ -17,8 +17,8 @@
  * partial text and real usage — never as an error, never as a retry.
  */
 
-import { log, logWarn } from "../../util/log.js";
-import { incrementCounter } from "../../storage/metrics.js";
+import { log, logWarn } from "../../../util/log.js";
+import { incrementCounter } from "../../../storage/metrics.js";
 
 const interrupts = new Map<string, () => void | Promise<void>>();
 

package/src/backend/{shared → runtime/turn}/turn-phases.ts

@@ -23,10 +23,10 @@ import {
   recordUsage,
   setSessionId,
   setSessionName,
-} from "../../storage/sessions.js";
-import { log } from "../../util/log.js";
-import { extractSessionName } from "../../util/session-name.js";
-import { traceMessage } from "../../util/trace.js";
+} from "../../../storage/sessions.js";
+import { log } from "../../../util/log.js";
+import { extractSessionName } from "../../../core/weaver/session-name.js";
+import { traceMessage } from "../../../util/trace.js";
 import {
   FLOW_VIOLATION_MAX_RETRIES,
   detectFlowViolation,
@@ -37,9 +37,9 @@ import {
   recordFailedTurnAccounting,
   recordFlowViolation,
   recordTurnMetrics,
-} from "./metrics.js";
+} from "../metrics.js";
 import type { StreamState } from "./stream-state.js";
-import { summarizeUsage, type TokenUsageSnapshot } from "./usage.js";
+import { summarizeUsage, type TokenUsageSnapshot } from "../usage.js";
 
 // ── Usage snapshot ──────────────────────────────────────────────────────────
 

package/src/bootstrap.ts

@@ -10,7 +10,7 @@
  */
 
 import { loadConfig, rebuildSystemPrompt } from "./core/config/index.js";
-import { initWorkspace } from "./util/workspace.js";
+import { initWorkspace } from "./core/vfs/workspace.js";
 import { loadSessions, resetSession } from "./storage/sessions.js";
 import { loadChatSettings } from "./storage/chat-settings.js";
 import { loadCronJobs } from "./storage/cron.js";
@@ -24,19 +24,19 @@ import {
 } from "./core/engine/dispatcher.js";
 import { bus } from "./core/bus/index.js";
 import { appendToJournal } from "./storage/journal.js";
-import { initPulse, resetPulseTimer } from "./core/background/pulse.js";
-import { initCron } from "./core/background/cron.js";
-import { initPlanAlerts } from "./core/background/plan-alerts.js";
+import { initPulse, resetPulseTimer } from "./core/background/pulse/pulse.js";
+import { initCron } from "./core/background/cron/scheduler.js";
+import { initPlanAlerts } from "./core/background/pulse/plan-alerts.js";
 import { setAdminNotifier } from "./core/frontend-runtime/admin-notify.js";
 import { startAuthExpiryMonitor } from "./core/auth/expiry-monitor.js";
 import {
   initTriggers,
   resumeAfterRestart as resumeTriggersAfterRestart,
 } from "./core/background/triggers/index.js";
-import { initDream, maybeStartDream } from "./core/background/dream.js";
+import { initDream, maybeStartDream } from "./core/background/dream/index.js";
 import { initHeartbeat } from "./core/background/heartbeat/index.js";
 import { log, logWarn, logDebug } from "./util/log.js";
-import { bootPhase } from "./util/boot-timer.js";
+import { bootPhase } from "./core/daemon/boot-timer.js";
 import { mapConcurrent } from "./util/concurrency.js";
 import type { TalonConfig } from "./core/config/index.js";
 import { resolveFrontendIdAmong } from "./core/frontend-runtime/routing.js";
@@ -489,21 +489,6 @@ export async function initBackendAndDispatcher(
   // alerts above.
   wireAdminNotifier(config, frontends);
 
-  // Soul — initialize the identity kernel singleton from config so the prompt
-  // injection / dream hooks see the right enabled state. Off by default; a
-  // failure here must never block startup.
-  try {
-    const { SoulService, setSoul } = await import("./core/soul/service.js");
-    setSoul(
-      SoulService.create({
-        enabled: config.soul?.enabled ?? false,
-        ...(config.soul?.path ? { path: config.soul.path } : {}),
-      }),
-    );
-  } catch (err) {
-    log("soul", `init skipped: ${String(err)}`);
-  }
-
   // Only enable mempalace dream integration if the plugin actually registered
   let mempalaceCfg: { pythonPath: string; palacePath: string } | undefined;
   if (config.mempalace?.enabled) {

package/src/cli.ts

@@ -16,13 +16,16 @@
  * `cli/index.ts`. Everything else lives under `cli/`.
  */
 
-import { MCP_LAUNCH_SUBCOMMAND, runSupervisor } from "./util/mcp-launcher.js";
-import { LUA_RUN_SUBCOMMAND, runLuaMain } from "./core/scripting/lua-runner.js";
+import {
+  MCP_LAUNCH_SUBCOMMAND,
+  runSupervisor,
+} from "./core/mcp-hub/launcher.js";
+import { LUA_RUN_SUBCOMMAND, runLuaMain } from "./core/scripts/lua.js";
 
 // Hidden subcommand dispatch — must run before anything else. Talon
 // supervises MCP stdio children (`_mcp-launch`) and runs WASM-sandboxed
 // Lua trigger scripts (`_lua-run`) by re-invoking its own entrypoint
-// (see util/mcp-launcher.ts). Neither call resolves; the helper process
+// (see core/mcp-hub/launcher.ts). Neither call resolves; the helper process
 // exits from its own handlers.
 if (process.argv[2] === MCP_LAUNCH_SUBCOMMAND) {
   await runSupervisor(process.argv.slice(3));

package/src/core/agent-runtime/README.md

@@ -69,6 +69,21 @@ onto the SQLite layer in `src/storage/` (typed tables or the `kv`
 singleton store). Legacy files import once at boot via
 `storage/legacy-import.ts` and are renamed `*.imported`.
 
+### `agent-host.ts`
+
+The seam the Claude Agent SDK moves out of the daemon across
+(`docs/agent-host-sidecar.md`). Two halves: the NDJSON wire vocabulary
+(`HostRequest` / `HostReply` / `HostEvent` / `HostNotice`, plus
+`parseHostMessage` / `serializeHostMessage` — unknown types parse to a
+typed `unknown` the reader logs and drops, never a throw), and
+`AgentHostClient`, the interface the daemon holds. Phase 1 ships one
+implementation, `backend/claude-sdk/host/in-process.ts`; Phase 2 adds a
+process-backed one behind the same interface. Fixtures:
+`protocol/fixtures/agent-host_v1.json`.
+
+The interface and codec live here, not in `backend/`, because `core/` may
+not import `backend/` — the implementations live under `backend/`.
+
 ### `contract-tests.ts`
 
 Backend contract assertions any conforming `Backend` must pass: