talon-agent 4.6.1 → 5.0.0

package/src/core/daemon/resource-sampler.ts

@@ -0,0 +1,121 @@
+/**
+ * The daemon's accounting of its own resources — boot cost and resident
+ * memory. Measurement only: nothing here changes a prompt byte or a
+ * turn's behaviour.
+ *
+ * Why these numbers exist (docs/ts-migration-plan.md, Phase 0 — Measure):
+ * the migration plan's entry gate is "numbers in hand", and its kill
+ * criterion is stated in terms of the control plane's share of a turn plus
+ * "RSS is acceptable". Neither can be argued without a baseline, and the
+ * metrics store had latency and tokens but nothing about the process
+ * itself.
+ *
+ *   - `boot.total_ms` — process start → frontends listening. The figure
+ *     `Ready in …` already logs, kept as a distribution so successive
+ *     restarts can be compared instead of grepped.
+ *   - `boot.<phase>_ms` — each awaited startup phase (`util/boot-timer.ts`
+ *     records them), so a slow boot names its own culprit.
+ *   - `boot.rss_mb` / `boot.heap_mb` — what the process costs the moment
+ *     it is serving, before any turn has run. The floor an alternative
+ *     runtime or language would have to beat.
+ *   - `rss.mb`, `heap_used.mb`, `external.mb`, `handles.count` — the same
+ *     picture sampled every minute for the process lifetime, which is what
+ *     answers "idle RSS" and "does it creep?" (the Phase 1 soak log asks
+ *     exactly this and is currently filled in by hand).
+ *
+ * Sampling must never throw: this runs on a timer inside a live daemon,
+ * and a metric that can take the process down is worse than no metric.
+ */
+
+import { bootPhases } from "../../util/boot-timer.js";
+import { recordHistogram } from "../../storage/metrics.js";
+
+/** One minute. Idle RSS moves slowly; a tighter loop would only add noise. */
+const SAMPLE_INTERVAL_MS = 60_000;
+
+const BYTES_PER_MB = 1024 * 1024;
+
+let timer: ReturnType<typeof setInterval> | null = null;
+
+function mb(bytes: number): number {
+  return Math.round(bytes / BYTES_PER_MB);
+}
+
+/**
+ * `frontends start` → `boot.frontends_start_ms`. Phase labels are prose
+ * ("backend + dispatcher"), metric names are not.
+ */
+function slug(label: string): string {
+  return (
+    label
+      .toLowerCase()
+      .replace(/[^a-z0-9]+/g, "_")
+      .replace(/^_+|_+$/g, "") || "unnamed"
+  );
+}
+
+/** How many active handles the runtime reports, when it can report it. */
+function activeResourceCount(): number | undefined {
+  // Node 18.11+ / Bun. Absent on older runtimes, and there is no cheap
+  // supported substitute — the metric is simply skipped there.
+  const read = (process as { getActiveResourcesInfo?: () => readonly string[] })
+    .getActiveResourcesInfo;
+  if (typeof read !== "function") return undefined;
+  const info = read.call(process);
+  return Array.isArray(info) ? info.length : undefined;
+}
+
+/** Record one resident-memory sample. Safe to call at any time. */
+function sampleResources(): void {
+  try {
+    const mem = process.memoryUsage();
+    recordHistogram("rss.mb", mb(mem.rss));
+    recordHistogram("heap_used.mb", mb(mem.heapUsed));
+    recordHistogram("external.mb", mb(mem.external));
+    const handles = activeResourceCount();
+    if (handles !== undefined) recordHistogram("handles.count", handles);
+  } catch {
+    // A failed sample is a missing data point, never a failed daemon.
+  }
+}
+
+/**
+ * Fold the boot into the metrics store: the total, every phase the boot
+ * timer recorded, and the memory the process holds now that it is serving.
+ *
+ * `totalMs` defaults to process uptime — the same figure `bootReport()`
+ * prints, so the log line and the histogram cannot disagree.
+ */
+export function recordBootMetrics(
+  totalMs = Math.round(process.uptime() * 1000),
+): void {
+  try {
+    recordHistogram("boot.total_ms", totalMs);
+    for (const phase of bootPhases()) {
+      recordHistogram(`boot.${slug(phase.label)}_ms`, phase.ms);
+    }
+    const mem = process.memoryUsage();
+    recordHistogram("boot.rss_mb", mb(mem.rss));
+    recordHistogram("boot.heap_mb", mb(mem.heapUsed));
+  } catch {
+    // As above: measurement never breaks the boot it is measuring.
+  }
+}
+
+/**
+ * Start the idle sampler. Unref'd, so it never holds the event loop open
+ * and never keeps a daemon alive that is otherwise done. Idempotent —
+ * a second call is a no-op rather than a second timer.
+ */
+export function startResourceSampler(): void {
+  if (timer) return;
+  timer = setInterval(sampleResources, SAMPLE_INTERVAL_MS);
+  timer.unref();
+}
+
+/** Stop the idle sampler (shutdown, and the test seam). */
+export function stopResourceSampler(): void {
+  if (!timer) return;
+  clearInterval(timer);
+  timer = null;
+}

package/src/core/engine/dispatcher.ts

@@ -10,6 +10,7 @@
 
 import type { ExecuteParams, ExecuteResult } from "../types.js";
 import { formatRelayBlock, takePendingRelay } from "./cross-chat-relay.js";
+import { recordMessageSignal } from "../memory/taps.js";
 import { log } from "../../util/log.js";
 import { taskTable, type KillOutcome } from "../tasks/index.js";
 import { initWeaver, type Weaver, type WeaverDeps } from "../weaver/index.js";
@@ -71,9 +72,24 @@ export function stopAllTurns(): number {
  * Every turn for every frontend and every source funnels through here,
  * which makes it the one place a cross-chat reply can be folded in
  * without teaching each frontend about the relay.
+ *
+ * It is also where the memory tap lives, for the same reason: a
+ * directive is a directive whether it arrives over Telegram, Discord,
+ * WhatsApp, Teams, the native bridge or the terminal, and the tap used
+ * to see only one of them. Only `source: "message"` is tapped — a pulse,
+ * a cron job or a trigger is Talon prompting itself, and "always reply
+ * in one line" written by a scheduler is not standing human intent.
  */
 export async function execute(params: ExecuteParams): Promise<ExecuteResult> {
   if (!weaver) throw new Error("Dispatcher not initialized");
+  // Fire-and-forget: the tap is a synchronous store write that must not
+  // be able to delay or fail the turn (it swallows its own errors).
+  if (params.source === "message")
+    recordMessageSignal({
+      text: params.prompt,
+      chatKey: params.chatId,
+      ...(params.senderName ? { actor: params.senderName } : {}),
+    });
   const relayed = takePendingRelay(params.chatId);
   const prompt = relayed.length
     ? formatRelayBlock(relayed) + params.prompt

package/src/core/engine/gateway-actions/cron.ts

@@ -12,8 +12,8 @@ import {
   describeSchedule,
   nextRunAt,
 } from "../../../storage/cron.js";
-import { runJobNow } from "../../background/cron.js";
-import { parseCronSpec } from "../../background/cron-spec.js";
+import { runJobNow } from "../../background/cron/scheduler.js";
+import { parseCronSpec } from "../../background/cron/spec.js";
 import { log } from "../../../util/log.js";
 import type { Backend } from "../../agent-runtime/capabilities.js";
 import {

package/src/core/engine/gateway-actions/index.ts

@@ -12,6 +12,7 @@
  *   - `cron`      — scheduled-job CRUD
  *   - `triggers`  — long-running watcher-script CRUD
  *   - `goals`     — persistent multi-turn objectives
+ *   - `memory`    — remember / recall / forget over the typed memory store
  *   - `scripts`   — reusable agent-authored scripts
  *   - `skills`    — markdown workflows
  *   - `plugins`   — plugin hot-reload
@@ -28,6 +29,7 @@ import { fetchUrlHandlers } from "./fetch-url.js";
 import { cronHandlers } from "./cron.js";
 import { triggerHandlers } from "./triggers.js";
 import { goalHandlers } from "./goals.js";
+import { memoryHandlers } from "./memory.js";
 import { scriptHandlers } from "./scripts.js";
 import { skillHandlers } from "./skills.js";
 import { pluginHandlers } from "./plugins.js";
@@ -52,6 +54,7 @@ const handlers: SharedActionHandlers = Object.assign(Object.create(null), {
   ...cronHandlers,
   ...triggerHandlers,
   ...goalHandlers,
+  ...memoryHandlers,
   ...scriptHandlers,
   ...skillHandlers,
   ...pluginHandlers,

package/src/core/engine/gateway-actions/memory.ts

@@ -0,0 +1,335 @@
+/**
+ * Memory — the in-band write path over the typed store: remember /
+ * recall / forget (docs/memory-persona-plan.md §3.2).
+ *
+ * Three rules shape this module, and none of them live in the store:
+ *
+ *   - **Single claim, supersede-candidate.** A `remember` that restates
+ *     something already on file does NOT insert a second row. The FTS
+ *     near-duplicate probe runs first, and a hit comes back as a refusal
+ *     naming the rows to supersede. The caller then either picks one
+ *     (`replace_id`) or declares the claim genuinely separate (`force`).
+ *   - **Trust comes from the chat, not the caller.** Anything learned on
+ *     a multi-party surface is `group_chat` — never pinnable, never in
+ *     the core view (plan §5) — and a `directive` cannot be planted from
+ *     one at all. The tier also gates what a context may *overwrite*:
+ *     a claim may only be superseded or forgotten from a context at its
+ *     own tier or stronger. Without that, both `replace_id` and `forget`
+ *     are escalations — the first because `supersedeMemory` copies the
+ *     old row's trust onto the successor, the second because the store
+ *     drops whatever id it is handed.
+ *   - **Writes never invalidate the prompt cache.** This module must not
+ *     import `core/prompt/invalidation.js`. A `remember` that dropped
+ *     every live session's prompt snapshot would turn a ~50-token claim
+ *     into a 60–90 k cache write (plan §3.6). A fact learned mid-session
+ *     reaches the model through turn retrieval, not the frozen core view.
+ */
+
+import {
+  assertMemory,
+  dropMemory,
+  findSimilarMemories,
+  formatMemory,
+  getMemory,
+  isMemoryKind,
+  replaceStateKey,
+  searchMemories,
+  supersedeMemory,
+  touchMemory,
+  MEMORY_KINDS,
+  type MemoryKind,
+  type MemoryRow,
+  type MemorySource,
+  type MemoryTrust,
+} from "../../../storage/memory.js";
+import { chatScope } from "../../../util/chat-id.js";
+import { log } from "../../../util/log.js";
+import type { ActionResult } from "../../types.js";
+import type { SharedActionHandlers } from "./types.js";
+
+/** `source.actor` on every row this path writes — the ownership marker. */
+const REMEMBER_ACTOR = "remember";
+
+/** Kinds whose subject is the chat itself when the caller names none. */
+const CHAT_SCOPED_KINDS: readonly MemoryKind[] = ["episode", "relationship"];
+
+/** Hard ceiling on `recall`, regardless of what the caller asks for. */
+const MAX_RECALL_LIMIT = 20;
+
+const KIND_LIST = MEMORY_KINDS.join(", ");
+
+/**
+ * The trust tier for a claim learned in this chat.
+ *
+ * A group chat is a multi-party surface: any participant can assert
+ * anything, so what is learned there is `group_chat` — a tier the store
+ * refuses to pin and the core view never carries (plan §5). A DM with
+ * the operator, or a local terminal/native session, is the agent writing
+ * for its own principal, so that is `agent`.
+ *
+ * "Group" is read off the canonical chat-id grammar (`chatScope` in
+ * util/chat-id.ts), which is the only identity a gateway action holds.
+ * When the grammar cannot tell — Teams' `teams_chat_…` is 1:1 and group
+ * alike — this fails closed to `group_chat`: over-restricting a claim
+ * costs a pin, under-restricting one is a permanent prompt injection.
+ */
+function trustForChat(chatKey: string): MemoryTrust {
+  return chatScope(chatKey) === "dm" ? "agent" : "group_chat";
+}
+
+/** The trust tiers, weakest first. */
+const TRUST_ORDER: readonly MemoryTrust[] = [
+  "group_chat",
+  "user_claim",
+  "agent",
+  "operator",
+];
+
+function trustRank(trust: MemoryTrust): number {
+  const rank = TRUST_ORDER.indexOf(trust);
+  // A tier this module does not know is treated as the strongest, so an
+  // unrecognised row is protected rather than freely overwritten.
+  return rank === -1 ? TRUST_ORDER.length : rank;
+}
+
+/**
+ * Whether a claim made in `context` may overwrite or retire a row
+ * recorded at `row` trust — its own tier or weaker, never stronger.
+ *
+ * This is what keeps `replace_id` from being an escalation:
+ * `supersedeMemory` inherits the old row's frame, trust included, so
+ * without this check a group participant could hand new text to an
+ * `agent` row and have it land live at `agent` trust. `forget` needs the
+ * same guard for the mirror reason — the store drops whatever id it is
+ * given, so the policy has to sit here.
+ */
+function outranks(context: MemoryTrust, row: MemoryTrust): boolean {
+  return trustRank(context) >= trustRank(row);
+}
+
+function fail(error: string): ActionResult {
+  return { ok: false, error };
+}
+
+/** Never let a store error escape as a throw — the gateway wants a result. */
+function attempt(run: () => ActionResult): ActionResult {
+  try {
+    return run();
+  } catch (err) {
+    return fail(err instanceof Error ? err.message : String(err));
+  }
+}
+
+function storedLine(id: number): ActionResult {
+  const row = getMemory(id);
+  return { ok: true, id, line: row ? formatMemory(row) : `#${id}` };
+}
+
+/** Clamp a caller-supplied limit into 1..max, defaulting to max. */
+function clampLimit(raw: unknown, max: number): number {
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n < 1) return max;
+  return Math.min(Math.floor(n), max);
+}
+
+/** `episode` / `relationship` default to the chat; everything else must say. */
+function resolveSubject(
+  kind: MemoryKind,
+  raw: unknown,
+  chatKey: string,
+): string {
+  const given = raw === undefined ? "" : String(raw).trim();
+  if (given) return given;
+  return CHAT_SCOPED_KINDS.includes(kind) ? chatKey : "";
+}
+
+/** The parts of a `remember` body that survive validation. */
+type RememberClaim = {
+  kind: MemoryKind;
+  subject: string;
+  text: string;
+  trust: MemoryTrust;
+  source: MemorySource;
+  confidence?: number;
+};
+
+function parseClaim(
+  body: Record<string, unknown>,
+  chatKey: string,
+): RememberClaim | ActionResult {
+  const kind = body.kind;
+  if (!isMemoryKind(kind))
+    return fail(
+      `Unknown kind "${String(kind ?? "")}" (expected one of ${KIND_LIST})`,
+    );
+  const text = String(body.text ?? "").trim();
+  if (!text) return fail("Missing text — a memory is one written claim");
+  const subject = resolveSubject(kind, body.subject, chatKey);
+  if (!subject)
+    return fail(`A ${kind} memory needs a subject (who or what it is about)`);
+  const trust = trustForChat(chatKey);
+  if (kind === "directive" && trust !== "agent")
+    return fail(
+      "A directive cannot be recorded from a group chat — durable standing " +
+        "intent only comes from a direct conversation with the operator. " +
+        "Record it as a fact instead, or ask the operator in a DM.",
+    );
+  const confidence =
+    body.confidence === undefined ? undefined : Number(body.confidence);
+  return {
+    kind,
+    subject,
+    text,
+    trust,
+    source: { chat: chatKey, actor: REMEMBER_ACTOR },
+    ...(confidence !== undefined ? { confidence } : {}),
+  };
+}
+
+/** The keyed-state path: a write replaces the live row for that key. */
+function rememberState(
+  body: Record<string, unknown>,
+  claim: RememberClaim,
+): ActionResult {
+  if (body.replace_id !== undefined)
+    return fail(
+      "A state memory replaces the live row for its key — omit replace_id",
+    );
+  const key = String(body.key ?? "").trim();
+  if (!key)
+    return fail("A state memory requires a key (e.g. heartbeat.health)");
+  const id = replaceStateKey(key, claim.text, claim.source, {
+    subject: claim.subject,
+    trust: claim.trust,
+    ...(claim.confidence !== undefined ? { confidence: claim.confidence } : {}),
+    reason: "remember: state",
+  });
+  return storedLine(id);
+}
+
+/** The explicit supersede: fold the new text into an existing live row. */
+function rememberReplacing(
+  replaceId: number,
+  claim: RememberClaim,
+): ActionResult {
+  const target = getMemory(replaceId);
+  if (!target) return fail(`No memory with id ${replaceId}`);
+  if (target.droppedAt !== undefined)
+    return fail(`Memory #${replaceId} is dropped; it cannot be superseded`);
+  if (target.supersededBy !== undefined)
+    return fail(
+      `Memory #${replaceId} is already superseded by #${target.supersededBy}`,
+    );
+  if (!outranks(claim.trust, target.trust))
+    return fail(
+      `Memory #${replaceId} was recorded at ${target.trust} trust; a ${claim.trust} context cannot replace it — record a separate claim instead`,
+    );
+  if (target.kind !== claim.kind)
+    return fail(
+      `Memory #${replaceId} is a ${target.kind}, not a ${claim.kind} — a supersede keeps the row's kind`,
+    );
+  return storedLine(
+    supersedeMemory(replaceId, claim.text, "remember: replaced"),
+  );
+}
+
+/** The refusal that offers supersede-instead-of-append (plan §3.2). */
+function nearDuplicate(similar: MemoryRow[]): ActionResult {
+  return {
+    ok: false,
+    error: `Near-duplicate of #${similar[0]!.id}`,
+    similar: similar.map((row) => formatMemory(row)),
+    hint: "pass replace_id to supersede it, or force: true to store a separate claim",
+  };
+}
+
+function rememberClaim(
+  body: Record<string, unknown>,
+  chatKey: string,
+): ActionResult {
+  const parsed = parseClaim(body, chatKey);
+  if ("ok" in parsed) return parsed;
+  if (parsed.kind === "state") return rememberState(body, parsed);
+
+  if (body.replace_id !== undefined) {
+    const replaceId = Number(body.replace_id);
+    if (!Number.isInteger(replaceId) || replaceId <= 0)
+      return fail(`Invalid replace_id "${String(body.replace_id)}"`);
+    return rememberReplacing(replaceId, parsed);
+  }
+
+  if (body.force !== true) {
+    const similar = findSimilarMemories(
+      parsed.kind,
+      parsed.subject,
+      parsed.text,
+    );
+    if (similar.length > 0) return nearDuplicate(similar);
+  }
+  const { id } = assertMemory(parsed);
+  log("gateway", `remember: [${parsed.kind}] ${parsed.subject} #${id}`);
+  return storedLine(id);
+}
+
+function recallRows(body: Record<string, unknown>): ActionResult {
+  const query = String(body.query ?? "").trim();
+  if (!query) return fail("Missing query");
+  const kind = body.kind;
+  if (kind !== undefined && !isMemoryKind(kind))
+    return fail(
+      `Unknown kind "${String(kind)}" (expected one of ${KIND_LIST})`,
+    );
+  const rows = searchMemories(query, {
+    ...(kind !== undefined ? { kind } : {}),
+    limit: clampLimit(body.limit, MAX_RECALL_LIMIT),
+  });
+  if (rows.length === 0)
+    return { ok: true, rows: [], note: "nothing stored matches" };
+  // A retrieval hit is ranking feedback, not a content change: `touchMemory`
+  // writes no history row, and nothing here invalidates the prompt cache.
+  for (const row of rows) touchMemory(row.id);
+  return { ok: true, rows: rows.map((row) => formatMemory(row)) };
+}
+
+/** Why a context is not allowed to retire this row. */
+function forgetRefusal(
+  id: number,
+  rowTrust: MemoryTrust,
+  context: MemoryTrust,
+): string {
+  // No chat context ever reaches `operator`, so an operator row can only
+  // be retired out of band — name the command rather than stonewalling.
+  if (rowTrust === "operator")
+    return `Memory #${id} is an operator memory — only the operator can forget it (talon memory forget ${id})`;
+  return `Memory #${id} was recorded at ${rowTrust} trust; a ${context} context cannot forget it`;
+}
+
+function forgetRow(
+  body: Record<string, unknown>,
+  chatKey: string,
+): ActionResult {
+  const id = Number(body.id);
+  if (!Number.isInteger(id) || id <= 0)
+    return fail(`Invalid id "${String(body.id ?? "")}"`);
+  const reason = String(body.reason ?? "").trim();
+  if (!reason)
+    return fail(
+      "A reason is required — every drop is auditable and reversible",
+    );
+  const row = getMemory(id);
+  if (!row) return fail(`No memory with id ${id}`);
+  const context = trustForChat(chatKey);
+  if (!outranks(context, row.trust))
+    return fail(forgetRefusal(id, row.trust, context));
+  dropMemory(id, reason);
+  log("gateway", `forget: #${id} (${reason})`);
+  return { ok: true, id, text: `Dropped #${id} to the graveyard: ${reason}` };
+}
+
+export const memoryHandlers: SharedActionHandlers = {
+  remember: (body, _chatId, _backend, chatKey) =>
+    attempt(() => rememberClaim(body, chatKey)),
+  recall: (body) => attempt(() => recallRows(body)),
+  forget: (body, _chatId, _backend, chatKey) =>
+    attempt(() => forgetRow(body, chatKey)),
+};

package/src/core/engine/gateway.ts

@@ -30,7 +30,6 @@ import {
 } from "./gateway-routes.js";
 import { handlePluginAction } from "../plugin/index.js";
 import type { FrontendActionHandler } from "../types.js";
-import { BOT_MESSAGE_ACTIONS, noteBotMessage } from "../soul/taps.js";
 import type { Backend } from "../agent-runtime/capabilities.js";
 import { resolveOwnerFrontendId } from "../frontend-runtime/routing.js";
 
@@ -320,11 +319,6 @@ export class Gateway {
       if (frontendHandler) {
         const result = await frontendHandler(body, chatId);
         if (result) {
-          // Soul tap: remember our own outgoing message ids so a later
-          // reaction update can be attributed to one of Talon's messages.
-          // No-op unless the soul is enabled.
-          if (BOT_MESSAGE_ACTIONS.has(action) && result.ok && result.message_id)
-            noteBotMessage(chatId, Number(result.message_id));
           logDebug("gateway", `${action} chat=${chatId} ${Date.now() - t0}ms`);
           return result;
         }

package/src/core/memory/import.ts

@@ -352,8 +352,9 @@ export function importMemoryFile(path: string = files.memory): ImportCounts {
 
 /**
  * Import the daily notes: one `episode` per `YYYY-MM-DD.md`, subject the
- * date. `diary-*.md` is the soul's first-person writing — a reflection,
- * never a fact source (plan §3.1) — so the filename pattern excludes it.
+ * date. `diary-*.md` is first-person writing — a reflection, never a fact
+ * source (plan §3.1) — so the filename pattern excludes it. The soul kernel
+ * that wrote those files is gone, but its output is still on disk.
  */
 export function importDailyNotes(dir: string = dirs.dailyMemory): ImportCounts {
   if (!existsSync(dir)) return emptyCounts();