talon-agent 4.4.0 → 4.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "talon-agent",
-  "version": "4.4.0",
+  "version": "4.6.0",
   "description": "Multi-frontend AI agent with full tool access, streaming, cron jobs, and plugin system",
   "author": "Dylan Neve",
   "license": "MIT",

package/prompts/system/memory-core-view.md

@@ -0,0 +1,7 @@
+## Persistent Memory
+
+The following is your memory store's core view — the durable claims that matter most: pinned items, standing directives, who you are talking to, your highest-salience facts, and current status. It is selected and budgeted, not a whole file, so anything not here is not gone. The rest of the store is one query away (`talon memory list`, or `/memory` in chat), and the full rendered projection is at ~/.talon/workspace/memory/memory.md.
+
+Durable facts, not live status. Reference it naturally; the Memory and Recall policy in this prompt governs how you add to it and keep it current.
+
+{{content}}

package/src/cli/index.ts

@@ -157,7 +157,7 @@ export async function runCli(): Promise<void> {
         `    ${pc.cyan("skill")}      Manage skills (install/enable/disable)`,
       );
       console.log(
-        `    ${pc.cyan("memory")}     Read/edit the memory store (list/search/remember)`,
+        `    ${pc.cyan("memory")}     Read/edit the memory store (list/search/import/render)`,
       );
       console.log(`    ${pc.cyan("config")}     View/edit configuration`);
       console.log(`    ${pc.cyan("logs")}       Tail log file`);

package/src/cli/memory.ts

@@ -8,6 +8,11 @@
  */
 
 import pc from "picocolors";
+import { importAll } from "../core/memory/import.js";
+import {
+  renderMemoryMarkdown,
+  writeRenderedMemory,
+} from "../core/memory/render.js";
 import {
   assertMemory,
   dropMemory,
@@ -33,6 +38,8 @@ const USAGE = [
   `    ${pc.cyan("remember <kind> <subject> <text>")} Record an operator claim`,
   `    ${pc.cyan("forget <id> [reason]")}           Drop a row to the graveyard`,
   `    ${pc.cyan("state <key> <text>")}             Replace the row for a state key`,
+  `    ${pc.cyan("import")}                         Fold memory.md + daily notes into the store`,
+  `    ${pc.cyan("render [--write]")}               Render the store as memory.md`,
   "",
   `  Kinds: ${MEMORY_KINDS.join(", ")}`,
   "",
@@ -134,6 +141,24 @@ function cmdState(args: readonly string[]): void {
   console.log(`  ${pc.green("●")} ${key} is now #${id}\n`);
 }
 
+function cmdImport(): void {
+  const { inserted, superseded, skipped } = importAll();
+  console.log(
+    `  ${pc.green("●")} Imported: ${inserted} new, ${superseded} updated, ${skipped} unchanged\n`,
+  );
+}
+
+function cmdRender(args: readonly string[]): void {
+  if (!args.includes("--write")) {
+    console.log(renderMemoryMarkdown());
+    return;
+  }
+  const { path, bytes, backup } = writeRenderedMemory();
+  console.log(`  ${pc.green("●")} Wrote ${bytes} chars to ${path}`);
+  if (backup) console.log(`  ${pc.dim(`Previous file archived to ${backup}`)}`);
+  console.log();
+}
+
 /** Route a `talon memory <command>` invocation. */
 export function runMemoryCommand(args: readonly string[]): void {
   try {
@@ -157,6 +182,12 @@ export function runMemoryCommand(args: readonly string[]): void {
       case "state":
         cmdState(args.slice(1));
         break;
+      case "import":
+        cmdImport();
+        break;
+      case "render":
+        cmdRender(args.slice(1));
+        break;
       default:
         console.log(USAGE);
     }

package/src/core/memory/core-view.ts

@@ -0,0 +1,158 @@
+/**
+ * The **core view** — the store's contribution to the static system
+ * prompt (docs/memory-persona-plan.md §3.4).
+ *
+ * Retrieval has two tiers, and the boundary between them is a
+ * prompt-cache invariant:
+ *
+ *   - **Core view** (this module) — pinned directives, the relationship
+ *     layer and the top durable facts plus fresh `state`, computed
+ *     **once per session build** and living in `staticText`. Budgeted,
+ *     never truncated mid-section.
+ *   - **Turn retrieval** (PR 8) — keyed to the incoming message and
+ *     injected into the *user turn*, never into `prepareSystemPrompt()`.
+ *
+ * Because `prepareSystemPrompt` freezes the assembled prompt per
+ * `(chatId, sessionEpoch)`, a fact learned at turn 3 cannot appear in
+ * that session's core view. That is the design, not a bug: reaching for
+ * `notifyPromptInputsChanged()` to "fix" it would force a full-prompt
+ * cache write across every live session on every learned fact (plan
+ * §3.6). Nothing here may ever invalidate a snapshot.
+ *
+ * The store is consulted exactly once per build — one `listMemories`
+ * call, ranked and filtered in memory — so the cost of the core view is
+ * one query per session, not one per kind and not one per turn.
+ */
+
+import { listMemories, type MemoryRow } from "../../storage/memory.js";
+import { renderMemoryMarkdown } from "./render.js";
+
+// ── Tunables ────────────────────────────────────────────────────────────────
+
+/**
+ * Char budget for the whole core view (~2 k tokens). Deliberately well
+ * under the 12 k file-injection cap: this block is paid for by every
+ * session on every frontend, and the rest of the store is a `talon
+ * memory list` away.
+ */
+export const CORE_VIEW_MAX_CHARS = 8_000;
+
+/**
+ * A `state` row older than this is stale and left out — a status
+ * snapshot nobody has touched in a week is noise in a prompt, and the
+ * heartbeat rewrites the live ones far more often than that. Pinning
+ * overrides it, which is what pinning is for.
+ */
+const STATE_FRESH_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * Rows pulled from the store before ranking. Generous: the budget, not
+ * this number, is what decides how much reaches the prompt.
+ */
+const CORE_VIEW_ROW_LIMIT = 500;
+
+/**
+ * Kinds eligible for the core view, most durable first. `episode` and
+ * `reflection` are absent on purpose (plan §3.1): episodes decay fast
+ * and are a retrieval-only source, and the diary is the persona layer
+ * and never a fact source. Neither enters the core view — not even
+ * pinned, since a pin must not promote a kind past the tier boundary.
+ */
+const CORE_KINDS: readonly MemoryRow["kind"][] = [
+  "directive",
+  "relationship",
+  "fact",
+];
+
+// ── Selection ───────────────────────────────────────────────────────────────
+
+/** Salience first, then recency; id breaks the tie so the order is stable. */
+function byRank(a: MemoryRow, b: MemoryRow): number {
+  return b.salience - a.salience || b.lastSeenAt - a.lastSeenAt || a.id - b.id;
+}
+
+function eligible(row: MemoryRow, now: number): boolean {
+  if (row.kind === "state")
+    return row.pinned || now - row.lastSeenAt <= STATE_FRESH_MS;
+  return CORE_KINDS.includes(row.kind);
+}
+
+/**
+ * Take rows until the budget is spent. A coarse pre-cut on row text
+ * only: the render does the exact accounting and cuts on section
+ * boundaries, so this exists to keep a 500-row store from being ranked
+ * into a document that can hold a fraction of it.
+ */
+function withinBudget(rows: readonly MemoryRow[], budget: number): MemoryRow[] {
+  const taken: MemoryRow[] = [];
+  let spent = 0;
+  for (const row of rows) {
+    if (spent > budget) break;
+    spent += row.text.length;
+    taken.push(row);
+  }
+  return taken;
+}
+
+/** Options shared by the selection and the render. */
+export type CoreViewOptions = {
+  /** Char budget for the rendered view. Defaults to `CORE_VIEW_MAX_CHARS`. */
+  budget?: number;
+  /** Clock for the `state` freshness window (tests pin it). */
+  now?: number;
+};
+
+/**
+ * The ordered rows behind the core view: pinned first (any eligible
+ * kind), then directives, then the relationship layer, then facts by
+ * salience, then fresh `state`.
+ *
+ * The render re-sorts with the same ranking (`render.ts` `compareRows`),
+ * so this order is what reaches the prompt rather than merely what is
+ * handed over — but the selection is asserted here, where the tiers are
+ * decided, and not through the rendered bytes.
+ */
+export function selectCoreRows(opts: CoreViewOptions = {}): MemoryRow[] {
+  const now = opts.now ?? Date.now();
+  const live = listMemories({ limit: CORE_VIEW_ROW_LIMIT }).filter((row) =>
+    eligible(row, now),
+  );
+  const tiers: MemoryRow[][] = [
+    live.filter((row) => row.pinned).sort(byRank),
+    ...CORE_KINDS.map((kind) =>
+      live.filter((row) => !row.pinned && row.kind === kind).sort(byRank),
+    ),
+    live.filter((row) => !row.pinned && row.kind === "state").sort(byRank),
+  ];
+  const seen = new Set<number>();
+  const ordered = tiers.flat().filter((row) => {
+    if (seen.has(row.id)) return false;
+    seen.add(row.id);
+    return true;
+  });
+  return withinBudget(ordered, opts.budget ?? CORE_VIEW_MAX_CHARS);
+}
+
+// ── Rendering ───────────────────────────────────────────────────────────────
+
+/** A rendered core view, with what it cost to carry. */
+export type CoreView = {
+  text: string;
+  /** Rows selected into the view (0 means the store had nothing to say). */
+  rows: number;
+  /** Rendered size — the number recorded as `prompt.memory_chars`. */
+  chars: number;
+};
+
+/**
+ * Render the core view as a `memory.md`-shaped document, reusing the
+ * projection renderer so the store reads identically in the prompt and
+ * in the file. Budgeted, not truncated: sections are dropped whole from
+ * the bottom of the ranking and the tail is named.
+ */
+export function renderCoreView(opts: CoreViewOptions = {}): CoreView {
+  const budget = opts.budget ?? CORE_VIEW_MAX_CHARS;
+  const rows = selectCoreRows({ ...opts, budget });
+  const text = renderMemoryMarkdown({ rows, budget });
+  return { text, rows: rows.length, chars: text.length };
+}

package/src/core/memory/flag.ts

@@ -0,0 +1,16 @@
+/**
+ * The one flag that gates the typed memory store's read path
+ * (docs/memory-persona-rollout.md, "One flag gates the whole new path").
+ *
+ * Kept in its own module because `core/prompt/assemble.ts` asks the
+ * question but must not reach into `storage/` to answer it: the prompt
+ * layer sees the flag and the core view, nothing else.
+ *
+ * Read per call, never cached: tests flip it between builds, and the
+ * cost is an env lookup on a path that already reads files from disk.
+ */
+
+/** True when `TALON_MEMORY_STORE=1`. Off by default until PR 9 is measured. */
+export function memoryStoreEnabled(): boolean {
+  return process.env.TALON_MEMORY_STORE === "1";
+}

package/src/core/memory/import.ts

@@ -0,0 +1,380 @@
+/**
+ * `memory.md` and the daily notes → typed memory rows.
+ *
+ * The store (storage/memory.ts) becomes the source of truth in PR 8, but
+ * the markdown files hold years of the operator's own writing and stay
+ * hand-editable afterwards. This module is the bridge: it seeds the store
+ * from the files, and — because it is idempotent by content hash — it is
+ * also how a hand-edit to the rendered `memory.md` folds back in
+ * (plan §3.1, rollout PR 5).
+ *
+ * Three rules make re-running it safe:
+ *
+ *   - **The parser is PR 1's.** Sections, families and tiers come from
+ *     prompt/memory-view.ts, so what the prompt considers one status
+ *     family is what the store considers one keyed state row. Nothing is
+ *     classified twice, two different ways.
+ *   - **Content hash decides.** A live import-owned row with the same
+ *     kind + subject + key and the same hash is left alone; a different
+ *     hash is a hand-edit, and supersedes the row so the old text stays
+ *     in the audit trail. Only a subject never seen before inserts.
+ *   - **The files are never written.** Import reads; render writes. A
+ *     failed or partial import loses nothing.
+ *
+ * Kind follows the heading's tier, because the tier already encodes the
+ * lifecycle: directives are intent, status snapshots are keyed state,
+ * historical sections are episodes, people sections are relationships,
+ * everything else is a durable fact.
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { basename, join } from "node:path";
+
+import {
+  MAX_SUBJECT_LENGTH,
+  MAX_TEXT_LENGTH,
+  assertMemory,
+  listMemories,
+  memoryContentHash,
+  replaceStateKey,
+  supersedeMemory,
+  type MemoryKind,
+  type MemoryRow,
+  type MemorySource,
+} from "../../storage/memory.js";
+import { dirs, files } from "../../util/paths.js";
+import {
+  TIER_ORDER,
+  classify,
+  collapseFamilies,
+  familyKey,
+  parseSections,
+  type Section,
+  type Tier,
+} from "../prompt/memory-view.js";
+
+// ── Tunables ────────────────────────────────────────────────────────────────
+
+/** `source.actor` on every row this module writes — the ownership marker. */
+const IMPORT_ACTOR = "import";
+
+/** Recorded on the supersede that a changed section causes. */
+const RE_IMPORT_REASON = "re-import: file changed";
+
+/** Daily notes only: `YYYY-MM-DD.md`, which excludes `diary-*.md`. */
+const DAILY_NOTE = /^(\d{4}-\d{2}-\d{2})\.md$/;
+
+/**
+ * A heading that is just a date is a day's episode — which is how the
+ * render emits an imported daily note, so reading one back produces the
+ * episode it came from rather than a fresh fact.
+ */
+const DATE_HEADING = /^\d{4}-\d{2}-\d{2}$/;
+
+/** How many same-subject rows are scanned for the import-owned one. */
+const LOOKUP_LIMIT = 200;
+
+/** Room left for a ` (n/N)` chunk or ` #n` duplicate suffix on a subject. */
+const SUBJECT_SUFFIX_ROOM = 16;
+
+/** Cap on a generated state key, inside the store's own 100-char limit. */
+const MAX_SLUG_LENGTH = 90;
+
+/**
+ * The lifecycle each heading tier maps to. `active` and `general` are
+ * durable knowledge with no special lifecycle, so both land as facts.
+ */
+const KIND_BY_TIER: Readonly<Record<Tier, MemoryKind>> = {
+  directive: "directive",
+  people: "relationship",
+  active: "fact",
+  general: "fact",
+  status: "state",
+  historical: "episode",
+};
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+/** What one import run did, per row. */
+export type ImportCounts = {
+  inserted: number;
+  superseded: number;
+  skipped: number;
+};
+
+/** One row an import wants to land — the unit of the idempotency check. */
+type Claim = {
+  kind: MemoryKind;
+  subject: string;
+  key?: string;
+  text: string;
+  source: MemorySource;
+};
+
+const emptyCounts = (): ImportCounts => ({
+  inserted: 0,
+  superseded: 0,
+  skipped: 0,
+});
+
+function addCounts(into: ImportCounts, from: ImportCounts): ImportCounts {
+  return {
+    inserted: into.inserted + from.inserted,
+    superseded: into.superseded + from.superseded,
+    skipped: into.skipped + from.skipped,
+  };
+}
+
+// ── Text shaping ────────────────────────────────────────────────────────────
+
+/**
+ * A state key from a family key: `Inbox / CI Watch` → `inbox.ci-watch`.
+ * The slash is the family separator the operator already writes, so it
+ * becomes the key's dot; everything else collapses to a dash.
+ */
+function slugKey(family: string): string {
+  const slug = family
+    .toLowerCase()
+    .replace(/\s*\/\s*/g, ".")
+    .replace(/\s+/g, "-")
+    .replace(/[^a-z0-9_.-]/g, "")
+    .replace(/-{2,}/g, "-")
+    .replace(/\.{2,}/g, ".")
+    .replace(/^[-._]+|[-._]+$/g, "");
+  return slug.slice(0, MAX_SLUG_LENGTH) || "section";
+}
+
+/**
+ * Demote `## ` headings inside a claim's body to `### `.
+ *
+ * A daily note is a whole document and routinely carries its own `## `
+ * headings. Stored verbatim, the render would emit them at the top level
+ * and the next import would read one row back as several sections — and
+ * an embedded `## Active Investigations` would collide with the real
+ * section of that name and supersede it. A claim's body therefore never
+ * holds a top-level heading. Running this twice changes nothing, which is
+ * what keeps the round trip a fixed point.
+ */
+function demoteHeadings(body: string): string {
+  return body.replace(/^##(?=\s)/gm, "###");
+}
+
+/** Blank-line-separated paragraphs, trimmed and normalized. */
+function paragraphsOf(text: string): string[] {
+  return demoteHeadings(text)
+    .split(/\n\s*\n/)
+    .map((para) => para.trim())
+    .filter(Boolean);
+}
+
+/** A single paragraph bigger than the cap — the one place text is lost. */
+function hardCut(para: string): string {
+  return `${para.slice(0, MAX_TEXT_LENGTH - 1).trimEnd()}…`;
+}
+
+/**
+ * Pack paragraphs into chunks of at most `MAX_TEXT_LENGTH`. Splitting on
+ * paragraph boundaries keeps every chunk readable on its own — a claim
+ * cut mid-sentence is worse than no claim. Rejoining the chunks with a
+ * blank line reproduces the input exactly, which is what makes
+ * import → render → import a fixed point.
+ */
+function chunkParagraphs(paras: readonly string[]): string[] {
+  const chunks: string[] = [];
+  let current = "";
+  for (const para of paras) {
+    const piece = para.length > MAX_TEXT_LENGTH ? hardCut(para) : para;
+    const joined = current ? `${current}\n\n${piece}` : piece;
+    if (joined.length <= MAX_TEXT_LENGTH) {
+      current = joined;
+      continue;
+    }
+    if (current) chunks.push(current);
+    current = piece;
+  }
+  if (current) chunks.push(current);
+  return chunks;
+}
+
+// ── Claims ──────────────────────────────────────────────────────────────────
+
+/** Clamp a subject, leaving room for the suffixes appended below. */
+function clampSubject(subject: string): string {
+  return subject.slice(0, MAX_SUBJECT_LENGTH - SUBJECT_SUFFIX_ROOM).trim();
+}
+
+/**
+ * Two sections can share a heading. Without a disambiguator the second
+ * would supersede the first on every run and vice versa, so occurrence
+ * `n > 1` gets a ` #n` marker — deliberately not a parenthetical, which
+ * `familyKey` would strip straight back off on the next import.
+ */
+function disambiguate(
+  seen: Map<string, number>,
+  kind: MemoryKind,
+  subject: string,
+): string {
+  const slot = `${kind}|${subject}`;
+  const occurrence = (seen.get(slot) ?? 0) + 1;
+  seen.set(slot, occurrence);
+  return occurrence === 1 ? subject : `${subject} #${occurrence}`;
+}
+
+/** Spread one section's text across as many rows as the cap needs. */
+function claimsFrom(
+  base: { kind: MemoryKind; subject: string; key?: string },
+  paras: readonly string[],
+  source: MemorySource,
+): Claim[] {
+  const chunks = chunkParagraphs(paras);
+  const many = chunks.length > 1;
+  return chunks.map((text, index) => {
+    const nth = index + 1;
+    const subject = many
+      ? `${base.subject} (${nth}/${chunks.length})`
+      : base.subject;
+    const key =
+      base.key !== undefined && many && nth > 1
+        ? `${base.key}.${nth}`
+        : base.key;
+    return {
+      kind: base.kind,
+      subject,
+      text,
+      source,
+      ...(key !== undefined ? { key } : {}),
+    };
+  });
+}
+
+/**
+ * One `## ` section → its rows. A `state` section's subject is its key:
+ * the render emits keyed rows under their key, so anchoring both on the
+ * slug is what keeps import → render → import a no-op.
+ */
+function sectionClaims(
+  section: Section,
+  source: MemorySource,
+  seen: Map<string, number>,
+): Claim[] {
+  const dated = DATE_HEADING.test(section.title);
+  const tier = TIER_ORDER[classify(section.title)] ?? "general";
+  const kind = dated ? "episode" : KIND_BY_TIER[tier];
+  const family = familyKey(section.title) || section.title;
+  // People sections are per-person and a dated section is one day, so the
+  // heading itself is the subject; everything else is keyed on the family
+  // so a family's snapshots line up on one row.
+  const named = dated || tier === "people" ? section.title : family;
+  const key = kind === "state" ? slugKey(family) : undefined;
+  const subject = disambiguate(
+    seen,
+    kind,
+    clampSubject(key ?? named) || "untitled",
+  );
+  const newline = section.body.indexOf("\n");
+  const paras = paragraphsOf(
+    newline === -1 ? "" : section.body.slice(newline + 1),
+  );
+  if (paras.length === 0) return [];
+  const anchor = key === undefined ? { kind, subject } : { kind, subject, key };
+  return claimsFrom(anchor, paras, source);
+}
+
+// ── Landing ─────────────────────────────────────────────────────────────────
+
+/** The live row this claim owns, if it has landed before. */
+function liveImportRow(claim: Claim): MemoryRow | undefined {
+  return listMemories({
+    kind: claim.kind,
+    subject: claim.subject,
+    limit: LOOKUP_LIMIT,
+  }).find((row) => row.source.actor === IMPORT_ACTOR && row.key === claim.key);
+}
+
+/**
+ * Land one claim: skip an unchanged row, supersede a changed one, insert
+ * a new one. Keyed state goes in through `replaceStateKey`, so the newest
+ * snapshot of a family is the single live row for its key.
+ */
+function landClaim(claim: Claim): keyof ImportCounts {
+  const hash = memoryContentHash(
+    claim.kind,
+    claim.subject,
+    claim.key,
+    claim.text,
+  );
+  const existing = liveImportRow(claim);
+  if (existing?.contentHash === hash) return "skipped";
+  if (existing) {
+    supersedeMemory(existing.id, claim.text, RE_IMPORT_REASON);
+    return "superseded";
+  }
+  if (claim.kind === "state" && claim.key !== undefined) {
+    replaceStateKey(claim.key, claim.text, claim.source, {
+      subject: claim.subject,
+      trust: "operator",
+    });
+  } else {
+    assertMemory({
+      kind: claim.kind,
+      subject: claim.subject,
+      text: claim.text,
+      source: claim.source,
+      trust: "operator",
+    });
+  }
+  return "inserted";
+}
+
+function landAll(claims: Iterable<Claim>): ImportCounts {
+  const counts = emptyCounts();
+  for (const claim of claims) counts[landClaim(claim)] += 1;
+  return counts;
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Import `memory.md`: one row per `## ` section, families collapsed
+ * first so only a status family's newest snapshot becomes its live state
+ * row. A missing file is not an error — a fresh install has none.
+ */
+export function importMemoryFile(path: string = files.memory): ImportCounts {
+  if (!existsSync(path)) return emptyCounts();
+  const content = readFileSync(path, "utf8");
+  const source: MemorySource = { actor: IMPORT_ACTOR, chat: basename(path) };
+  const { kept } = collapseFamilies(parseSections(content).sections);
+  const seen = new Map<string, number>();
+  return landAll(
+    kept.flatMap((section) => sectionClaims(section, source, seen)),
+  );
+}
+
+/**
+ * Import the daily notes: one `episode` per `YYYY-MM-DD.md`, subject the
+ * date. `diary-*.md` is the soul's first-person writing — a reflection,
+ * never a fact source (plan §3.1) — so the filename pattern excludes it.
+ */
+export function importDailyNotes(dir: string = dirs.dailyMemory): ImportCounts {
+  if (!existsSync(dir)) return emptyCounts();
+  let counts = emptyCounts();
+  for (const name of readdirSync(dir).sort()) {
+    const date = DAILY_NOTE.exec(name)?.[1];
+    if (!date) continue;
+    const paras = paragraphsOf(readFileSync(join(dir, name), "utf8"));
+    if (paras.length === 0) continue;
+    const source: MemorySource = { actor: IMPORT_ACTOR, chat: name };
+    const claims = claimsFrom(
+      { kind: "episode", subject: date },
+      paras,
+      source,
+    );
+    counts = addCounts(counts, landAll(claims));
+  }
+  return counts;
+}
+
+/** Both sources, one set of counts — what `talon memory import` runs. */
+export function importAll(): ImportCounts {
+  return addCounts(importMemoryFile(), importDailyNotes());
+}