talon-agent 4.3.3 → 4.5.0

package/src/core/memory/import.ts

@@ -0,0 +1,380 @@
+/**
+ * `memory.md` and the daily notes → typed memory rows.
+ *
+ * The store (storage/memory.ts) becomes the source of truth in PR 8, but
+ * the markdown files hold years of the operator's own writing and stay
+ * hand-editable afterwards. This module is the bridge: it seeds the store
+ * from the files, and — because it is idempotent by content hash — it is
+ * also how a hand-edit to the rendered `memory.md` folds back in
+ * (plan §3.1, rollout PR 5).
+ *
+ * Three rules make re-running it safe:
+ *
+ *   - **The parser is PR 1's.** Sections, families and tiers come from
+ *     prompt/memory-view.ts, so what the prompt considers one status
+ *     family is what the store considers one keyed state row. Nothing is
+ *     classified twice, two different ways.
+ *   - **Content hash decides.** A live import-owned row with the same
+ *     kind + subject + key and the same hash is left alone; a different
+ *     hash is a hand-edit, and supersedes the row so the old text stays
+ *     in the audit trail. Only a subject never seen before inserts.
+ *   - **The files are never written.** Import reads; render writes. A
+ *     failed or partial import loses nothing.
+ *
+ * Kind follows the heading's tier, because the tier already encodes the
+ * lifecycle: directives are intent, status snapshots are keyed state,
+ * historical sections are episodes, people sections are relationships,
+ * everything else is a durable fact.
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { basename, join } from "node:path";
+
+import {
+  MAX_SUBJECT_LENGTH,
+  MAX_TEXT_LENGTH,
+  assertMemory,
+  listMemories,
+  memoryContentHash,
+  replaceStateKey,
+  supersedeMemory,
+  type MemoryKind,
+  type MemoryRow,
+  type MemorySource,
+} from "../../storage/memory.js";
+import { dirs, files } from "../../util/paths.js";
+import {
+  TIER_ORDER,
+  classify,
+  collapseFamilies,
+  familyKey,
+  parseSections,
+  type Section,
+  type Tier,
+} from "../prompt/memory-view.js";
+
+// ── Tunables ────────────────────────────────────────────────────────────────
+
+/** `source.actor` on every row this module writes — the ownership marker. */
+const IMPORT_ACTOR = "import";
+
+/** Recorded on the supersede that a changed section causes. */
+const RE_IMPORT_REASON = "re-import: file changed";
+
+/** Daily notes only: `YYYY-MM-DD.md`, which excludes `diary-*.md`. */
+const DAILY_NOTE = /^(\d{4}-\d{2}-\d{2})\.md$/;
+
+/**
+ * A heading that is just a date is a day's episode — which is how the
+ * render emits an imported daily note, so reading one back produces the
+ * episode it came from rather than a fresh fact.
+ */
+const DATE_HEADING = /^\d{4}-\d{2}-\d{2}$/;
+
+/** How many same-subject rows are scanned for the import-owned one. */
+const LOOKUP_LIMIT = 200;
+
+/** Room left for a ` (n/N)` chunk or ` #n` duplicate suffix on a subject. */
+const SUBJECT_SUFFIX_ROOM = 16;
+
+/** Cap on a generated state key, inside the store's own 100-char limit. */
+const MAX_SLUG_LENGTH = 90;
+
+/**
+ * The lifecycle each heading tier maps to. `active` and `general` are
+ * durable knowledge with no special lifecycle, so both land as facts.
+ */
+const KIND_BY_TIER: Readonly<Record<Tier, MemoryKind>> = {
+  directive: "directive",
+  people: "relationship",
+  active: "fact",
+  general: "fact",
+  status: "state",
+  historical: "episode",
+};
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+/** What one import run did, per row. */
+export type ImportCounts = {
+  inserted: number;
+  superseded: number;
+  skipped: number;
+};
+
+/** One row an import wants to land — the unit of the idempotency check. */
+type Claim = {
+  kind: MemoryKind;
+  subject: string;
+  key?: string;
+  text: string;
+  source: MemorySource;
+};
+
+const emptyCounts = (): ImportCounts => ({
+  inserted: 0,
+  superseded: 0,
+  skipped: 0,
+});
+
+function addCounts(into: ImportCounts, from: ImportCounts): ImportCounts {
+  return {
+    inserted: into.inserted + from.inserted,
+    superseded: into.superseded + from.superseded,
+    skipped: into.skipped + from.skipped,
+  };
+}
+
+// ── Text shaping ────────────────────────────────────────────────────────────
+
+/**
+ * A state key from a family key: `Inbox / CI Watch` → `inbox.ci-watch`.
+ * The slash is the family separator the operator already writes, so it
+ * becomes the key's dot; everything else collapses to a dash.
+ */
+function slugKey(family: string): string {
+  const slug = family
+    .toLowerCase()
+    .replace(/\s*\/\s*/g, ".")
+    .replace(/\s+/g, "-")
+    .replace(/[^a-z0-9_.-]/g, "")
+    .replace(/-{2,}/g, "-")
+    .replace(/\.{2,}/g, ".")
+    .replace(/^[-._]+|[-._]+$/g, "");
+  return slug.slice(0, MAX_SLUG_LENGTH) || "section";
+}
+
+/**
+ * Demote `## ` headings inside a claim's body to `### `.
+ *
+ * A daily note is a whole document and routinely carries its own `## `
+ * headings. Stored verbatim, the render would emit them at the top level
+ * and the next import would read one row back as several sections — and
+ * an embedded `## Active Investigations` would collide with the real
+ * section of that name and supersede it. A claim's body therefore never
+ * holds a top-level heading. Running this twice changes nothing, which is
+ * what keeps the round trip a fixed point.
+ */
+function demoteHeadings(body: string): string {
+  return body.replace(/^##(?=\s)/gm, "###");
+}
+
+/** Blank-line-separated paragraphs, trimmed and normalized. */
+function paragraphsOf(text: string): string[] {
+  return demoteHeadings(text)
+    .split(/\n\s*\n/)
+    .map((para) => para.trim())
+    .filter(Boolean);
+}
+
+/** A single paragraph bigger than the cap — the one place text is lost. */
+function hardCut(para: string): string {
+  return `${para.slice(0, MAX_TEXT_LENGTH - 1).trimEnd()}…`;
+}
+
+/**
+ * Pack paragraphs into chunks of at most `MAX_TEXT_LENGTH`. Splitting on
+ * paragraph boundaries keeps every chunk readable on its own — a claim
+ * cut mid-sentence is worse than no claim. Rejoining the chunks with a
+ * blank line reproduces the input exactly, which is what makes
+ * import → render → import a fixed point.
+ */
+function chunkParagraphs(paras: readonly string[]): string[] {
+  const chunks: string[] = [];
+  let current = "";
+  for (const para of paras) {
+    const piece = para.length > MAX_TEXT_LENGTH ? hardCut(para) : para;
+    const joined = current ? `${current}\n\n${piece}` : piece;
+    if (joined.length <= MAX_TEXT_LENGTH) {
+      current = joined;
+      continue;
+    }
+    if (current) chunks.push(current);
+    current = piece;
+  }
+  if (current) chunks.push(current);
+  return chunks;
+}
+
+// ── Claims ──────────────────────────────────────────────────────────────────
+
+/** Clamp a subject, leaving room for the suffixes appended below. */
+function clampSubject(subject: string): string {
+  return subject.slice(0, MAX_SUBJECT_LENGTH - SUBJECT_SUFFIX_ROOM).trim();
+}
+
+/**
+ * Two sections can share a heading. Without a disambiguator the second
+ * would supersede the first on every run and vice versa, so occurrence
+ * `n > 1` gets a ` #n` marker — deliberately not a parenthetical, which
+ * `familyKey` would strip straight back off on the next import.
+ */
+function disambiguate(
+  seen: Map<string, number>,
+  kind: MemoryKind,
+  subject: string,
+): string {
+  const slot = `${kind}|${subject}`;
+  const occurrence = (seen.get(slot) ?? 0) + 1;
+  seen.set(slot, occurrence);
+  return occurrence === 1 ? subject : `${subject} #${occurrence}`;
+}
+
+/** Spread one section's text across as many rows as the cap needs. */
+function claimsFrom(
+  base: { kind: MemoryKind; subject: string; key?: string },
+  paras: readonly string[],
+  source: MemorySource,
+): Claim[] {
+  const chunks = chunkParagraphs(paras);
+  const many = chunks.length > 1;
+  return chunks.map((text, index) => {
+    const nth = index + 1;
+    const subject = many
+      ? `${base.subject} (${nth}/${chunks.length})`
+      : base.subject;
+    const key =
+      base.key !== undefined && many && nth > 1
+        ? `${base.key}.${nth}`
+        : base.key;
+    return {
+      kind: base.kind,
+      subject,
+      text,
+      source,
+      ...(key !== undefined ? { key } : {}),
+    };
+  });
+}
+
+/**
+ * One `## ` section → its rows. A `state` section's subject is its key:
+ * the render emits keyed rows under their key, so anchoring both on the
+ * slug is what keeps import → render → import a no-op.
+ */
+function sectionClaims(
+  section: Section,
+  source: MemorySource,
+  seen: Map<string, number>,
+): Claim[] {
+  const dated = DATE_HEADING.test(section.title);
+  const tier = TIER_ORDER[classify(section.title)] ?? "general";
+  const kind = dated ? "episode" : KIND_BY_TIER[tier];
+  const family = familyKey(section.title) || section.title;
+  // People sections are per-person and a dated section is one day, so the
+  // heading itself is the subject; everything else is keyed on the family
+  // so a family's snapshots line up on one row.
+  const named = dated || tier === "people" ? section.title : family;
+  const key = kind === "state" ? slugKey(family) : undefined;
+  const subject = disambiguate(
+    seen,
+    kind,
+    clampSubject(key ?? named) || "untitled",
+  );
+  const newline = section.body.indexOf("\n");
+  const paras = paragraphsOf(
+    newline === -1 ? "" : section.body.slice(newline + 1),
+  );
+  if (paras.length === 0) return [];
+  const anchor = key === undefined ? { kind, subject } : { kind, subject, key };
+  return claimsFrom(anchor, paras, source);
+}
+
+// ── Landing ─────────────────────────────────────────────────────────────────
+
+/** The live row this claim owns, if it has landed before. */
+function liveImportRow(claim: Claim): MemoryRow | undefined {
+  return listMemories({
+    kind: claim.kind,
+    subject: claim.subject,
+    limit: LOOKUP_LIMIT,
+  }).find((row) => row.source.actor === IMPORT_ACTOR && row.key === claim.key);
+}
+
+/**
+ * Land one claim: skip an unchanged row, supersede a changed one, insert
+ * a new one. Keyed state goes in through `replaceStateKey`, so the newest
+ * snapshot of a family is the single live row for its key.
+ */
+function landClaim(claim: Claim): keyof ImportCounts {
+  const hash = memoryContentHash(
+    claim.kind,
+    claim.subject,
+    claim.key,
+    claim.text,
+  );
+  const existing = liveImportRow(claim);
+  if (existing?.contentHash === hash) return "skipped";
+  if (existing) {
+    supersedeMemory(existing.id, claim.text, RE_IMPORT_REASON);
+    return "superseded";
+  }
+  if (claim.kind === "state" && claim.key !== undefined) {
+    replaceStateKey(claim.key, claim.text, claim.source, {
+      subject: claim.subject,
+      trust: "operator",
+    });
+  } else {
+    assertMemory({
+      kind: claim.kind,
+      subject: claim.subject,
+      text: claim.text,
+      source: claim.source,
+      trust: "operator",
+    });
+  }
+  return "inserted";
+}
+
+function landAll(claims: Iterable<Claim>): ImportCounts {
+  const counts = emptyCounts();
+  for (const claim of claims) counts[landClaim(claim)] += 1;
+  return counts;
+}
+
+// ── Public API ──────────────────────────────────────────────────────────────
+
+/**
+ * Import `memory.md`: one row per `## ` section, families collapsed
+ * first so only a status family's newest snapshot becomes its live state
+ * row. A missing file is not an error — a fresh install has none.
+ */
+export function importMemoryFile(path: string = files.memory): ImportCounts {
+  if (!existsSync(path)) return emptyCounts();
+  const content = readFileSync(path, "utf8");
+  const source: MemorySource = { actor: IMPORT_ACTOR, chat: basename(path) };
+  const { kept } = collapseFamilies(parseSections(content).sections);
+  const seen = new Map<string, number>();
+  return landAll(
+    kept.flatMap((section) => sectionClaims(section, source, seen)),
+  );
+}
+
+/**
+ * Import the daily notes: one `episode` per `YYYY-MM-DD.md`, subject the
+ * date. `diary-*.md` is the soul's first-person writing — a reflection,
+ * never a fact source (plan §3.1) — so the filename pattern excludes it.
+ */
+export function importDailyNotes(dir: string = dirs.dailyMemory): ImportCounts {
+  if (!existsSync(dir)) return emptyCounts();
+  let counts = emptyCounts();
+  for (const name of readdirSync(dir).sort()) {
+    const date = DAILY_NOTE.exec(name)?.[1];
+    if (!date) continue;
+    const paras = paragraphsOf(readFileSync(join(dir, name), "utf8"));
+    if (paras.length === 0) continue;
+    const source: MemorySource = { actor: IMPORT_ACTOR, chat: name };
+    const claims = claimsFrom(
+      { kind: "episode", subject: date },
+      paras,
+      source,
+    );
+    counts = addCounts(counts, landAll(claims));
+  }
+  return counts;
+}
+
+/** Both sources, one set of counts — what `talon memory import` runs. */
+export function importAll(): ImportCounts {
+  return addCounts(importMemoryFile(), importDailyNotes());
+}

package/src/core/memory/render.ts

@@ -0,0 +1,243 @@
+/**
+ * Typed memory rows → a `memory.md`-shaped document.
+ *
+ * The inverse of import.ts, and the second half of "markdown becomes a
+ * view" (plan §3.1): the store holds the claims, the file is a rendered
+ * projection of them — still human-readable, still `Read`-able, still
+ * editable, but no longer authoritative.
+ *
+ * Two properties matter more than prettiness:
+ *
+ *   - **Deterministic.** The same rows render to the same bytes, every
+ *     time. A file that churns on every render is a file nobody can diff
+ *     and a prompt prefix that never caches.
+ *   - **A fixed point with import.** Chunked rows are regrouped under
+ *     their base subject and keyed state renders under its key, so
+ *     importing this document back produces exactly the rows it came
+ *     from and reports zero inserts. That round trip is what lets a
+ *     hand-edit be folded back instead of duplicated.
+ *
+ * Truncation is selection, not slicing: sections are dropped whole, from
+ * the bottom of the ranking, and the tail is named so the reader knows
+ * to ask the store for the rest.
+ */
+
+import { copyFileSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import writeFileAtomic from "write-file-atomic";
+
+import {
+  listMemories,
+  type MemoryKind,
+  type MemoryRow,
+} from "../../storage/memory.js";
+import { dirs, files } from "../../util/paths.js";
+import { toYMD } from "../../util/time.js";
+import {
+  MEMORY_INJECT_MAX_CHARS,
+  headingTitle,
+} from "../prompt/memory-view.js";
+
+// ── Tunables ────────────────────────────────────────────────────────────────
+
+/**
+ * Kind order in the document, most durable first. `reflection` is absent
+ * on purpose: the diary is the persona layer and never a fact source, so
+ * it is not part of the memory projection (plan §3.1).
+ */
+const KIND_ORDER: readonly MemoryKind[] = [
+  "directive",
+  "relationship",
+  "fact",
+  "state",
+  "episode",
+];
+
+/** How many rows per kind the default render pulls from the store. */
+const KIND_ROW_LIMIT = 500;
+
+/** The preamble. Import ignores everything above the first `## `. */
+const HEADER =
+  "# Memory\n\n" +
+  "_Rendered from the typed memory store. Edit freely — " +
+  "`talon memory import` folds changes back._\n";
+
+/** The ` (n/N)` marker import appends when a section outgrew the text cap. */
+const CHUNK_SUFFIX = /^(.*) \((\d+)\/(\d+)\)$/;
+
+// ── Ordering ────────────────────────────────────────────────────────────────
+
+/** One row placed under its section heading. */
+type Piece = { label: string; index: number; row: MemoryRow };
+
+/**
+ * A keyed row renders under its key, which is the more specific label and
+ * the one import reads back. Chunk `n > 1` carries a `.n` key suffix so
+ * each chunk owns a distinct key; the heading drops it again.
+ */
+function stateLabel(row: MemoryRow, index: number): string {
+  const key = row.key ?? row.subject;
+  const suffix = `.${index}`;
+  return index > 1 && key.endsWith(suffix) ? key.slice(0, -suffix.length) : key;
+}
+
+function pieceOf(row: MemoryRow): Piece {
+  const chunk = CHUNK_SUFFIX.exec(row.subject);
+  const index = chunk ? Number(chunk[2]) : 1;
+  const base = chunk?.[1] ?? row.subject;
+  return {
+    label: row.kind === "state" ? stateLabel(row, index) : base,
+    index,
+    row,
+  };
+}
+
+/** Pinned first, then kind, then salience, then recency; id breaks ties. */
+function compareRows(a: MemoryRow, b: MemoryRow): number {
+  return (
+    Number(b.pinned) - Number(a.pinned) ||
+    KIND_ORDER.indexOf(a.kind) - KIND_ORDER.indexOf(b.kind) ||
+    b.salience - a.salience ||
+    b.lastSeenAt - a.lastSeenAt ||
+    a.id - b.id
+  );
+}
+
+// ── Rendering ───────────────────────────────────────────────────────────────
+
+/** One heading plus its rows, blank-line separated, chunks back in order. */
+function renderSection(label: string, pieces: readonly Piece[]): string {
+  const body = [...pieces]
+    .sort((a, b) => a.index - b.index)
+    .map((piece) => piece.row.text)
+    .join("\n\n");
+  return `## ${headingTitle(label)}\n\n${body}\n`;
+}
+
+/**
+ * Group ranked rows into sections. A section takes the position of its
+ * best-ranked row, so the ranking survives the grouping.
+ */
+function sectionsFrom(rows: readonly MemoryRow[]): string[] {
+  const groups = new Map<string, Piece[]>();
+  const ranked = rows
+    .filter((row) => KIND_ORDER.includes(row.kind))
+    .sort(compareRows);
+  for (const row of ranked) {
+    const piece = pieceOf(row);
+    const held = groups.get(piece.label);
+    if (held) held.push(piece);
+    else groups.set(piece.label, [piece]);
+  }
+  return [...groups].map(([label, pieces]) => renderSection(label, pieces));
+}
+
+/** The pointer that replaces whatever the budget could not carry. */
+function moreLine(omitted: number): string {
+  const plural = omitted === 1 ? "section" : "sections";
+  return `_… ${omitted} more ${plural} in the store — \`talon memory list\`_\n`;
+}
+
+/**
+ * Take whole sections until the budget is spent, then give back sections
+ * until the "N more" pointer fits too — the document as a whole stays
+ * inside the budget, and every cut lands on a section boundary. The
+ * header plus that pointer is the floor: a budget smaller than those two
+ * still gets them, because a document that cannot say what it dropped is
+ * worse than one slightly over budget.
+ */
+function selectSections(
+  sections: readonly string[],
+  budget: number,
+): { taken: readonly string[]; omitted: number } {
+  let spent = HEADER.length;
+  let take = 0;
+  for (const section of sections) {
+    if (spent + section.length + 1 > budget) break;
+    spent += section.length + 1;
+    take += 1;
+  }
+  let omitted = sections.length - take;
+  while (
+    take > 0 &&
+    omitted > 0 &&
+    spent + moreLine(omitted).length + 1 > budget
+  ) {
+    take -= 1;
+    spent -= sections[take]!.length + 1;
+    omitted += 1;
+  }
+  return { taken: sections.slice(0, take), omitted };
+}
+
+/** Options shared by the render and the file write. */
+export type RenderOptions = {
+  /** Char budget for the whole document. Defaults to the inject cap. */
+  budget?: number;
+  /**
+   * Rows to render instead of the live store — the seam the tests and
+   * (from PR 9) the pre-ranked core view use.
+   */
+  rows?: readonly MemoryRow[];
+};
+
+function liveRows(): MemoryRow[] {
+  return KIND_ORDER.flatMap((kind) =>
+    listMemories({ kind, limit: KIND_ROW_LIMIT }),
+  );
+}
+
+/**
+ * Render the live store as a `memory.md` document. Same rows in, same
+ * bytes out.
+ */
+export function renderMemoryMarkdown(opts: RenderOptions = {}): string {
+  const sections = sectionsFrom(opts.rows ?? liveRows());
+  const { taken, omitted } = selectSections(
+    sections,
+    opts.budget ?? MEMORY_INJECT_MAX_CHARS,
+  );
+  const parts = [HEADER, ...taken];
+  if (omitted > 0) parts.push(moreLine(omitted));
+  return parts.join("\n");
+}
+
+// ── Writing ─────────────────────────────────────────────────────────────────
+
+/** Where a render went, and what it displaced. */
+export type RenderWrite = {
+  path: string;
+  bytes: number;
+  /** The archived copy of the previous file, when one was taken. */
+  backup?: string;
+};
+
+/**
+ * Copy the file aside before it is replaced — once per day, so a render
+ * loop can't bury the archive, and never overwriting an existing backup,
+ * so the first render of the day keeps the hand-written original.
+ */
+function backUp(path: string, archiveDir: string): string | undefined {
+  if (!existsSync(path)) return undefined;
+  const dest = join(archiveDir, `memory-before-render-${toYMD(new Date())}.md`);
+  if (existsSync(dest)) return undefined;
+  mkdirSync(archiveDir, { recursive: true });
+  copyFileSync(path, dest);
+  return dest;
+}
+
+/** Render the store over `memory.md`, keeping a dated copy of what was there. */
+export function writeRenderedMemory(
+  path: string = files.memory,
+  opts: RenderOptions & { archiveDir?: string } = {},
+): RenderWrite {
+  const markdown = renderMemoryMarkdown(opts);
+  const backup = backUp(path, opts.archiveDir ?? dirs.memoryArchive);
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileAtomic.sync(path, markdown);
+  return {
+    path,
+    bytes: markdown.length,
+    ...(backup !== undefined ? { backup } : {}),
+  };
+}

package/src/core/models/active-model.ts

@@ -73,7 +73,7 @@ import {
   type ModelSource,
 } from "../agent-runtime/model-ref.js";
 import type { UnifiedModelInfo } from "../types.js";
-import type { TalonConfig } from "../../util/config.js";
+import type { TalonConfig } from "../config/index.js";
 import { logWarn } from "../../util/log.js";
 
 /**

package/src/core/plugin/builtins.ts

@@ -4,7 +4,7 @@
  */
 
 import { log, logError, logWarn } from "../../util/log.js";
-import type { TalonConfig } from "../../util/config.js";
+import type { TalonConfig } from "../config/index.js";
 import { registry, reloadState } from "./registry.js";
 import type { ProvisionOutcome } from "./provision.js";
 import { NATIVE_RUNTIMES, type NativePluginId } from "./native-runtimes.js";
@@ -226,7 +226,7 @@ export async function reloadPlugins(
 ): Promise<{ names: string[]; config: TalonConfig }> {
   // Validate config BEFORE tearing down existing plugins. If the config is
   // malformed the error propagates and current plugins stay intact.
-  const { loadConfig, getFrontends } = await import("../../util/config.js");
+  const { loadConfig, getFrontends } = await import("../config/index.js");
   const config = loadConfig();
 
   // Derive frontends from config if not explicitly provided

package/src/core/plugin/manage.ts

@@ -8,7 +8,7 @@
  * talon.json so the surface owns its own write path.
  */
 
-import type { TalonConfig } from "../../util/config.js";
+import type { TalonConfig } from "../config/index.js";
 import { isPathPlugin, type PluginEntry } from "./types.js";
 import {
   BUILTIN_PLUGINS,