talon-agent 4.6.1 → 5.0.1

package/src/core/soul/consolidate.ts

@@ -1,179 +0,0 @@
-/**
- * Soul Kernel — organic consolidation ("the dream").
- *
- * Crystallization clusters loose evidence once. Consolidation is what makes the
- * value graph *alive*: it periodically re-examines the whole lattice and lets it
- * reorganize as experience accumulates — values that have drifted together merge
- * into one, carrying their learned salience and evidence forward.
- *
- * Two principles keep it organic rather than over-programmed:
- *
- *   1. The merge threshold is ADAPTIVE — derived from the current distribution of
- *      nearest-neighbor distances among values, not a magic constant. The lattice
- *      decides its own granularity from its own shape, and that shape changes as
- *      it grows.
- *
- *   2. Learned state is MIGRATED, never lost. A merge supersedes the old values
- *      with a new content-addressed node, sums their salience/evidence into it,
- *      and records `supersedes` edges + a Spine event. Nothing is deleted — the
- *      old nodes simply fall out of projection (filtered as superseded) and decay.
- *
- * No model decides what merges. Geometry and accumulated weight do.
- */
-
-import type { SoulDag } from "./dag.js";
-import { cosineDistance, medoidIndex, type Embedder } from "./embedder.js";
-import { effectiveSalience } from "./salience.js";
-import type { Hash, SoulConfig, ValuePayload } from "./types.js";
-import { halfLifeForKind } from "./types.js";
-
-/** A value is dead once something supersedes it. */
-export function isSuperseded(dag: SoulDag, value: Hash): boolean {
-  return dag.allEdges("supersedes").some((e) => e.to === value);
-}
-
-/** Live (non-superseded) value nodes. */
-export function liveValues(dag: SoulDag): Hash[] {
-  return dag
-    .nodesOfKind("value")
-    .map((n) => n.hash)
-    .filter((h) => !isSuperseded(dag, h));
-}
-
-function evidenceText(dag: SoulDag, hash: Hash): string {
-  const node = dag.getNode(hash);
-  return node?.payload.kind === "evidence" ? node.payload.text : "";
-}
-
-function valueOf(dag: SoulDag, hash: Hash): ValuePayload {
-  return dag.getNode(hash)!.payload as ValuePayload;
-}
-
-/** Median of a numeric array (0 for empty). */
-function median(xs: number[]): number {
-  if (xs.length === 0) return 0;
-  const s = [...xs].sort((a, b) => a - b);
-  const mid = Math.floor(s.length / 2);
-  return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2;
-}
-
-export interface ConsolidateOptions {
-  readonly now: number;
-  /** Override the adaptive threshold (mainly for tests). */
-  readonly mergeThreshold?: number;
-  /** Fraction of the median NN distance used as the adaptive threshold. */
-  readonly adaptiveFactor?: number;
-}
-
-export interface ConsolidateResult {
-  readonly created: Hash[];
-  readonly superseded: Hash[];
-  readonly groups: number;
-  readonly threshold: number;
-}
-
-/**
- * Run one consolidation pass. Returns what merged. Caller commits.
- */
-export async function consolidate(
-  dag: SoulDag,
-  embedder: Embedder,
-  cfg: SoulConfig,
-  opts: ConsolidateOptions,
-): Promise<ConsolidateResult> {
-  const values = liveValues(dag);
-  if (values.length < 2) {
-    return { created: [], superseded: [], groups: 0, threshold: 0 };
-  }
-
-  // Represent each value by its medoid embedding.
-  const repTexts = values.map((h) => evidenceText(dag, valueOf(dag, h).medoid));
-  const reps = await embedder.embed(repTexts);
-
-  // Adaptive threshold: how close is "unusually close" for THIS lattice right now.
-  const nn: number[] = [];
-  for (let i = 0; i < values.length; i++) {
-    let best = Infinity;
-    for (let j = 0; j < values.length; j++) {
-      if (i === j) continue;
-      best = Math.min(best, cosineDistance(reps[i]!, reps[j]!));
-    }
-    if (Number.isFinite(best)) nn.push(best);
-  }
-  const factor = opts.adaptiveFactor ?? 0.6;
-  const threshold = opts.mergeThreshold ?? median(nn) * factor;
-
-  // Connected components over the "close enough to merge" graph (union-find).
-  const parent = values.map((_, i) => i);
-  const find = (x: number): number => {
-    while (parent[x] !== x) {
-      parent[x] = parent[parent[x]!]!;
-      x = parent[x]!;
-    }
-    return x;
-  };
-  const union = (a: number, b: number): void => {
-    parent[find(a)] = find(b);
-  };
-  for (let i = 0; i < values.length; i++) {
-    for (let j = i + 1; j < values.length; j++) {
-      if (cosineDistance(reps[i]!, reps[j]!) <= threshold) union(i, j);
-    }
-  }
-
-  // Gather groups of size > 1.
-  const groups = new Map<number, number[]>();
-  for (let i = 0; i < values.length; i++) {
-    const root = find(i);
-    (groups.get(root) ?? groups.set(root, []).get(root)!).push(i);
-  }
-
-  const created: Hash[] = [];
-  const superseded: Hash[] = [];
-
-  for (const members of groups.values()) {
-    if (members.length < 2) continue;
-
-    const oldValues = members.map((i) => values[i]!);
-    // Union of all member evidence across the merging values.
-    const evidenceSet = new Set<Hash>();
-    for (const v of oldValues) {
-      const p = valueOf(dag, v);
-      evidenceSet.add(p.medoid);
-      for (const m of p.members) evidenceSet.add(m);
-    }
-    const evidence = [...evidenceSet].sort();
-
-    // Recompute the medoid over the merged evidence — its new label emerges.
-    const evTexts = evidence.map((h) => evidenceText(dag, h));
-    const evVecs = await embedder.embed(evTexts);
-    const medoid = evidence[medoidIndex(evVecs)]!;
-
-    const merged = dag.addNode(
-      { kind: "value", members: evidence, medoid },
-      opts.now,
-    );
-    created.push(merged);
-
-    // Migrate learned state forward: sum decayed salience + evidence + activations.
-    const target = dag.stateOf(merged);
-    for (const v of oldValues) {
-      const s = dag.stateOf(v);
-      target.salience += effectiveSalience(
-        s,
-        opts.now,
-        halfLifeForKind(cfg, "value"),
-      );
-      target.evidence += s.evidence;
-      target.activations += s.activations;
-      const e = dag.edge(merged, "supersedes", v, opts.now);
-      e.weight = 1;
-      e.updatedAt = opts.now;
-      if (v !== merged) superseded.push(v);
-    }
-    target.lastActivatedAt = opts.now;
-    dag.touch(merged);
-  }
-
-  return { created, superseded, groups: created.length, threshold };
-}

package/src/core/soul/critic.ts

@@ -1,162 +0,0 @@
-/**
- * Soul Kernel — the Critic. A self-model of failure modes, made mechanical.
- *
- * In v3 the Critic is not a reasoning model second-guessing the first; it is a
- * set of frozen, feature-based classifiers. Each scores a candidate text (a
- * drafted reply, or a projection surface) against one of Talon's documented
- * failure modes — wall-of-text, sycophancy, emoji-overload — using a linear
- * combination of cheap lexical/structural features and a fixed threshold. A
- * classifier is a fixed function, not an agent, so this stays strictly within
- * the model-free principle.
- *
- * Embedding-based gates (contradiction against the KG, drift from the core
- * centroid) are a later phase; they too are distance comparisons, not model
- * calls. The lexical classifiers here are the immediately-useful, fully
- * dependency-free core.
- */
-
-import { estimateTokens } from "./projector.js";
-
-type FailureMode = "wall-of-text" | "sycophancy" | "emoji-overload";
-
-export interface CritiqueFeatures {
-  readonly tokens: number;
-  readonly sentences: number;
-  /** Hedging phrases per 100 tokens. */
-  readonly hedgeRate: number;
-  /** Emoji characters per 100 tokens. */
-  readonly emojiDensity: number;
-  /** Count of sycophancy-lexicon hits. */
-  readonly sycophancyHits: number;
-}
-
-export interface Critique {
-  readonly mode: FailureMode;
-  /** Continuous severity; >= 1 means the fixed threshold was crossed. */
-  readonly score: number;
-  readonly flagged: boolean;
-  readonly reason: string;
-}
-
-const HEDGES = [
-  "i think",
-  "i guess",
-  "maybe",
-  "perhaps",
-  "sort of",
-  "kind of",
-  "i suppose",
-  "probably",
-  "it seems",
-];
-
-const SYCOPHANCY = [
-  "great question",
-  "excellent question",
-  "you're absolutely right",
-  "you are absolutely right",
-  "i'd be happy to",
-  "i would be happy to",
-  "absolutely!",
-  "certainly!",
-  "of course!",
-  "happy to help",
-  "great point",
-];
-
-// Emoji detection without a heavy dependency: pictographic + symbol ranges.
-const EMOJI_RE =
-  // Variation selectors are combining marks — alternated separately so the
-  // character class holds only standalone pictographic ranges
-  // (no-misleading-character-class).
-  /[\u{1F300}-\u{1FAFF}\u{2600}-\u{27BF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}\u{1F000}-\u{1F0FF}]|[\u{FE00}-\u{FE0F}]/gu;
-
-function countOccurrences(
-  haystack: string,
-  needles: readonly string[],
-): number {
-  let n = 0;
-  for (const needle of needles) {
-    let from = 0;
-    for (;;) {
-      const i = haystack.indexOf(needle, from);
-      if (i === -1) break;
-      n++;
-      from = i + needle.length;
-    }
-  }
-  return n;
-}
-
-/** Extract the cheap features the classifiers run over. Pure + deterministic. */
-export function extractFeatures(text: string): CritiqueFeatures {
-  const lower = text.toLowerCase();
-  const tokens = Math.max(1, estimateTokens(text));
-  const sentences = Math.max(1, (text.match(/[.!?](\s|$)/g) ?? []).length);
-  const emoji = (text.match(EMOJI_RE) ?? []).length;
-  const hedges = countOccurrences(lower, HEDGES);
-  return {
-    tokens,
-    sentences,
-    hedgeRate: (hedges / tokens) * 100,
-    emojiDensity: (emoji / tokens) * 100,
-    sycophancyHits: countOccurrences(lower, SYCOPHANCY),
-  };
-}
-
-export interface CritiqueThresholds {
-  /** Token count above which prose is "wall-of-text". */
-  readonly maxTokens: number;
-  /** Sycophancy score (hits + hedge contribution) threshold. */
-  readonly maxSycophancy: number;
-  /** Emoji-per-100-tokens threshold. */
-  readonly maxEmojiDensity: number;
-}
-
-export const DEFAULT_THRESHOLDS: CritiqueThresholds = {
-  maxTokens: 320,
-  maxSycophancy: 1,
-  maxEmojiDensity: 6,
-};
-
-/**
- * Run the frozen classifiers over a candidate text. Returns one critique per
- * failure mode; `flagged` is true when its normalized score reaches 1.
- */
-export function critique(
-  text: string,
-  thresholds: CritiqueThresholds = DEFAULT_THRESHOLDS,
-): Critique[] {
-  const f = extractFeatures(text);
-
-  const sycophancyScore = f.sycophancyHits + f.hedgeRate / 5;
-  const wall = f.tokens / thresholds.maxTokens;
-  const syco = sycophancyScore / Math.max(0.0001, thresholds.maxSycophancy);
-  const emoji = f.emojiDensity / Math.max(0.0001, thresholds.maxEmojiDensity);
-
-  return [
-    {
-      mode: "wall-of-text",
-      score: wall,
-      flagged: wall >= 1,
-      reason: `${f.tokens} tokens vs budget ${thresholds.maxTokens}`,
-    },
-    {
-      mode: "sycophancy",
-      score: syco,
-      flagged: syco >= 1,
-      reason: `${f.sycophancyHits} sycophancy hits, hedge ${f.hedgeRate.toFixed(1)}/100tok`,
-    },
-    {
-      mode: "emoji-overload",
-      score: emoji,
-      flagged: emoji >= 1,
-      reason: `${f.emojiDensity.toFixed(1)} emoji/100tok vs ${thresholds.maxEmojiDensity}`,
-    },
-  ];
-}
-
-/** True if any failure mode tripped its threshold. */
-export function isFlagged(critiques: readonly Critique[]): boolean {
-  return critiques.some((c) => c.flagged);
-}

package/src/core/soul/dag.ts

@@ -1,265 +0,0 @@
-/**
- * Soul Kernel — the content-addressed DAG store.
- *
- * Holds three things, kept rigorously separate:
- *
- *   1. Nodes   — immutable, content-addressed identity content (the Merkle DAG).
- *   2. State   — mutable per-node "weather" (salience, evidence weight).
- *   3. Edges   — mutable associative links (Hebbian co-activation, tension).
- *
- * Referential integrity is enforced on insert: a node may only reference child
- * hashes that already exist, so the DAG can never contain a dangling pointer and
- * the Merkle root is always sound.
- *
- * Dirty-tracking is structural and underpins "partial recompilation": only the
- * nodes added or whose state mutated since the last commit are reported dirty, so
- * downstream consumers (projector, critic) reprocess a handful of nodes, not the
- * whole self.
- */
-
-import { hashPayload, merkleRoot } from "./hash.js";
-import type {
-  ActivationState,
-  AssocEdge,
-  EdgeKind,
-  Hash,
-  NodePayload,
-  SoulNode,
-} from "./types.js";
-
-/** The structural (hashed) children a payload references, by kind. */
-export function structuralChildren(payload: NodePayload): Hash[] {
-  switch (payload.kind) {
-    case "evidence":
-    case "reflex":
-      return [];
-    case "value":
-      return dedupe([payload.medoid, ...payload.members]);
-    case "theme":
-      return dedupe([payload.medoid, ...payload.values]);
-    case "spine":
-      return dedupe([
-        ...payload.affects,
-        ...(payload.prev ? [payload.prev] : []),
-      ]);
-    case "lens":
-      return dedupe([
-        ...payload.amplify.map((a) => a.node),
-        ...payload.evidence,
-      ]);
-  }
-}
-
-function dedupe(hashes: Hash[]): Hash[] {
-  return [...new Set(hashes)];
-}
-
-function freshState(now: number): ActivationState {
-  return { salience: 0, evidence: 0, activations: 0, lastActivatedAt: now };
-}
-
-/** Edge map key — one edge per (from, kind, to) triple. */
-function edgeKey(from: Hash, kind: EdgeKind, to: Hash): string {
-  return `${from}|${kind}|${to}`;
-}
-
-export interface DagSnapshot {
-  readonly nodes: readonly SoulNode[];
-  readonly state: readonly (readonly [Hash, ActivationState])[];
-  readonly edges: readonly AssocEdge[];
-}
-
-export class SoulDag {
-  private readonly nodes = new Map<Hash, SoulNode>();
-  private readonly state = new Map<Hash, ActivationState>();
-  private readonly edges = new Map<string, AssocEdge>();
-  /** Reverse adjacency for structural children → parents, for dirty propagation. */
-  private readonly parents = new Map<Hash, Set<Hash>>();
-  private readonly dirty = new Set<Hash>();
-
-  // ── Nodes ──────────────────────────────────────────────────────────────────
-
-  /**
-   * Insert a node by content. Idempotent: identical content returns the same
-   * hash and does not re-dirty. Throws if any structural child is absent.
-   */
-  addNode(payload: NodePayload, now = Date.now()): Hash {
-    const hash = hashPayload(payload);
-    if (this.nodes.has(hash)) return hash;
-
-    const children = structuralChildren(payload);
-    for (const child of children) {
-      if (!this.nodes.has(child)) {
-        throw new Error(
-          `SoulDag.addNode: dangling reference ${child} from ${payload.kind} node`,
-        );
-      }
-    }
-
-    this.nodes.set(hash, { hash, payload });
-    this.state.set(hash, freshState(now));
-    for (const child of children) {
-      let set = this.parents.get(child);
-      if (!set) {
-        set = new Set();
-        this.parents.set(child, set);
-      }
-      set.add(hash);
-    }
-    this.dirty.add(hash);
-    return hash;
-  }
-
-  getNode(hash: Hash): SoulNode | undefined {
-    return this.nodes.get(hash);
-  }
-
-  hasNode(hash: Hash): boolean {
-    return this.nodes.has(hash);
-  }
-
-  /** All node hashes of a given kind. */
-  nodesOfKind(kind: NodePayload["kind"]): SoulNode[] {
-    const out: SoulNode[] = [];
-    for (const node of this.nodes.values()) {
-      if (node.payload.kind === kind) out.push(node);
-    }
-    return out;
-  }
-
-  get size(): number {
-    return this.nodes.size;
-  }
-
-  // ── State ──────────────────────────────────────────────────────────────────
-
-  /** Mutable activation state for a node (lazily created). */
-  stateOf(hash: Hash): ActivationState {
-    let s = this.state.get(hash);
-    if (!s) {
-      s = freshState(Date.now());
-      this.state.set(hash, s);
-    }
-    return s;
-  }
-
-  /** Mark a node (and its structural ancestors) dirty after a state change. */
-  touch(hash: Hash): void {
-    if (!this.nodes.has(hash)) return;
-    this.dirty.add(hash);
-  }
-
-  // ── Edges ──────────────────────────────────────────────────────────────────
-
-  /** Create or fetch the edge for a triple; weight starts at 0. */
-  edge(from: Hash, kind: EdgeKind, to: Hash, now = Date.now()): AssocEdge {
-    const key = edgeKey(from, kind, to);
-    let e = this.edges.get(key);
-    if (!e) {
-      e = { from, to, kind, weight: 0, updatedAt: now };
-      this.edges.set(key, e);
-    }
-    return e;
-  }
-
-  getEdge(from: Hash, kind: EdgeKind, to: Hash): AssocEdge | undefined {
-    return this.edges.get(edgeKey(from, kind, to));
-  }
-
-  /** All edges originating at `from`, optionally filtered by kind. */
-  edgesFrom(from: Hash, kind?: EdgeKind): AssocEdge[] {
-    const out: AssocEdge[] = [];
-    for (const e of this.edges.values()) {
-      if (e.from === from && (!kind || e.kind === kind)) out.push(e);
-    }
-    return out;
-  }
-
-  /** Every edge in the graph, optionally filtered by kind. */
-  allEdges(kind?: EdgeKind): AssocEdge[] {
-    const out: AssocEdge[] = [];
-    for (const e of this.edges.values()) {
-      if (!kind || e.kind === kind) out.push(e);
-    }
-    return out;
-  }
-
-  // ── Structure ────────────────────────────────────────────────────────────────
-
-  childrenOf(hash: Hash): Hash[] {
-    const node = this.nodes.get(hash);
-    return node ? structuralChildren(node.payload) : [];
-  }
-
-  parentsOf(hash: Hash): Hash[] {
-    return [...(this.parents.get(hash) ?? [])];
-  }
-
-  /** Merkle root over the full set of node content present. The version id. */
-  root(): Hash {
-    return merkleRoot(this.nodes.keys());
-  }
-
-  // ── Dirty-tracking ───────────────────────────────────────────────────────────
-
-  /** Nodes added or touched since the last clear, plus their ancestors. */
-  dirtySet(): Set<Hash> {
-    const out = new Set<Hash>();
-    const stack = [...this.dirty];
-    while (stack.length) {
-      const h = stack.pop()!;
-      if (out.has(h)) continue;
-      out.add(h);
-      for (const parent of this.parents.get(h) ?? []) stack.push(parent);
-    }
-    return out;
-  }
-
-  clearDirty(): void {
-    this.dirty.clear();
-  }
-
-  // ── Serialization ────────────────────────────────────────────────────────────
-
-  snapshot(): DagSnapshot {
-    return {
-      nodes: [...this.nodes.values()],
-      state: [...this.state.entries()].map(([h, s]) => [h, { ...s }]),
-      edges: [...this.edges.values()].map((e) => ({ ...e })),
-    };
-  }
-
-  /**
-   * Rebuild a DAG from a snapshot. Nodes are re-inserted in dependency order so
-   * referential integrity is re-verified on load (a corrupt snapshot throws
-   * rather than silently producing a broken graph).
-   */
-  static restore(snap: DagSnapshot): SoulDag {
-    const dag = new SoulDag();
-    const byHash = new Map(snap.nodes.map((n) => [n.hash, n]));
-    const inserted = new Set<Hash>();
-
-    const insert = (node: SoulNode): void => {
-      if (inserted.has(node.hash)) return;
-      for (const child of structuralChildren(node.payload)) {
-        const childNode = byHash.get(child);
-        if (!childNode) {
-          throw new Error(`SoulDag.restore: snapshot missing child ${child}`);
-        }
-        insert(childNode);
-      }
-      dag.addNode(node.payload);
-      inserted.add(node.hash);
-    };
-    for (const node of snap.nodes) insert(node);
-
-    for (const [hash, s] of snap.state) {
-      if (dag.nodes.has(hash)) dag.state.set(hash, { ...s });
-    }
-    for (const e of snap.edges) {
-      dag.edges.set(edgeKey(e.from, e.kind, e.to), { ...e });
-    }
-    dag.clearDirty();
-    return dag;
-  }
-}