recallnest 1.4.1

package/src/capture-heuristic.ts

@@ -0,0 +1,177 @@
+/**
+ * Capture Heuristic — lightweight salience filter + pattern-based signal extraction.
+ *
+ * Borrowed from UltraMemory's auto-capture.ts, adapted for RecallNest:
+ * - Reuses RecallNest's own noise-filter (richer than UltraMemory's)
+ * - Extended Chinese patterns (primary user language)
+ * - Returns items compatible with CaptureMemoryInput for batch persistence
+ *
+ * Zero LLM calls — pure regex/heuristic.
+ * Persistence is the caller's responsibility (via persistMemoryBatch).
+ */
+
+import { isNoise } from "./noise-filter.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface AutoCaptureItem {
+  text: string;
+  category: "profile" | "preferences" | "events" | "cases" | "patterns";
+  importance: number;
+  sourceContext: string;
+  /** Replay priority for consolidation: higher = consolidate first. */
+  replayPriority: number;
+}
+
+export interface AutoCaptureResult {
+  /** True if text was rejected by salience pre-filter (too short / noise / greeting) */
+  skippedSalience: boolean;
+  items: AutoCaptureItem[];
+}
+
+// ---------------------------------------------------------------------------
+// Salience pre-filter
+// ---------------------------------------------------------------------------
+
+/** Max items extracted per turn to avoid flooding the store. */
+const MAX_ITEMS_PER_TURN = 5;
+
+/** Minimum text length worth analyzing (below this is almost always noise). */
+const MIN_TEXT_LENGTH = 20;
+
+/** Greetings / affirmations / single-word responses (EN + ZH). */
+const GREETING_RE =
+  /^(hi|hello|hey|thanks|thank you|ok|yes|no|sure|got it|好的|谢谢|嗯|是的|好吧|行|可以|没问题|收到|明白|了解|知道了)[\s!.！。]*$/i;
+
+/**
+ * Heuristic salience check — should this conversation turn be analyzed
+ * for memory extraction?
+ */
+export function shouldCapture(text: string): boolean {
+  if (!text || text.trim().length < MIN_TEXT_LENGTH) return false;
+  if (isNoise(text)) return false;
+  if (GREETING_RE.test(text.trim().toLowerCase())) return false;
+  return true;
+}
+
+// ---------------------------------------------------------------------------
+// Heuristic extraction patterns
+// ---------------------------------------------------------------------------
+
+interface SignalPattern {
+  re: RegExp;
+  category: AutoCaptureItem["category"];
+  importance: number;
+  sourceContext: string;
+  /** Replay priority: corrections/explicit-memory > decisions > preferences/patterns. */
+  replayPriority: number;
+}
+
+const SIGNAL_PATTERNS: SignalPattern[] = [
+  // --- Preference signals (0.8) ---
+  {
+    re: /(?:\b(?:i prefer|i like|i don't like|i hate|i love|i want|i need|i always use|i never use)\b|我喜欢|我不喜欢|我偏好|我想要|我习惯|我一般用|我从不用|我讨厌|以后都用|以后不要)/i,
+    category: "preferences",
+    importance: 0.8,
+    sourceContext: "preference signal",
+    replayPriority: 0.5,
+  },
+  // --- Identity / profile signals (0.9) ---
+  {
+    re: /(?:\b(?:my name is|i am a|i'm a|i work at|i live in|my role is|i'm responsible for)\b|我叫|我是|我在|我的角色|我负责|我的工作是|我住在)/i,
+    category: "profile",
+    importance: 0.9,
+    sourceContext: "identity signal",
+    replayPriority: 0.6,
+  },
+  // --- Decision signals (0.7) ---
+  {
+    re: /(?:\b(?:i decided|we decided|let's go with|the decision is|we agreed|final call)\b|决定了|我们选择|最终方案|确定用|就这么定了|敲定)/i,
+    category: "events",
+    importance: 0.7,
+    sourceContext: "decision signal",
+    replayPriority: 0.7,
+  },
+  // --- Correction signals (0.85 — high value, user explicitly correcting agent) ---
+  {
+    re: /(?:\b(?:actually|no,? not|that's wrong|correction:|you're wrong|that's incorrect)\b|更正|其实不是|不对|搞错了|纠正一下|你说错了|应该是)/i,
+    category: "cases",
+    importance: 0.85,
+    sourceContext: "correction signal",
+    replayPriority: 0.9,
+  },
+  // --- Explicit memory instruction signals (0.85) ---
+  {
+    re: /(?:\b(?:remember that|don't forget|keep in mind|note that|important:)\b|记住|别忘了|注意|以后记得|帮我记|你要记住)/i,
+    category: "events",
+    importance: 0.85,
+    sourceContext: "explicit memory instruction",
+    replayPriority: 0.9,
+  },
+  // --- Pattern / workflow signals (0.75) ---
+  {
+    re: /(?:\b(?:the pattern is|the workflow is|the process is|step 1|the rule is|always do)\b|流程是|步骤是|规则是|每次都要|固定做法|标准流程)/i,
+    category: "patterns",
+    importance: 0.75,
+    sourceContext: "pattern signal",
+    replayPriority: 0.6,
+  },
+];
+
+/**
+ * Extract memory-worthy items from conversation text using simple heuristics.
+ *
+ * Lightweight alternative to LLM-based extraction.
+ * Returns items ready for persistence via persistMemoryBatch.
+ */
+export function extractHeuristic(text: string): AutoCaptureItem[] {
+  const items: AutoCaptureItem[] = [];
+  const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(text);
+  const minSentenceLen = hasCJK ? 6 : 15;
+
+  const sentences = text
+    .split(/[.!?。！？\n]+/)
+    .map((s) => s.trim())
+    .filter((s) => s.length > minSentenceLen);
+
+  for (let i = 0; i < sentences.length; i++) {
+    const sentence = sentences[i];
+
+    for (const pattern of SIGNAL_PATTERNS) {
+      if (pattern.re.test(sentence)) {
+        // For correction signals, include the next sentence to capture
+        // the actual corrected fact (e.g. "Actually, that's wrong. The limit is 200.")
+        let capturedText = sentence;
+        if (pattern.sourceContext === "correction signal" && i + 1 < sentences.length) {
+          capturedText = `${sentence}. ${sentences[i + 1]}`;
+          i++; // skip next sentence since we consumed it
+        }
+
+        items.push({
+          text: capturedText,
+          category: pattern.category,
+          importance: pattern.importance,
+          sourceContext: pattern.sourceContext,
+          replayPriority: pattern.replayPriority,
+        });
+        break; // first matching pattern wins for this sentence
+      }
+    }
+
+    if (items.length >= MAX_ITEMS_PER_TURN) break;
+  }
+
+  return items.slice(0, MAX_ITEMS_PER_TURN);
+}
+
+/**
+ * Full auto-capture pipeline: salience check → heuristic extraction.
+ */
+export function autoCapture(text: string): AutoCaptureResult {
+  if (!shouldCapture(text)) {
+    return { skippedSalience: true, items: [] };
+  }
+  return { skippedSalience: false, items: extractHeuristic(text) };
+}

package/src/cascade-forget.ts

@@ -0,0 +1,135 @@
+/**
+ * Cascade Forget — Directed forgetting with association propagation.
+ *
+ * Agent-First reasoning: when a user says "forget X", the primary memory
+ * gets deleted/archived. But if related memories (meeting notes, decisions,
+ * preferences mentioning X) keep their original importance, X still
+ * surfaces indirectly through retrieval. Cascade forget finds related
+ * entries and demotes them so X doesn't leak back via associations.
+ *
+ * Brain-science label: "directed forgetting" — suppressing a memory also
+ * weakens retrieval of associated memories. We keep the name for branding;
+ * the real mechanism is vector search + importance reduction.
+ *
+ * Rules:
+ * - Does NOT delete related entries — only demotes importance and tier
+ * - Demotion is proportional to similarity (higher sim = bigger demotion)
+ * - Writes metadata breadcrumb for auditability
+ * - No LLM calls — pure vector search + arithmetic
+ */
+
+import type { MemoryStore, MemoryEntry } from "./store.js";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface CascadeForgetConfig {
+  /** Minimum vector similarity to consider "related" (default: 0.70) */
+  similarityThreshold: number;
+  /** Maximum entries to demote per forget operation (default: 10) */
+  maxDemotePerForget: number;
+  /** Maximum importance reduction (at similarity=1.0) (default: 0.3) */
+  maxDemotion: number;
+  /** Floor: importance never drops below this (default: 0.05) */
+  importanceFloor: number;
+}
+
+export const DEFAULT_CASCADE_FORGET_CONFIG: CascadeForgetConfig = {
+  similarityThreshold: 0.70,
+  maxDemotePerForget: 10,
+  maxDemotion: 0.3,
+  importanceFloor: 0.05,
+};
+
+// ---------------------------------------------------------------------------
+// Core
+// ---------------------------------------------------------------------------
+
+export interface CascadeForgetResult {
+  demotedCount: number;
+  demotedIds: string[];
+}
+
+/**
+ * After deleting/archiving a memory, cascade-demote related entries.
+ *
+ * @param store - Memory store
+ * @param forgottenEntry - The entry that was just deleted/archived (need its vector + scope)
+ * @param config - Cascade config
+ */
+export async function cascadeForget(
+  store: MemoryStore,
+  forgottenEntry: Pick<MemoryEntry, "id" | "vector" | "scope">,
+  config: CascadeForgetConfig = DEFAULT_CASCADE_FORGET_CONFIG,
+): Promise<CascadeForgetResult> {
+  if (!forgottenEntry.vector?.length) {
+    return { demotedCount: 0, demotedIds: [] };
+  }
+
+  // Find related entries in the same scope
+  const candidates = await store.vectorSearch(
+    forgottenEntry.vector,
+    config.maxDemotePerForget * 2, // fetch extra, filter down
+    config.similarityThreshold,
+    [forgottenEntry.scope],
+  );
+
+  const demotedIds: string[] = [];
+
+  for (const candidate of candidates) {
+    if (demotedIds.length >= config.maxDemotePerForget) break;
+
+    const entry = candidate.entry;
+
+    // Skip the forgotten entry itself (may still be in index)
+    if (entry.id === forgottenEntry.id) continue;
+
+    // Skip already-archived entries
+    let meta: Record<string, any> = {};
+    try { meta = JSON.parse(entry.metadata || "{}"); } catch { /* skip */ }
+    if (meta.state === "archived" || meta.state === "superseded") continue;
+
+    // Proportional demotion: higher similarity → bigger cut
+    // At sim=1.0 → full maxDemotion, at threshold → near-zero
+    const simRange = 1.0 - config.similarityThreshold;
+    const simNormalized = simRange > 0
+      ? (candidate.score - config.similarityThreshold) / simRange
+      : 1.0;
+    const demotion = config.maxDemotion * simNormalized;
+
+    const newImportance = Math.max(
+      entry.importance - demotion,
+      config.importanceFloor,
+    );
+
+    // Skip if no meaningful change
+    if (Math.abs(newImportance - entry.importance) < 0.01) continue;
+
+    // Audit trail
+    if (!Array.isArray(meta.cascade_forget)) meta.cascade_forget = [];
+    meta.cascade_forget.push({
+      forgottenId: forgottenEntry.id.slice(0, 8),
+      from: entry.importance,
+      to: newImportance,
+      similarity: candidate.score,
+      date: new Date().toISOString().slice(0, 10),
+    });
+
+    // Demote tier if importance drops below working threshold
+    if (meta.tier === "working" && newImportance < 0.5) {
+      meta.tier = "peripheral";
+    } else if (meta.tier === "core" && newImportance < 0.8) {
+      meta.tier = "working";
+    }
+
+    await store.update(entry.id, {
+      importance: newImportance,
+      metadata: JSON.stringify(meta),
+    }, [forgottenEntry.scope]);
+
+    demotedIds.push(entry.id);
+  }
+
+  return { demotedCount: demotedIds.length, demotedIds };
+}

package/src/chunker.ts

@@ -0,0 +1,253 @@
+/**
+ * Long Context Chunking System
+ *
+ * Goal: split documents that exceed embedding model context limits into smaller,
+ * semantically coherent chunks with overlap.
+ *
+ * Notes:
+ * - We use *character counts* as a conservative proxy for tokens.
+ * - The embedder triggers this only after a provider throws a context-length error.
+ */
+
+// ============================================================================
+// Types & Constants
+// ============================================================================
+
+export interface ChunkMetadata {
+  startIndex: number;
+  endIndex: number;
+  length: number;
+}
+
+export interface ChunkResult {
+  chunks: string[];
+  metadatas: ChunkMetadata[];
+  totalOriginalLength: number;
+  chunkCount: number;
+}
+
+export interface ChunkerConfig {
+  /** Maximum characters per chunk. */
+  maxChunkSize: number;
+  /** Overlap between chunks in characters. */
+  overlapSize: number;
+  /** Minimum chunk size (except the final chunk). */
+  minChunkSize: number;
+  /** Attempt to split on sentence boundaries for better semantic coherence. */
+  semanticSplit: boolean;
+  /** Max lines per chunk before we try to split earlier on a line boundary. */
+  maxLinesPerChunk: number;
+}
+
+// Common embedding context limits (provider/model specific). These are typically
+// token limits, but we treat them as inputs to a conservative char-based heuristic.
+export const EMBEDDING_CONTEXT_LIMITS: Record<string, number> = {
+  // Jina v5
+  "jina-embeddings-v5-text-small": 8192,
+  "jina-embeddings-v5-text-nano": 8192,
+
+  // OpenAI
+  "text-embedding-3-small": 8192,
+  "text-embedding-3-large": 8192,
+
+  // Google
+  "text-embedding-004": 8192,
+  "gemini-embedding-001": 2048,
+
+  // Local/common
+  "nomic-embed-text": 8192,
+  "all-MiniLM-L6-v2": 512,
+  "all-mpnet-base-v2": 512,
+};
+
+export const DEFAULT_CHUNKER_CONFIG: ChunkerConfig = {
+  maxChunkSize: 4000,
+  overlapSize: 200,
+  minChunkSize: 200,
+  semanticSplit: true,
+  maxLinesPerChunk: 50,
+};
+
+// Sentence ending patterns (English + CJK-ish punctuation)
+const SENTENCE_ENDING = /[.!?。！？]/;
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+function clamp(n: number, lo: number, hi: number): number {
+  return Math.max(lo, Math.min(hi, n));
+}
+
+function countLines(s: string): number {
+  // Count \n (treat CRLF as one line break)
+  return s.split(/\r\n|\n|\r/).length;
+}
+
+function findLastIndexWithin(text: string, re: RegExp, start: number, end: number): number {
+  // Find last match start index for regex within [start, end).
+  // NOTE: `re` must NOT be global; we will scan manually.
+  let last = -1;
+  for (let i = end - 1; i >= start; i--) {
+    if (re.test(text[i])) return i;
+  }
+  return last;
+}
+
+function findSplitEnd(text: string, start: number, maxEnd: number, minEnd: number, config: ChunkerConfig): number {
+  const safeMinEnd = clamp(minEnd, start + 1, maxEnd);
+  const safeMaxEnd = clamp(maxEnd, safeMinEnd, text.length);
+
+  // Respect line limit: if we exceed maxLinesPerChunk, force earlier split at a line break.
+  if (config.maxLinesPerChunk > 0) {
+    const candidate = text.slice(start, safeMaxEnd);
+    if (countLines(candidate) > config.maxLinesPerChunk) {
+      // Find the position of the Nth line break.
+      let breaks = 0;
+      for (let i = start; i < safeMaxEnd; i++) {
+        const ch = text[i];
+        if (ch === "\n") {
+          breaks++;
+          if (breaks >= config.maxLinesPerChunk) {
+            // Split right after this newline.
+            return Math.max(i + 1, safeMinEnd);
+          }
+        }
+      }
+    }
+  }
+
+  if (config.semanticSplit) {
+    // Prefer a sentence boundary near the end.
+    // Scan backward from safeMaxEnd to safeMinEnd.
+    for (let i = safeMaxEnd - 1; i >= safeMinEnd; i--) {
+      if (SENTENCE_ENDING.test(text[i])) {
+        // Include trailing whitespace after punctuation.
+        let j = i + 1;
+        while (j < safeMaxEnd && /\s/.test(text[j])) j++;
+        return j;
+      }
+    }
+
+    // Next best: newline boundary.
+    for (let i = safeMaxEnd - 1; i >= safeMinEnd; i--) {
+      if (text[i] === "\n") return i + 1;
+    }
+  }
+
+  // Fallback: last whitespace boundary.
+  for (let i = safeMaxEnd - 1; i >= safeMinEnd; i--) {
+    if (/\s/.test(text[i])) return i;
+  }
+
+  return safeMaxEnd;
+}
+
+function sliceTrimWithIndices(text: string, start: number, end: number): { chunk: string; meta: ChunkMetadata } {
+  const raw = text.slice(start, end);
+  const leading = raw.match(/^\s*/)?.[0]?.length ?? 0;
+  const trailing = raw.match(/\s*$/)?.[0]?.length ?? 0;
+  const chunk = raw.trim();
+
+  const trimmedStart = start + leading;
+  const trimmedEnd = end - trailing;
+
+  return {
+    chunk,
+    meta: {
+      startIndex: trimmedStart,
+      endIndex: Math.max(trimmedStart, trimmedEnd),
+      length: chunk.length,
+    },
+  };
+}
+
+// ============================================================================
+// Chunking Core
+// ============================================================================
+
+export function chunkDocument(text: string, config: ChunkerConfig = DEFAULT_CHUNKER_CONFIG): ChunkResult {
+  if (!text || text.trim().length === 0) {
+    return { chunks: [], metadatas: [], totalOriginalLength: 0, chunkCount: 0 };
+  }
+
+  const totalOriginalLength = text.length;
+  const chunks: string[] = [];
+  const metadatas: ChunkMetadata[] = [];
+
+  let pos = 0;
+  const maxGuard = Math.max(4, Math.ceil(text.length / Math.max(1, config.maxChunkSize - config.overlapSize)) + 5);
+  let guard = 0;
+
+  while (pos < text.length && guard < maxGuard) {
+    guard++;
+
+    const remaining = text.length - pos;
+    if (remaining <= config.maxChunkSize) {
+      const { chunk, meta } = sliceTrimWithIndices(text, pos, text.length);
+      if (chunk.length > 0) {
+        chunks.push(chunk);
+        metadatas.push(meta);
+      }
+      break;
+    }
+
+    const maxEnd = Math.min(pos + config.maxChunkSize, text.length);
+    const minEnd = Math.min(pos + config.minChunkSize, maxEnd);
+
+    const end = findSplitEnd(text, pos, maxEnd, minEnd, config);
+    const { chunk, meta } = sliceTrimWithIndices(text, pos, end);
+
+    // If trimming made it too small, fall back to a hard split.
+    if (chunk.length < config.minChunkSize) {
+      const hardEnd = Math.min(pos + config.maxChunkSize, text.length);
+      const hard = sliceTrimWithIndices(text, pos, hardEnd);
+      if (hard.chunk.length > 0) {
+        chunks.push(hard.chunk);
+        metadatas.push(hard.meta);
+      }
+      if (hardEnd >= text.length) break;
+      pos = Math.max(hardEnd - config.overlapSize, pos + 1);
+      continue;
+    }
+
+    chunks.push(chunk);
+    metadatas.push(meta);
+
+    if (end >= text.length) break;
+
+    // Move forward with overlap.
+    const nextPos = Math.max(end - config.overlapSize, pos + 1);
+    pos = nextPos;
+  }
+
+  return {
+    chunks,
+    metadatas,
+    totalOriginalLength,
+    chunkCount: chunks.length,
+  };
+}
+
+/**
+ * Smart chunker that adapts to model context limits.
+ *
+ * We intentionally pick conservative char limits (70% of the reported limit)
+ * since token/char ratios vary.
+ */
+export function smartChunk(text: string, embedderModel?: string): ChunkResult {
+  const limit = embedderModel ? EMBEDDING_CONTEXT_LIMITS[embedderModel] : undefined;
+  const base = limit ?? 8192;
+
+  const config: ChunkerConfig = {
+    maxChunkSize: Math.max(1000, Math.floor(base * 0.7)),
+    overlapSize: Math.max(0, Math.floor(base * 0.05)),
+    minChunkSize: Math.max(100, Math.floor(base * 0.1)),
+    semanticSplit: true,
+    maxLinesPerChunk: 50,
+  };
+
+  return chunkDocument(text, config);
+}
+
+export default chunkDocument;