recallnest 1.4.1 → 2.2.0

package/src/ingest.ts

@@ -11,6 +11,7 @@
 import { readFileSync, readdirSync, existsSync, statSync, writeFileSync } from "node:fs";
 import { join, basename, resolve, dirname } from "node:path";
 import { homedir } from "node:os";
+import { detectLang, tokenizeFts } from "./language-hook.js";
 import type { MemoryStore, MemoryEntry } from "./store.js";
 import type { Embedder } from "./embedder.js";
 import { chunkDocument, type ChunkerConfig } from "./chunker.js";
@@ -26,6 +27,7 @@ import {
   samePreferenceSlot,
 } from "./preference-slots.js";
 import { compressToolOutput } from "./tool-output-compressor.js";
+import { tagNarrativeIfEnabled } from "./narrative-tagger.js";
 
 // ============================================================================
 // Types
@@ -178,7 +180,7 @@ function normalizeDedupText(value: string): string {
  * 3. Explicit memory instructions (记住, remember, 以后注意, 别忘了)
  * 4. Feedback/corrections (不要, 别再, stop doing, don't)
  */
-function isTemporalOrActionContent(text: string): boolean {
+export function isTemporalOrActionContent(text: string): boolean {
   // Date patterns: ISO dates, Chinese relative dates
   const datePattern = /\b20\d{2}[-/]\d{2}[-/]\d{2}\b|昨天|今天|前天|上次|刚才|刚刚|本周|这周|上周/;
   // File operation patterns
@@ -586,13 +588,18 @@ const PENDING_EXTRACTION_FILE = resolve(
 function queueForLaterExtraction(chunks: Array<{ text: string; scope: string }>): void {
   let pending: Array<{ text: string; scope: string; queuedAt: string }> = [];
   try {
-    pending = JSON.parse(readFileSync(PENDING_EXTRACTION_FILE, "utf-8"));
+    const raw = JSON.parse(readFileSync(PENDING_EXTRACTION_FILE, "utf-8"));
+    if (Array.isArray(raw)) pending = raw;
   } catch { /* empty or missing */ }
   const now = new Date().toISOString();
   for (const chunk of chunks) {
     pending.push({ ...chunk, queuedAt: now });
   }
-  writeFileSync(PENDING_EXTRACTION_FILE, JSON.stringify(pending, null, 2));
+  try {
+    writeFileSync(PENDING_EXTRACTION_FILE, JSON.stringify(pending, null, 2));
+  } catch (err) {
+    console.error("[recallnest] Failed to write pending extraction queue:", err instanceof Error ? err.message : String(err));
+  }
 }
 
 export async function drainPendingQueue(
@@ -602,7 +609,12 @@ export async function drainPendingQueue(
 ): Promise<{ processed: number; errors: number }> {
   let pending: Array<{ text: string; scope: string; queuedAt: string }> = [];
   try {
-    pending = JSON.parse(readFileSync(PENDING_EXTRACTION_FILE, "utf-8"));
+    const raw = JSON.parse(readFileSync(PENDING_EXTRACTION_FILE, "utf-8"));
+    if (!Array.isArray(raw)) {
+      console.error("[recallnest] Pending extraction queue is malformed (not an array), resetting");
+      return { processed: 0, errors: 0 };
+    }
+    pending = raw;
   } catch { return { processed: 0, errors: 0 }; }
 
   if (pending.length === 0) return { processed: 0, errors: 0 };
@@ -623,13 +635,18 @@ export async function drainPendingQueue(
 
     for (const j of keepIndices) {
       try {
+        const entryText = extractions[j].l1 || extractions[j].l0;
+        const language = detectLang(entryText);
+        const fts_text = tokenizeFts(entryText, language);
         await store.store({
-          text: extractions[j].l1 || extractions[j].l0,
+          text: entryText,
           vector: vectors[j],
           category: extractions[j].category as any,
           scope: batch[j].scope,
           importance: extractions[j].importance,
           metadata: JSON.stringify({ source: batch[j].scope.split(":")[0], l0_abstract: extractions[j].l0, l1_overview: extractions[j].l1, l2_content: extractions[j].l1 || extractions[j].l0 }),
+          language,
+          fts_text,
         });
         processed++;
       } catch { errors++; }
@@ -637,7 +654,11 @@ export async function drainPendingQueue(
   }
 
   // Clear the queue
-  writeFileSync(PENDING_EXTRACTION_FILE, "[]");
+  try {
+    writeFileSync(PENDING_EXTRACTION_FILE, "[]");
+  } catch (err) {
+    console.error("[recallnest] Failed to clear pending extraction queue:", err instanceof Error ? err.message : String(err));
+  }
   return { processed, errors };
 }
 
@@ -684,6 +705,14 @@ function buildIngestedEntry(params: {
         importance: params.extraction.importance,
       });
 
+  // HP-narrative: Tag with autobiographical narrative metadata when enabled
+  const narrative = tagNarrativeIfEnabled({
+    scope: params.scope,
+    text: params.text,
+    timestamp: Date.now(),
+    sessionId: params.sessionId,
+  });
+
   return {
     text: params.text,
     vector: params.vector,
@@ -701,6 +730,7 @@ function buildIngestedEntry(params: {
       ...(params.coreSummary ? { core_summary: params.coreSummary } : {}),
       tier,
       boundary: resolution.boundary,
+      ...(narrative ? { narrative } : {}),
     }),
   };
 }

package/src/interference-detector.ts

@@ -0,0 +1,187 @@
+/**
+ * F2: Interference Detector — detects semantic interference clusters
+ * and measures interference density within a scope.
+ *
+ * Brain-science basis: proactive interference (PI) occurs when
+ * semantically similar memories compete during retrieval, degrading
+ * signal-to-noise. This module identifies interference clusters and
+ * ranks members so the weakest can be deprioritized.
+ */
+
+import type { MemoryEntry } from "./store.js";
+import { parseEvolution } from "./memory-evolution.js";
+import { getConfidence } from "./confidence-tracker.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type InterferenceRisk = "high" | "medium" | "low";
+
+export interface InterferenceResult {
+  entryId: string;
+  clusterId: number;
+  clusterRank: number; // 0 = strongest in cluster
+  interferenceRisk: InterferenceRisk;
+  clusterSize: number;
+}
+
+export interface InterferenceDensity {
+  totalMemories: number;
+  clusterCount: number;
+  avgClusterSize: number;
+  highRiskCount: number;
+}
+
+// ---------------------------------------------------------------------------
+// Cosine Similarity (shared with rif.ts — keep inline to avoid circular dep)
+// ---------------------------------------------------------------------------
+
+function cosineSimilarity(a: number[] | Float32Array, b: number[] | Float32Array): number {
+  if (!a || !b || a.length !== b.length || a.length === 0) return 0;
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom > 0 ? dot / denom : 0;
+}
+
+// ---------------------------------------------------------------------------
+// Composite score for cluster ranking
+// ---------------------------------------------------------------------------
+
+function compositeScore(entry: MemoryEntry, now: number): number {
+  const evo = parseEvolution(entry.metadata, entry.timestamp);
+  const confidence = getConfidence(entry);
+  const importance = entry.importance ?? 0.5;
+  // Recency: exponential decay with 60-day half-life
+  const daysSince = Math.max(0, (now - entry.timestamp) / 86_400_000);
+  const recency = Math.pow(0.5, daysSince / 60);
+  return confidence * importance * recency;
+}
+
+// ---------------------------------------------------------------------------
+// Cluster detection via greedy single-linkage
+// ---------------------------------------------------------------------------
+
+/**
+ * Detect interference clusters within a set of memories.
+ * Uses greedy single-linkage: if sim(A,B) > threshold, they join the same cluster.
+ *
+ * @param memories - Memory entries with vectors
+ * @param similarityThreshold - Cosine sim threshold for "same cluster" (default: 0.80)
+ */
+export function detectInterference(
+  memories: MemoryEntry[],
+  similarityThreshold = 0.80,
+): InterferenceResult[] {
+  if (memories.length === 0) return [];
+
+  const now = Date.now();
+  // Union-Find for cluster assignment
+  const parent = memories.map((_, i) => i);
+  function find(x: number): number {
+    while (parent[x] !== x) { parent[x] = parent[parent[x]]; x = parent[x]; }
+    return x;
+  }
+  function union(a: number, b: number): void {
+    const ra = find(a), rb = find(b);
+    if (ra !== rb) parent[ra] = rb;
+  }
+
+  // Pairwise similarity check
+  for (let i = 0; i < memories.length; i++) {
+    for (let j = i + 1; j < memories.length; j++) {
+      const sim = cosineSimilarity(memories[i].vector, memories[j].vector);
+      if (sim > similarityThreshold) {
+        union(i, j);
+      }
+    }
+  }
+
+  // Group by cluster root
+  const clusters = new Map<number, number[]>(); // root → indices
+  for (let i = 0; i < memories.length; i++) {
+    const root = find(i);
+    if (!clusters.has(root)) clusters.set(root, []);
+    clusters.get(root)!.push(i);
+  }
+
+  const results: InterferenceResult[] = [];
+  let clusterId = 0;
+
+  for (const members of clusters.values()) {
+    if (members.length < 2) {
+      // Singleton — no interference
+      results.push({
+        entryId: memories[members[0]].id,
+        clusterId,
+        clusterRank: 0,
+        interferenceRisk: "low",
+        clusterSize: 1,
+      });
+      clusterId++;
+      continue;
+    }
+
+    // Rank within cluster by composite score
+    const ranked = members
+      .map(idx => ({ idx, score: compositeScore(memories[idx], now) }))
+      .sort((a, b) => b.score - a.score);
+
+    for (let rank = 0; rank < ranked.length; rank++) {
+      const risk: InterferenceRisk =
+        rank === 0 ? "low" :
+        rank <= 2 ? "medium" : "high";
+
+      results.push({
+        entryId: memories[ranked[rank].idx].id,
+        clusterId,
+        clusterRank: rank,
+        interferenceRisk: risk,
+        clusterSize: ranked.length,
+      });
+    }
+    clusterId++;
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Density measurement
+// ---------------------------------------------------------------------------
+
+/**
+ * Measure interference density: how clustered are the memories?
+ */
+export function measureInterferenceDensity(
+  memories: MemoryEntry[],
+  similarityThreshold = 0.80,
+): InterferenceDensity {
+  const results = detectInterference(memories, similarityThreshold);
+  const multiMemberClusters = new Map<number, number>();
+
+  for (const r of results) {
+    if (r.clusterSize >= 2) {
+      multiMemberClusters.set(r.clusterId, r.clusterSize);
+    }
+  }
+
+  const clusterSizes = [...multiMemberClusters.values()];
+  const clusterCount = clusterSizes.length;
+  const avgClusterSize = clusterCount > 0
+    ? clusterSizes.reduce((a, b) => a + b, 0) / clusterCount
+    : 0;
+  const highRiskCount = results.filter(r => r.interferenceRisk === "high").length;
+
+  return {
+    totalMemories: memories.length,
+    clusterCount,
+    avgClusterSize,
+    highRiskCount,
+  };
+}

package/src/kg-extractor.ts

@@ -7,6 +7,7 @@
 import type { LLMClient } from "./llm-client.js";
 import type { KGStore } from "./kg-store.js";
 import { logInfo, logWarn } from "./stderr-log.js";
+import { detectLang, getKgPromptHook } from "./language-hook.js";
 
 // ============================================================================
 // Types
@@ -86,33 +87,19 @@ export function normalizePredicate(raw: string): string {
 }
 
 // ============================================================================
-// Extraction Prompt
+// Extraction Prompt (fallback when babel-memory is not installed)
 // ============================================================================
 
-const KG_EXTRACT_SYSTEM = `You are a knowledge graph extraction assistant. Extract (subject, predicate, object) triples from text. Respond with valid JSON only.`;
-
-function buildKGExtractionPrompt(text: string): string {
-  return `Extract knowledge graph triples from the text below.
+const DEFAULT_KG_SYSTEM = `You are a knowledge graph extraction assistant. Extract (subject, predicate, object, confidence) tuples from text. Respond with valid JSON only.`;
 
+const DEFAULT_KG_USER_TEMPLATE = `Extract knowledge graph triples from the following text.
 Rules:
-- Extract relationships between named entities, concepts, tools, people, projects
-- Use simple predicates: uses, created_by, works_with, friend_of, depends_on, located_in, member_of, prefers, knows, discussed, mentioned_in, related_to, is_a, has, manages, built, wrote, lives_in
-- Assign confidence (0-1) based on how explicit the relationship is
-- Skip vague or speculative relationships (confidence < 0.5)
-- 3-8 triples per text typically
-
-Examples:
-
-Text: "Alice uses Python for her data science projects at Google"
-{"triples":[{"subject":"Alice","predicate":"uses","object":"Python","confidence":0.95},{"subject":"Alice","predicate":"works_at","object":"Google","confidence":0.90}]}
+- Use simple predicates: uses, created_by, works_with, is_a, part_of, has, located_in, belongs_to, depends_on, implements, extends, related_to, caused_by, results_in, precedes, follows
+- Assign confidence 0.0-1.0
+- Extract 3-8 triples
+- Return JSON: { "triples": [{ "subject": "", "predicate": "", "object": "", "confidence": 0.0 }] }
 
-Text: "RecallNest depends on LanceDB for vector storage and uses OpenAI for embeddings"
-{"triples":[{"subject":"RecallNest","predicate":"depends_on","object":"LanceDB","confidence":0.95},{"subject":"RecallNest","predicate":"uses","object":"OpenAI","confidence":0.90}]}
-
-Now extract triples from:
-
-${text}`;
-}
+Text: {text}`;
 
 // ============================================================================
 // Extractor
@@ -163,12 +150,16 @@ export class KGExtractor {
   async extract(text: string): Promise<RawTriple[]> {
     if (!text || text.length < 10) return [];
 
-    const prompt = buildKGExtractionPrompt(text);
+    const lang = detectLang(text);
+    const babelPrompt = getKgPromptHook(lang);
+    const system = babelPrompt?.system ?? DEFAULT_KG_SYSTEM;
+    const userTemplate = babelPrompt?.userTemplate ?? DEFAULT_KG_USER_TEMPLATE;
+    const prompt = userTemplate.replace("{text}", text);
 
     let response: LlmExtractionResponse | null = null;
     try {
       response = await this.llm.chatJson<LlmExtractionResponse>(
-        KG_EXTRACT_SYSTEM,
+        system,
         prompt,
       );
     } catch (err) {

package/src/kg-store.ts

@@ -125,8 +125,9 @@ export class KGStore {
         const lance = await loadLanceDB();
         await table.createIndex("subject", { config: (lance as any).Index.fts() });
       }
-    } catch {
+    } catch (err) {
       // FTS on subject is optional — BFS still works without it
+      console.error("[recallnest] KG FTS index creation skipped:", err instanceof Error ? err.message : String(err));
     }
 
     this.db = db;

package/src/language-hook.ts

@@ -0,0 +1,67 @@
+/**
+ * Language processing hook interface.
+ * Allows RecallNest to use any language processor (babel-memory or custom).
+ * If no processor is registered, all functions return safe defaults.
+ */
+export interface LanguageProcessor {
+  detectLanguage(text: string): string;
+  tokenizeForFts(text: string, lang: string): string;
+}
+
+interface PromptProvider {
+  getKgPrompt(lang: string): { system: string; userTemplate: string };
+  getSessionPrompt(lang: string): { system: string; dimensionLabels: Record<string, string> };
+}
+
+let processor: LanguageProcessor | null = null;
+let promptProvider: PromptProvider | null = null;
+
+export function registerLanguageProcessor(p: LanguageProcessor): void {
+  processor = p;
+}
+
+export function registerPromptProvider(p: PromptProvider): void {
+  promptProvider = p;
+}
+
+export function detectLang(text: string): string {
+  return processor?.detectLanguage(text) ?? "en";
+}
+
+export function tokenizeFts(text: string, lang: string): string {
+  return processor?.tokenizeForFts(text, lang) ?? text;
+}
+
+export function getKgPromptHook(lang: string): { system: string; userTemplate: string } | null {
+  return promptProvider?.getKgPrompt(lang) ?? null;
+}
+
+export function getSessionPromptHook(lang: string): { system: string; dimensionLabels: Record<string, string> } | null {
+  return promptProvider?.getSessionPrompt(lang) ?? null;
+}
+
+/**
+ * Auto-detect and register babel-memory if installed.
+ * Call once at server startup. Non-blocking.
+ */
+export async function autoRegisterBabelMemory(): Promise<boolean> {
+  try {
+    const bm = await import("babel-memory");
+    if (typeof bm.initTokenizer === "function") {
+      await bm.initTokenizer();
+    }
+    registerLanguageProcessor({
+      detectLanguage: bm.detectLanguage,
+      tokenizeForFts: bm.tokenizeForFts,
+    });
+    if (bm.getKgPrompt && bm.getSessionPrompt) {
+      registerPromptProvider({
+        getKgPrompt: bm.getKgPrompt,
+        getSessionPrompt: bm.getSessionPrompt,
+      });
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}