recallnest 1.4.1 → 2.2.1

package/src/consolidation-engine.ts

@@ -19,6 +19,10 @@
 
 import type { MemoryEntry, MemorySearchResult, MemoryStore } from "./store.js";
 import { createVersionGroup } from "./version-manager.js";
+import { isActiveMemory, parseEvolution, buildSupersedeMetadata, buildConsolidatedMetadata, patchEvolution } from "./memory-evolution.js";
+import { cosineSimilarity } from "./multi-vector.js";
+import type { LLMClient } from "./llm-client.js";
+import type { Embedder } from "./embedder.js";
 
 // ---------------------------------------------------------------------------
 // Config & Types
@@ -64,18 +68,17 @@ function parseMetadata(entry: MemoryEntry): Record<string, unknown> {
 }
 
 function isActive(entry: MemoryEntry): boolean {
-  const meta = parseMetadata(entry);
-  return meta.state !== "archived" && meta.state !== "superseded";
+  // Unified lifecycle check via evolution.status (no legacy meta.state fallback)
+  return isActiveMemory(entry.metadata);
 }
 
 function canonicalScore(entry: MemoryEntry): number {
-  const meta = parseMetadata(entry);
-  const accessCount = typeof meta.accessCount === "number" ? meta.accessCount : 0;
-  return entry.importance * (1 + Math.log(accessCount + 1));
+  const evo = parseEvolution(entry.metadata, entry.timestamp);
+  return entry.importance * (1 + Math.log(evo.accessCount + 1));
 }
 
 /** Simple heuristic contradiction: negation pattern check between two texts. */
-function detectHeuristicContradiction(textA: string, textB: string): boolean {
+export function detectHeuristicContradiction(textA: string, textB: string): boolean {
   const a = textA.toLowerCase();
   const b = textB.toLowerCase();
 
@@ -197,9 +200,19 @@ export class ConsolidationEngine {
           const sim = scoreMap.get(member.id) ?? clusterThreshold;
 
           if (sim >= mergeThreshold) {
-            // Tier 3.3: Version coexistence instead of SUPERSEDE.
-            // Both entries stay active but are grouped — retrieval picks the best.
+            // Tier 3.3: Version coexistence — both stay but grouped.
             await createVersionGroup(this.store, canonical, member, scope);
+            // C-1: Also mark the weaker entry as consolidated via evolution metadata
+            const consolidatedMeta = buildConsolidatedMetadata(member.metadata, canonical.id);
+            await this.store.update(member.id, { metadata: consolidatedMeta }, [scope]);
+            // Mark canonical as having source memories
+            const canonEvo = parseEvolution(canonical.metadata, canonical.timestamp);
+            if (!canonEvo.sourceMemories.includes(member.id)) {
+              const updatedCanon = patchEvolution(canonical.metadata, {
+                sourceMemories: [...canonEvo.sourceMemories, member.id],
+              });
+              await this.store.update(canonical.id, { metadata: updatedCanon }, [scope]);
+            }
             mergedCount++;
           } else {
             // Link: add clustered_with relation
@@ -236,6 +249,243 @@ export class ConsolidationEngine {
   }
 }
 
+// ---------------------------------------------------------------------------
+// C-2: Cluster Consolidation — group similar memories and generate insights
+// ---------------------------------------------------------------------------
+
+export interface ClusterConsolidationResult {
+  clustersFound: number;
+  clustersConsolidated: number;
+  insightsGenerated: number;
+  /** HP-5: Number of cross-memory patterns discovered */
+  patternsExtracted: number;
+  entriesLinked: number;
+  /** CC-9: Set when consecutive low-yield rounds trigger early termination */
+  earlyStop?: "diminishing_returns";
+}
+
+/**
+ * C-2: Cluster consolidation — group similar memories and generate insights.
+ *
+ * Algorithm:
+ * 1. Take active memories in a scope
+ * 2. Cluster by embedding similarity (simple greedy clustering, not full K-means)
+ * 3. For clusters with > minClusterSize members, generate a high-level insight via LLM
+ * 4. Store insight as new memory, link source memories via evolution sourceMemories field
+ * 5. Source memories marked consolidated_into but remain active (still individually searchable)
+ */
+export async function clusterAndConsolidate(params: {
+  entries: MemoryEntry[];
+  embedder: Pick<Embedder, "embedPassage">;
+  llm: LLMClient;
+  store: Pick<MemoryStore, "store" | "update">;
+  scope: string;
+  /** Minimum cluster size to trigger consolidation (default: 3) */
+  minClusterSize?: number;
+  /** Similarity threshold for clustering (default: 0.75) */
+  clusterThreshold?: number;
+  /** Max clusters to process per run (default: 5) */
+  maxClusters?: number;
+  /** HP-5: Enable cross-memory pattern extraction after insight generation (default: false) */
+  extractPatterns?: boolean;
+}): Promise<ClusterConsolidationResult> {
+  const {
+    entries,
+    llm,
+    store,
+    scope,
+    minClusterSize = 3,
+    clusterThreshold = 0.75,
+    maxClusters = 5,
+    extractPatterns = false,
+  } = params;
+
+  const result: ClusterConsolidationResult = {
+    clustersFound: 0,
+    clustersConsolidated: 0,
+    insightsGenerated: 0,
+    patternsExtracted: 0,
+    entriesLinked: 0,
+  };
+
+  // Filter to active entries with vectors
+  const active = entries.filter(e => isActiveMemory(e.metadata) && e.vector?.length > 0);
+  if (active.length === 0) return result;
+
+  // Step 1: Greedy clustering by embedding similarity
+  const clusters: MemoryEntry[][] = [];
+  const centroids: number[][] = [];
+  const assigned = new Set<string>();
+
+  for (const entry of active) {
+    if (assigned.has(entry.id)) continue;
+
+    let bestClusterIdx = -1;
+    let bestSim = -1;
+
+    for (let ci = 0; ci < centroids.length; ci++) {
+      const sim = cosineSimilarity(entry.vector, centroids[ci]);
+      if (sim > clusterThreshold && sim > bestSim) {
+        bestSim = sim;
+        bestClusterIdx = ci;
+      }
+    }
+
+    if (bestClusterIdx >= 0) {
+      clusters[bestClusterIdx].push(entry);
+      assigned.add(entry.id);
+      // Update centroid as running average
+      const members = clusters[bestClusterIdx];
+      const dim = centroids[bestClusterIdx].length;
+      const newCentroid = new Array<number>(dim);
+      for (let d = 0; d < dim; d++) {
+        let sum = 0;
+        for (const m of members) sum += m.vector[d];
+        newCentroid[d] = sum / members.length;
+      }
+      centroids[bestClusterIdx] = newCentroid;
+    } else {
+      // Start a new cluster
+      clusters.push([entry]);
+      centroids.push([...entry.vector]);
+      assigned.add(entry.id);
+    }
+  }
+
+  // Step 2: Filter to clusters meeting minClusterSize
+  const qualifiedClusters = clusters.filter(c => c.length >= minClusterSize);
+  result.clustersFound = qualifiedClusters.length;
+
+  if (qualifiedClusters.length === 0) return result;
+
+  // Step 3: Process up to maxClusters, with CC-9 diminishing returns detection
+  const toProcess = qualifiedClusters.slice(0, maxClusters);
+  let consecutiveLowYield = 0;
+
+  for (const cluster of toProcess) {
+    // CC-9: Check diminishing returns — 2 consecutive rounds with <= 1 insight
+    if (consecutiveLowYield >= 2) {
+      result.earlyStop = "diminishing_returns";
+      break;
+    }
+
+    // Generate insight via LLM from cluster member texts
+    const combinedText = cluster.map(m => m.text).join("\n---\n");
+    const insight = await llm.generateL0(combinedText);
+
+    if (!insight) {
+      // CC-9: No insight produced — this is a zero-yield round
+      consecutiveLowYield++;
+      result.clustersConsolidated++;
+      continue;
+    }
+
+    // Determine category (majority vote from cluster members)
+    const catCounts = new Map<string, number>();
+    for (const m of cluster) {
+      catCounts.set(m.category, (catCounts.get(m.category) ?? 0) + 1);
+    }
+    let bestCat = cluster[0].category;
+    let bestCount = 0;
+    for (const [cat, count] of catCounts) {
+      if (count > bestCount) {
+        bestCat = cat as MemoryEntry["category"];
+        bestCount = count;
+      }
+    }
+
+    // Importance = max of cluster members
+    const maxImportance = Math.max(...cluster.map(m => m.importance));
+
+    // Embed the insight text
+    const insightVector = await params.embedder.embedPassage(insight);
+
+    // Store the insight as a new memory
+    const insightEntry = await store.store({
+      text: insight,
+      vector: insightVector,
+      category: bestCat,
+      scope,
+      importance: maxImportance,
+      metadata: JSON.stringify({
+        evolution: {
+          status: "active",
+          version: 1,
+          accessCount: 0,
+          lastAccessedAt: null,
+          supersededBy: null,
+          consolidatedInto: null,
+          sourceMemories: cluster.map(m => m.id),
+          validFrom: Date.now(),
+          validUntil: null,
+        },
+        cluster_insight: true,
+      }),
+    });
+
+    result.insightsGenerated++;
+
+    // Mark source memories as consolidated_into (but keep them active)
+    for (const member of cluster) {
+      const patched = patchEvolution(member.metadata, {
+        consolidatedInto: insightEntry.id,
+        // Keep status active — still individually searchable
+      });
+      await store.update(member.id, { metadata: patched }, [scope]);
+      result.entriesLinked++;
+    }
+
+    result.clustersConsolidated++;
+
+    // HP-5: Cross-memory pattern extraction
+    if (extractPatterns && cluster.length >= 3) {
+      const patternText = await llm.extractPattern(cluster.map(m => m.text));
+      if (patternText) {
+        const patternImportance = Math.min(maxImportance + 0.1, 1.0);
+        const patternVector = await params.embedder.embedPassage(patternText);
+        const patternEntry = await store.store({
+          text: patternText,
+          vector: patternVector,
+          category: "patterns",
+          scope,
+          importance: patternImportance,
+          metadata: JSON.stringify({
+            evolution: {
+              status: "active",
+              version: 1,
+              accessCount: 0,
+              lastAccessedAt: null,
+              supersededBy: null,
+              consolidatedInto: null,
+              contributedToPattern: null,
+              sourceMemories: cluster.map(m => m.id),
+              validFrom: Date.now(),
+              validUntil: null,
+            },
+            cross_memory_pattern: true,
+            source_cluster_size: cluster.length,
+          }),
+        });
+
+        // Mark source memories with contributedToPattern
+        for (const member of cluster) {
+          const patched = patchEvolution(member.metadata, {
+            contributedToPattern: patternEntry.id,
+          });
+          await store.update(member.id, { metadata: patched }, [scope]);
+        }
+
+        result.patternsExtracted++;
+      }
+    }
+
+    // CC-9: Successful insight generated — reset diminishing returns counter
+    consecutiveLowYield = 0;
+  }
+
+  return result;
+}
+
 /** Format a ConsolidationResult for display. */
 export function formatConsolidationResult(result: ConsolidationResult): string {
   const lines = [
@@ -254,3 +504,41 @@ export function formatConsolidationResult(result: ConsolidationResult): string {
   }
   return lines.join("\n");
 }
+
+// ---------------------------------------------------------------------------
+// LC-P2: Cluster-aware deduplication for retrieval results
+// ---------------------------------------------------------------------------
+
+/**
+ * LC-P2: When a cluster insight and its source memories both appear in
+ * retrieval results, keep only the cluster insight (it subsumes the
+ * individual entries). This saves token budget during context injection.
+ *
+ * Works with any result shape that has { entry: MemoryEntry; score: number }.
+ */
+export function deduplicateByClusterInsight<T extends { entry: MemoryEntry; score: number }>(
+  results: T[],
+): T[] {
+  // Collect source memory IDs covered by cluster insights in this result set
+  const coveredByInsight = new Set<string>();
+
+  for (const r of results) {
+    if (!r.entry.metadata) continue;
+    try {
+      const meta = JSON.parse(r.entry.metadata);
+      if (meta.cluster_insight === true || meta.cross_memory_pattern === true) {
+        const sources: unknown[] = meta.evolution?.sourceMemories;
+        if (Array.isArray(sources)) {
+          for (const id of sources) {
+            if (typeof id === "string") coveredByInsight.add(id);
+          }
+        }
+      }
+    } catch { /* skip unparseable */ }
+  }
+
+  if (coveredByInsight.size === 0) return results;
+
+  // Filter out source memories that are subsumed by an insight
+  return results.filter(r => !coveredByInsight.has(r.entry.id));
+}

package/src/context-collapse-renderer.ts

@@ -0,0 +1,222 @@
+/**
+ * CC-7: Context Collapse — dynamic granularity rendering layer.
+ *
+ * Given retrieval results + scores, renders each item at L0/L1/L2 granularity
+ * based on relevance score. High-relevance items get full content (L2),
+ * medium gets overview (L1), low gets one-liner (L0).
+ *
+ * Also includes CC-2 staleness hints: items older than 7 days get an age warning
+ * so the model can judge whether to trust the information.
+ *
+ * Reuses existing L0/L1/L2 text stored in metadata by the multi-vector pipeline.
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type RenderLevel = "L0" | "L1" | "L2";
+
+export interface CollapseInput {
+  entryId: string;
+  text: string;
+  metadata?: string;
+  score: number;
+  timestamp: number;
+}
+
+export interface CollapseOutput {
+  entryId: string;
+  text: string;
+  renderLevel: RenderLevel;
+  stalenessHint?: string;
+}
+
+export interface CollapseConfig {
+  /** Total token budget for all rendered items (default: 8000) */
+  tokenBudget: number;
+  /** Score thresholds for each level */
+  thresholds: { l2: number; l1: number; l0: number };
+  /** Days before staleness hint is added (default: 7) */
+  stalenessThresholdDays: number;
+}
+
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+
+export const DEFAULT_COLLAPSE_CONFIG: CollapseConfig = {
+  tokenBudget: 8000,
+  thresholds: { l2: 0.85, l1: 0.65, l0: 0.50 },
+  stalenessThresholdDays: 7,
+};
+
+// ---------------------------------------------------------------------------
+// Text extraction at each level
+// ---------------------------------------------------------------------------
+
+function parseMetaJson(metadata?: string): Record<string, unknown> | null {
+  if (!metadata) return null;
+  try {
+    return JSON.parse(metadata);
+  } catch {
+    return null;
+  }
+}
+
+/** Extract L0 text (one-liner, ≤80 chars). Falls back to first sentence of full text. */
+function extractL0(text: string, metadata?: string): string {
+  const meta = parseMetaJson(metadata);
+  if (meta) {
+    if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 5) return meta.l0_abstract;
+    if (typeof meta.anchor === "string" && meta.anchor.length > 5) return meta.anchor;
+  }
+  // Fallback: first sentence or first 80 chars
+  const firstSentence = text.match(/^[^.!?。！？\n]+[.!?。！？]?/)?.[0];
+  if (firstSentence && firstSentence.length <= 120) return firstSentence;
+  return text.slice(0, 80) + (text.length > 80 ? "…" : "");
+}
+
+/** Extract L1 text (structured overview, 2-5 lines). Falls back to L0. */
+function extractL1(text: string, metadata?: string): string {
+  const meta = parseMetaJson(metadata);
+  if (meta) {
+    if (typeof meta.core_summary === "string" && meta.core_summary.length > 5) return meta.core_summary;
+    if (typeof meta.l1_overview === "string" && meta.l1_overview.length > 5) return meta.l1_overview;
+  }
+  // Fallback: truncated full text
+  if (text.length <= 300) return text;
+  return text.slice(0, 300) + "…";
+}
+
+/** Extract L2 text (full content). */
+function extractL2(text: string): string {
+  return text;
+}
+
+// ---------------------------------------------------------------------------
+// Token estimation
+// ---------------------------------------------------------------------------
+
+/** Rough token estimate: CJK chars ≈ 1.5 tokens each, others ≈ 0.25 per char. */
+function estimateTokens(text: string): number {
+  let tokens = 0;
+  for (const char of text) {
+    tokens += /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]/.test(char) ? 1.5 : 0.25;
+  }
+  return Math.ceil(tokens);
+}
+
+// ---------------------------------------------------------------------------
+// Staleness hint (CC-2)
+// ---------------------------------------------------------------------------
+
+function buildStalenessHint(timestamp: number, thresholdDays: number, now?: number): string | undefined {
+  const ts = now ?? Date.now();
+  const daysSinceUpdate = Math.floor((ts - timestamp) / 86_400_000);
+  if (daysSinceUpdate <= thresholdDays) return undefined;
+  return `[Updated ${daysSinceUpdate} days ago — verify before acting on this]`;
+}
+
+// ---------------------------------------------------------------------------
+// Core: collapse renderer
+// ---------------------------------------------------------------------------
+
+/**
+ * Render retrieval results at adaptive granularity based on relevance score.
+ *
+ * - score >= l2 threshold → full content (L2)
+ * - score >= l1 threshold → overview (L1)
+ * - score >= l0 threshold → one-liner (L0)
+ * - below l0 threshold → excluded
+ *
+ * Token budget is enforced: once budget is exhausted, remaining items
+ * are downgraded to L0 or excluded.
+ */
+export function collapseResults(
+  items: CollapseInput[],
+  config?: Partial<CollapseConfig>,
+): CollapseOutput[] {
+  const cfg: CollapseConfig = { ...DEFAULT_COLLAPSE_CONFIG, ...config };
+  const { thresholds, tokenBudget, stalenessThresholdDays } = cfg;
+
+  // Sort by score descending (highest relevance first)
+  const sorted = [...items].sort((a, b) => b.score - a.score);
+
+  const output: CollapseOutput[] = [];
+  let tokensUsed = 0;
+
+  for (const item of sorted) {
+    // Below minimum threshold → skip
+    if (item.score < thresholds.l0) continue;
+
+    // Determine target level based on score
+    let targetLevel: RenderLevel;
+    if (item.score >= thresholds.l2) {
+      targetLevel = "L2";
+    } else if (item.score >= thresholds.l1) {
+      targetLevel = "L1";
+    } else {
+      targetLevel = "L0";
+    }
+
+    // Extract text at target level
+    let text: string;
+    let actualLevel: RenderLevel = targetLevel;
+
+    if (targetLevel === "L2") {
+      text = extractL2(item.text);
+    } else if (targetLevel === "L1") {
+      text = extractL1(item.text, item.metadata);
+    } else {
+      text = extractL0(item.text, item.metadata);
+    }
+
+    // Check token budget — downgrade if needed
+    const tokens = estimateTokens(text);
+    if (tokensUsed + tokens > tokenBudget) {
+      // Try downgrading to L1
+      if (actualLevel === "L2") {
+        text = extractL1(item.text, item.metadata);
+        actualLevel = "L1";
+        const l1Tokens = estimateTokens(text);
+        if (tokensUsed + l1Tokens > tokenBudget) {
+          // Further downgrade to L0
+          text = extractL0(item.text, item.metadata);
+          actualLevel = "L0";
+          const l0Tokens = estimateTokens(text);
+          if (tokensUsed + l0Tokens > tokenBudget) continue; // Skip entirely
+          tokensUsed += l0Tokens;
+        } else {
+          tokensUsed += l1Tokens;
+        }
+      } else if (actualLevel === "L1") {
+        text = extractL0(item.text, item.metadata);
+        actualLevel = "L0";
+        const l0Tokens = estimateTokens(text);
+        if (tokensUsed + l0Tokens > tokenBudget) continue;
+        tokensUsed += l0Tokens;
+      } else {
+        // Already L0 and still over budget → skip
+        continue;
+      }
+    } else {
+      tokensUsed += tokens;
+    }
+
+    // CC-2: Staleness hint
+    const stalenessHint = buildStalenessHint(item.timestamp, stalenessThresholdDays);
+
+    output.push({
+      entryId: item.entryId,
+      text,
+      renderLevel: actualLevel,
+      ...(stalenessHint ? { stalenessHint } : {}),
+    });
+  }
+
+  return output;
+}
+
+// Re-export helpers for testing
+export { extractL0, extractL1, extractL2, estimateTokens, buildStalenessHint };

package/src/context-composer-stable-selection.ts

@@ -2,6 +2,7 @@ import { extractBoundaryMetadata, extractCanonicalKey, shouldUseStableMemoryResu
 import { buildProjectScopeCueTerms, normalizeScopedValue } from "./context-composer-scope.js";
 import { bestSummaryText, cleanText, dedupeText, stripConversationMarkers } from "./context-composer-text.js";
 import type { RetrievalResult } from "./retriever.js";
+import { getConfidence } from "./confidence-tracker.js";
 import {
   GENERIC_ENTITY_TASK_TERMS,
   GENERIC_SCOPE_TERMS,
@@ -158,7 +159,10 @@ function hasUnsupportedPreferenceSpecificity(result: RetrievalResult, taskSeed?:
 
 function formatStableResult(category: StableCategory, result: RetrievalResult): string {
   const text = bestSummaryText(result.entry.text, result.entry.metadata);
-  return cleanText(`${STABLE_CATEGORY_LABELS[category]}: ${text}`, 220);
+  // F1: Tag low-confidence memories in resume_context output
+  const conf = getConfidence(result.entry);
+  const tag = conf < 0.5 ? " [低置信]" : "";
+  return cleanText(`${STABLE_CATEGORY_LABELS[category]}: ${text}${tag}`, 230);
 }
 
 export function selectStableResults(