recallnest 2.3.0 → 2.5.4

package/src/__tests__/retrieval-trace.test.ts

@@ -0,0 +1,213 @@
+import { describe, expect, it } from "bun:test";
+import { TraceCollector } from "../retrieval-trace.js";
+import { createRetriever } from "../retriever.js";
+
+describe("TraceCollector", () => {
+  it("accumulates stages correctly", () => {
+    const trace = new TraceCollector();
+    trace.startStage("vector_search", 0);
+    trace.endStage(20, [0.3, 0.9]);
+
+    trace.startStage("min_score_filter", 20);
+    trace.endStage(15, [0.35, 0.9]);
+
+    const result = trace.finalize("test query", "hybrid");
+    expect(result.query).toBe("test query");
+    expect(result.mode).toBe("hybrid");
+    expect(result.stages).toHaveLength(2);
+    expect(result.finalCount).toBe(15);
+    expect(result.totalMs).toBeGreaterThanOrEqual(0);
+  });
+
+  it("computes droppedCount as input - output", () => {
+    const trace = new TraceCollector();
+    trace.startStage("hard_min_score", 20);
+    trace.endStage(12);
+
+    const result = trace.finalize("q", "vector");
+    expect(result.stages[0].droppedCount).toBe(8);
+  });
+
+  it("computes scoreRange from surviving scores", () => {
+    const trace = new TraceCollector();
+    trace.startStage("rerank", 10);
+    trace.endStage(8, [0.2, 0.5, 0.8, 0.3, 0.6, 0.7, 0.4, 0.9]);
+
+    const stage = trace.finalize("q", "hybrid").stages[0];
+    expect(stage.scoreRange).not.toBeNull();
+    expect(stage.scoreRange![0]).toBeCloseTo(0.2);
+    expect(stage.scoreRange![1]).toBeCloseTo(0.9);
+  });
+
+  it("returns null scoreRange when no scores provided", () => {
+    const trace = new TraceCollector();
+    trace.startStage("noise_filter", 5);
+    trace.endStage(3);
+
+    expect(trace.finalize("q", "hybrid").stages[0].scoreRange).toBeNull();
+  });
+
+  it("summarize() produces readable multi-line output", () => {
+    const trace = new TraceCollector();
+    trace.startStage("vector_search", 0);
+    trace.endStage(20, [0.31, 0.89]);
+    trace.startStage("min_score_filter", 20);
+    trace.endStage(18, [0.35, 0.89]);
+
+    const summary = trace.summarize("auth middleware", "hybrid");
+    expect(summary).toContain('[retrieve] query="auth middleware" mode=hybrid');
+    expect(summary).toContain("vector_search: 0 → 20");
+    expect(summary).toContain("min_score_filter: 20 → 18 (2 dropped)");
+    expect(summary).toContain("final: 18 results");
+  });
+});
+
+describe("retriever with trace", () => {
+  it("populates trace stages without changing retrieval behavior", async () => {
+    const trace = new TraceCollector();
+
+    const retriever = createRetriever(
+      {
+        hasFtsSupport: false,
+        async vectorSearch() {
+          return [
+            {
+              entry: {
+                id: "m1",
+                text: "test memory",
+                vector: [1, 0, 0],
+                category: "entities",
+                scope: "project:test",
+                importance: 0.8,
+                timestamp: Date.now(),
+                metadata: "{}",
+              },
+              score: 0.8,
+            },
+          ];
+        },
+      } as any,
+      {
+        async embedQuery() { return [1, 0, 0]; },
+        async embedPassage() { return [1, 0, 0]; },
+      } as any,
+      {
+        mode: "vector",
+        rerank: "none",
+        filterNoise: false,
+        hardMinScore: 0,
+        minScore: 0,
+        recencyWeight: 0,
+        timeDecayHalfLifeDays: 0,
+        lengthNormAnchor: 0,
+      },
+    );
+
+    const results = await retriever.retrieve({
+      query: "authentication middleware config",
+      limit: 5,
+      trace,
+    });
+
+    // Results unchanged
+    expect(results).toHaveLength(1);
+    expect(results[0].entry.id).toBe("m1");
+
+    // Trace populated
+    const traceResult = trace.finalize("test", "vector");
+    expect(traceResult.stages.length).toBeGreaterThan(0);
+    expect(traceResult.stages[0].name).toBe("vector_search");
+    expect(traceResult.finalCount).toBe(1);
+  });
+
+  it("works identically without trace (no overhead)", async () => {
+    const retriever = createRetriever(
+      {
+        hasFtsSupport: false,
+        async vectorSearch() {
+          return [
+            {
+              entry: {
+                id: "m1",
+                text: "test memory",
+                vector: [1, 0, 0],
+                category: "entities",
+                scope: "project:test",
+                importance: 0.8,
+                timestamp: Date.now(),
+                metadata: "{}",
+              },
+              score: 0.8,
+            },
+          ];
+        },
+      } as any,
+      {
+        async embedQuery() { return [1, 0, 0]; },
+        async embedPassage() { return [1, 0, 0]; },
+      } as any,
+      {
+        mode: "vector",
+        rerank: "none",
+        filterNoise: false,
+        hardMinScore: 0,
+        minScore: 0,
+        recencyWeight: 0,
+        timeDecayHalfLifeDays: 0,
+        lengthNormAnchor: 0,
+      },
+    );
+
+    // No trace passed — should work exactly as before
+    const results = await retriever.retrieve({
+      query: "authentication middleware config",
+      limit: 5,
+    });
+
+    expect(results).toHaveLength(1);
+    expect(results[0].entry.id).toBe("m1");
+  });
+
+  it("trace captures hard_min_score drops", async () => {
+    const trace = new TraceCollector();
+
+    const retriever = createRetriever(
+      {
+        hasFtsSupport: false,
+        async vectorSearch() {
+          return [
+            {
+              entry: { id: "high", text: "good", vector: [1, 0, 0], category: "entities", scope: "p:t", importance: 0.8, timestamp: Date.now(), metadata: "{}" },
+              score: 0.9,
+            },
+            {
+              entry: { id: "low", text: "weak", vector: [0, 1, 0], category: "entities", scope: "p:t", importance: 0.5, timestamp: Date.now(), metadata: "{}" },
+              score: 0.2,
+            },
+          ];
+        },
+      } as any,
+      {
+        async embedQuery() { return [1, 0, 0]; },
+        async embedPassage() { return [1, 0, 0]; },
+      } as any,
+      {
+        mode: "vector",
+        rerank: "none",
+        filterNoise: false,
+        hardMinScore: 0.35,
+        minScore: 0,
+        recencyWeight: 0,
+        timeDecayHalfLifeDays: 0,
+        lengthNormAnchor: 0,
+      },
+    );
+
+    await retriever.retrieve({ query: "authentication middleware config", limit: 5, trace });
+
+    const result = trace.finalize("test", "vector");
+    const hardStage = result.stages.find(s => s.name === "hard_min_score");
+    expect(hardStage).toBeDefined();
+    expect(hardStage!.droppedCount).toBeGreaterThan(0);
+  });
+});

package/src/__tests__/retriever-boundary-weight.test.ts

@@ -0,0 +1,136 @@
+import { describe, expect, it } from "bun:test";
+
+import { createRetriever } from "../retriever.js";
+
+function buildEntry(id: string, scoreText: string, scope: string, metadata: Record<string, unknown>) {
+  return {
+    entry: {
+      id,
+      text: scoreText,
+      vector: [1, 0, 0],
+      category: "preferences",
+      scope,
+      importance: 0.8,
+      timestamp: Date.parse("2026-03-16T00:00:00.000Z"),
+      metadata: JSON.stringify(metadata),
+    },
+    score: 0.8,
+  };
+}
+
+describe("retriever provenance weighting", () => {
+  it("prefers durable structured memories over evidence when scores are close", async () => {
+    const retriever = createRetriever({
+      hasFtsSupport: false,
+      async vectorSearch() {
+        return [
+          {
+            ...buildEntry("evidence-1", "Transcript hint", "cc:session1", {
+              source: "cc",
+              boundary: {
+                layer: "evidence",
+                authority: "transcript-ingest",
+                conflictPolicy: "append-only",
+                originalCategory: "preferences",
+              },
+            }),
+            score: 0.8,
+          },
+          {
+            ...buildEntry("durable-1", "Curated durable preference", "memory:agent", {
+              source: "agent",
+              boundary: {
+                layer: "durable",
+                authority: "structured-memory",
+                conflictPolicy: "latest-wins",
+                originalCategory: "preferences",
+              },
+            }),
+            score: 0.79,
+          },
+        ];
+      },
+    } as any, {
+      async embedQuery() {
+        return [1, 0, 0];
+      },
+      async embedPassage() {
+        return [1, 0, 0];
+      },
+    } as any, {
+      mode: "vector",
+      rerank: "none",
+      filterNoise: false,
+      hardMinScore: 0,
+      minScore: 0,
+      recencyWeight: 0,
+      timeDecayHalfLifeDays: 0,
+    });
+
+    const results = await retriever.retrieve({
+      query: "user reply style preference",
+      limit: 5,
+      category: "preferences",
+    });
+
+    expect(results[0]?.entry.id).toBe("durable-1");
+    expect(results[1]?.entry.id).toBe("evidence-1");
+  });
+
+  it("does not override clearly stronger relevance with provenance weighting", async () => {
+    const retriever = createRetriever({
+      hasFtsSupport: false,
+      async vectorSearch() {
+        return [
+          {
+            ...buildEntry("evidence-strong", "Very strong transcript evidence", "cc:session1", {
+              source: "cc",
+              boundary: {
+                layer: "evidence",
+                authority: "transcript-ingest",
+                conflictPolicy: "append-only",
+                originalCategory: "preferences",
+              },
+            }),
+            score: 0.95,
+          },
+          {
+            ...buildEntry("durable-weaker", "Weaker durable preference", "memory:agent", {
+              source: "agent",
+              boundary: {
+                layer: "durable",
+                authority: "structured-memory",
+                conflictPolicy: "latest-wins",
+                originalCategory: "preferences",
+              },
+            }),
+            score: 0.7,
+          },
+        ];
+      },
+    } as any, {
+      async embedQuery() {
+        return [1, 0, 0];
+      },
+      async embedPassage() {
+        return [1, 0, 0];
+      },
+    } as any, {
+      mode: "vector",
+      rerank: "none",
+      filterNoise: false,
+      hardMinScore: 0,
+      minScore: 0,
+      recencyWeight: 0,
+      timeDecayHalfLifeDays: 0,
+    });
+
+    const results = await retriever.retrieve({
+      query: "user reply style preference",
+      limit: 5,
+      category: "preferences",
+    });
+
+    expect(results[0]?.entry.id).toBe("evidence-strong");
+  });
+});

package/src/__tests__/retroactive-boost.test.ts

@@ -0,0 +1,223 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+  retroactiveBoost,
+  DEFAULT_RETROACTIVE_BOOST_CONFIG,
+  type RetroactiveBoostConfig,
+} from "../retroactive-boost.js";
+import type { MemoryEntry, MemorySearchResult } from "../store.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeEntry(id: string, text: string, overrides: Partial<MemoryEntry> = {}): MemoryEntry {
+  return {
+    id,
+    text,
+    vector: [1, 0, 0],
+    category: "events",
+    scope: "project:test",
+    importance: 0.5,
+    timestamp: Date.now() - 7 * 86_400_000, // 7 days old by default
+    metadata: "{}",
+    ...overrides,
+  };
+}
+
+function createMockStore(
+  entries: MemoryEntry[],
+  searchResults: MemorySearchResult[] = [],
+) {
+  const data = new Map(entries.map(e => [e.id, { ...e }]));
+  const updates: Array<{ id: string; importance?: number; metadata?: string }> = [];
+
+  return {
+    updates,
+    data,
+    async vectorSearch(_vector: number[], limit = 5, minScore = 0.3, _scopeFilter?: string[]) {
+      return searchResults.filter(r => r.score >= minScore).slice(0, limit);
+    },
+    async update(id: string, upd: any, _scopeFilter?: string[]) {
+      const entry = data.get(id);
+      if (!entry) return null;
+      if (upd.importance !== undefined) entry.importance = upd.importance;
+      if (upd.metadata !== undefined) entry.metadata = upd.metadata;
+      updates.push({ id, ...upd });
+      return entry;
+    },
+  } as any;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("retroactive-boost (STC)", () => {
+  it("skips when new entry has low importance", async () => {
+    const store = createMockStore([]);
+    const newEntry = makeEntry("new-1", "low importance note", { importance: 0.5 });
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.triggered).toBe(false);
+    expect(result.reason).toBe("below_importance_threshold");
+  });
+
+  it("skips when new entry has no vector", async () => {
+    const store = createMockStore([]);
+    const newEntry = makeEntry("new-1", "important note", {
+      importance: 0.9,
+      vector: [],
+    });
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.triggered).toBe(false);
+    expect(result.reason).toBe("no_vector");
+  });
+
+  it("boosts related low-importance old entries", async () => {
+    const oldEntry = makeEntry("old-1", "Singapore weather info", {
+      importance: 0.3,
+      timestamp: Date.now() - 30 * 86_400_000, // 30 days old
+    });
+    const searchResults: MemorySearchResult[] = [
+      { entry: oldEntry, score: 0.85 },
+    ];
+    const store = createMockStore([oldEntry], searchResults);
+
+    const newEntry = makeEntry("new-1", "I'm moving to Singapore", {
+      importance: 0.9,
+      vector: [1, 0, 0],
+    });
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.triggered).toBe(true);
+    expect(result.boostedCount).toBe(1);
+    expect(result.boostedIds).toContain("old-1");
+
+    // Verify the update was applied
+    expect(store.updates.length).toBe(1);
+    expect(store.updates[0].importance).toBe(0.3 + 0.15); // 0.45
+  });
+
+  it("does not boost entries that are already high importance", async () => {
+    const highEntry = makeEntry("high-1", "already important", {
+      importance: 0.7,
+      timestamp: Date.now() - 30 * 86_400_000,
+    });
+    const searchResults: MemorySearchResult[] = [
+      { entry: highEntry, score: 0.85 },
+    ];
+    const store = createMockStore([highEntry], searchResults);
+
+    const newEntry = makeEntry("new-1", "trigger", { importance: 0.9, vector: [1, 0, 0] });
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.triggered).toBe(true);
+    expect(result.boostedCount).toBe(0); // high-1 was already above threshold
+  });
+
+  it("does not boost very recent entries", async () => {
+    const recentEntry = makeEntry("recent-1", "just stored", {
+      importance: 0.3,
+      timestamp: Date.now() - 3_600_000, // 1 hour old
+    });
+    const searchResults: MemorySearchResult[] = [
+      { entry: recentEntry, score: 0.85 },
+    ];
+    const store = createMockStore([recentEntry], searchResults);
+
+    const newEntry = makeEntry("new-1", "trigger", { importance: 0.9, vector: [1, 0, 0] });
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.triggered).toBe(true);
+    expect(result.boostedCount).toBe(0); // too recent
+  });
+
+  it("caps boosted importance at maxBoostedImportance", async () => {
+    const oldEntry = makeEntry("old-1", "related note", {
+      importance: 0.55,
+      timestamp: Date.now() - 30 * 86_400_000,
+    });
+    const searchResults: MemorySearchResult[] = [
+      { entry: oldEntry, score: 0.85 },
+    ];
+    const store = createMockStore([oldEntry], searchResults);
+
+    const newEntry = makeEntry("new-1", "trigger", { importance: 0.9, vector: [1, 0, 0] });
+
+    const config: RetroactiveBoostConfig = {
+      ...DEFAULT_RETROACTIVE_BOOST_CONFIG,
+      boostAmount: 0.3,           // would push to 0.85
+      maxBoostedImportance: 0.75, // but capped at 0.75
+    };
+
+    const result = await retroactiveBoost(store, newEntry, config);
+    expect(result.boostedCount).toBe(1);
+    expect(store.updates[0].importance).toBe(0.75); // capped
+  });
+
+  it("promotes tier from peripheral to working when boosted above 0.5", async () => {
+    const oldEntry = makeEntry("old-1", "peripheral memory", {
+      importance: 0.3,
+      timestamp: Date.now() - 30 * 86_400_000,
+      metadata: JSON.stringify({ tier: "peripheral" }),
+    });
+    const searchResults: MemorySearchResult[] = [
+      { entry: oldEntry, score: 0.85 },
+    ];
+    const store = createMockStore([oldEntry], searchResults);
+
+    const newEntry = makeEntry("new-1", "important trigger", { importance: 0.9, vector: [1, 0, 0] });
+
+    const config: RetroactiveBoostConfig = {
+      ...DEFAULT_RETROACTIVE_BOOST_CONFIG,
+      boostAmount: 0.25, // 0.3 + 0.25 = 0.55 > 0.5
+    };
+
+    const result = await retroactiveBoost(store, newEntry, config);
+    expect(result.boostedCount).toBe(1);
+
+    // Check metadata for tier promotion
+    const updatedMeta = JSON.parse(store.updates[0].metadata);
+    expect(updatedMeta.tier).toBe("working");
+    expect(updatedMeta.stc_boosts).toHaveLength(1);
+    expect(updatedMeta.stc_boosts[0].triggeredBy).toBe("new-1".slice(0, 8));
+  });
+
+  it("respects maxBoostPerTrigger limit", async () => {
+    const entries: MemoryEntry[] = [];
+    const searchResults: MemorySearchResult[] = [];
+
+    for (let i = 0; i < 10; i++) {
+      const e = makeEntry(`old-${i}`, `entry ${i}`, {
+        importance: 0.3,
+        timestamp: Date.now() - 30 * 86_400_000,
+      });
+      entries.push(e);
+      searchResults.push({ entry: e, score: 0.85 - i * 0.01 });
+    }
+
+    const store = createMockStore(entries, searchResults);
+    const newEntry = makeEntry("new-1", "trigger", { importance: 0.9, vector: [1, 0, 0] });
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.boostedCount).toBe(5); // default maxBoostPerTrigger
+  });
+
+  it("skips self in search results", async () => {
+    const newEntry = makeEntry("new-1", "I'm important", {
+      importance: 0.9,
+      vector: [1, 0, 0],
+    });
+    // Search returns the new entry itself
+    const searchResults: MemorySearchResult[] = [
+      { entry: newEntry, score: 1.0 },
+    ];
+    const store = createMockStore([newEntry], searchResults);
+
+    const result = await retroactiveBoost(store, newEntry);
+    expect(result.triggered).toBe(true);
+    expect(result.boostedCount).toBe(0);
+  });
+});

package/src/__tests__/salience-scoring.test.ts

@@ -0,0 +1,106 @@
+import { describe, it, expect } from "bun:test";
+import type { LLMClient } from "../llm-client.js";
+
+function createMockLLM(response: string): LLMClient {
+  return {
+    async chat() {
+      return response;
+    },
+    async assessImportance(text: string) {
+      const parsed = JSON.parse(response);
+      return parsed.importance ?? null;
+    },
+    async reassessImportanceBatch(
+      entries: Array<{ id: string; text: string; category: string; currentImportance: number }>,
+    ) {
+      const results = new Map<string, number>();
+      try {
+        const parsed = JSON.parse(response);
+        if (Array.isArray(parsed)) {
+          for (const item of parsed) {
+            if (item.idx >= 0 && item.idx < entries.length) {
+              results.set(entries[item.idx].id, item.importance);
+            }
+          }
+        }
+      } catch {
+        // match real implementation: non-fatal
+      }
+      return results;
+    },
+  } as any;
+}
+
+describe("HP-4: Salience scoring anchoring", () => {
+  describe("assessImportance anchored tiers", () => {
+    it("identity/correction → 0.85+", async () => {
+      const llm = createMockLLM(JSON.stringify({ importance: 0.92, reason: "用户明确纠正" }));
+      const result = await llm.assessImportance("不要用 rm 删除文件", "profile");
+      expect(result).toBeGreaterThanOrEqual(0.85);
+    });
+
+    it("decision/case → 0.70-0.84", async () => {
+      const llm = createMockLLM(JSON.stringify({ importance: 0.75, reason: "架构决策" }));
+      const result = await llm.assessImportance("选用 LanceDB 作为向量存储", "cases");
+      expect(result).toBeGreaterThanOrEqual(0.70);
+      expect(result).toBeLessThan(0.85);
+    });
+
+    it("entity/background → 0.55-0.69", async () => {
+      const llm = createMockLLM(JSON.stringify({ importance: 0.60, reason: "项目信息" }));
+      const result = await llm.assessImportance("RecallNest 使用 Bun 运行时", "entities");
+      expect(result).toBeGreaterThanOrEqual(0.55);
+      expect(result).toBeLessThan(0.70);
+    });
+
+    it("event/one-off → 0.40-0.54", async () => {
+      const llm = createMockLLM(JSON.stringify({ importance: 0.45, reason: "一次性操作" }));
+      const result = await llm.assessImportance("今天部署了 v2.1", "events");
+      expect(result).toBeGreaterThanOrEqual(0.40);
+      expect(result).toBeLessThan(0.55);
+    });
+
+    it("chatter/temp → 0.10-0.30", async () => {
+      const llm = createMockLLM(JSON.stringify({ importance: 0.15, reason: "临时上下文" }));
+      const result = await llm.assessImportance("好的，我看一下", "events");
+      expect(result).toBeGreaterThanOrEqual(0.10);
+      expect(result).toBeLessThan(0.40);
+    });
+  });
+
+  describe("reassessImportanceBatch", () => {
+    it("returns adjusted entries with ≥ 0.1 delta", async () => {
+      const batchResponse = JSON.stringify([
+        { idx: 0, importance: 0.90 },
+        { idx: 2, importance: 0.30 },
+      ]);
+      const llm = createMockLLM(batchResponse);
+      const entries = [
+        { id: "a", text: "永远不要用 rm", category: "profile", currentImportance: 0.70 },
+        { id: "b", text: "选用 LanceDB", category: "cases", currentImportance: 0.75 },
+        { id: "c", text: "今天天气不错", category: "events", currentImportance: 0.70 },
+      ];
+      const result = await llm.reassessImportanceBatch(entries);
+      expect(result.size).toBe(2);
+      expect(result.get("a")).toBe(0.90);
+      expect(result.get("c")).toBe(0.30);
+      expect(result.has("b")).toBe(false);
+    });
+
+    it("returns empty map on empty input", async () => {
+      const llm = createMockLLM("[]");
+      const result = await llm.reassessImportanceBatch([]);
+      expect(result.size).toBe(0);
+    });
+
+    it("handles invalid LLM response gracefully", async () => {
+      const llm = createMockLLM("not-json");
+      const entries = [
+        { id: "a", text: "test", category: "events", currentImportance: 0.5 },
+      ];
+      // Should not throw
+      const result = await llm.reassessImportanceBatch(entries);
+      expect(result.size).toBe(0);
+    });
+  });
+});