recallnest 2.3.0 → 2.6.0

package/src/__tests__/memory-promotion.test.ts

@@ -0,0 +1,315 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+  scanMemoryPromotions,
+  type PromoteScanDeps,
+  type PromoteRequest,
+} from "../memory-promotion.js";
+import type { MemoryEntry } from "../store.js";
+import type { StoredPromotedMemoryRecord } from "../memory-schema.js";
+
+// ---------------------------------------------------------------------------
+// Mock helpers
+// ---------------------------------------------------------------------------
+
+let idSeq = 0;
+
+const ACTIVE_EVOLUTION = {
+  status: "active",
+  version: 1,
+  accessCount: 0,
+  lastAccessedAt: null,
+  supersededBy: null,
+  consolidatedInto: null,
+  sourceMemories: [],
+  validFrom: 1_700_000_000_000,
+  validUntil: null,
+};
+
+/**
+ * Build a transcript-downgraded evidence entry (category=events) with an EMPTY
+ * vector, faithfully reproducing store.list — which never returns vectors.
+ */
+function makeDowngradedEvent(
+  downgradedFrom: "profile" | "preferences",
+  overrides: Partial<MemoryEntry> = {},
+): MemoryEntry {
+  idSeq += 1;
+  return {
+    id: `ev-${String(idSeq).padStart(4, "0")}`,
+    text: `${downgradedFrom} hint #${idSeq}`,
+    vector: [], // list() never returns vectors
+    category: "events",
+    scope: "cc:project:test",
+    importance: 0.7,
+    timestamp: 1_700_000_000_000 + idSeq,
+    metadata: JSON.stringify({
+      boundary: {
+        layer: "evidence",
+        authority: "transcript-ingest",
+        conflictPolicy: "append-only",
+        originalCategory: downgradedFrom,
+        downgradedFrom,
+      },
+      evolution: ACTIVE_EVOLUTION,
+    }),
+    ...overrides,
+  };
+}
+
+interface MockOptions {
+  /** Per-id vector overrides; ids absent from the map get no vector. */
+  vectorMap?: Map<string, number[]>;
+  /** Fallback vector handed to every listed id when vectorMap is omitted. */
+  defaultVector?: number[];
+}
+
+function createScanDeps(entries: MemoryEntry[], options: MockOptions = {}) {
+  const promoteCalls: PromoteRequest[] = [];
+  const deps: PromoteScanDeps = {
+    store: {
+      async list(
+        _scopeFilter?: string[],
+        category?: string,
+        _limit?: number,
+        _offset?: number,
+      ): Promise<MemoryEntry[]> {
+        return category ? entries.filter((e) => e.category === category) : entries;
+      },
+      async getVectors(ids: string[]): Promise<Map<string, number[]>> {
+        const map = new Map<string, number[]>();
+        for (const id of ids) {
+          if (options.vectorMap?.has(id)) {
+            map.set(id, options.vectorMap.get(id)!);
+          } else if (options.defaultVector && !options.vectorMap) {
+            map.set(id, options.defaultVector);
+          }
+        }
+        return map;
+      },
+    },
+    async promote(req: PromoteRequest): Promise<StoredPromotedMemoryRecord> {
+      promoteCalls.push(req);
+      return {
+        text: "promoted text",
+        category: req.category,
+        importance: req.importance,
+        scope: req.scope,
+        source: "agent",
+        tags: [],
+        canonicalKey: `${req.category}:k${promoteCalls.length}`,
+        id: `durable-${req.memoryId}`,
+        resolvedScope: req.scope,
+        storedAt: "2026-05-29T00:00:00.000Z",
+        disposition: "promoted",
+        sourceMemoryId: req.memoryId,
+        sourceCategory: "events",
+      };
+    },
+  };
+  return { deps, promoteCalls };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("scanMemoryPromotions", () => {
+  it("promotes a cluster once it has >= minOccurrences similar members", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(1);
+    expect(result.candidates[0].occurrences).toBe(3);
+    expect(result.candidates[0].downgradedFrom).toBe("preferences");
+    expect(result.promoted).toBe(1);
+    expect(promoteCalls.length).toBe(1);
+    expect(promoteCalls[0].category).toBe("preferences");
+    expect(promoteCalls[0].scope).toBe("cc:project:test");
+  });
+
+  it("does not promote when occurrences are below minOccurrences", async () => {
+    const entries = [makeDowngradedEvent("preferences"), makeDowngradedEvent("preferences")];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("does not promote a cluster whose average importance is below minImportance", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences", { importance: 0.4 }),
+      makeDowngradedEvent("preferences", { importance: 0.4 }),
+      makeDowngradedEvent("preferences", { importance: 0.4 }),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("promotes with the target category equal to downgradedFrom (profile)", async () => {
+    const entries = [
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(promoteCalls.length).toBe(1);
+    expect(promoteCalls[0].category).toBe("profile");
+  });
+
+  it("finds candidates but does NOT call promote in dryRun mode", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: true });
+
+    expect(result.candidates.length).toBe(1);
+    expect(result.dryRun).toBe(true);
+    expect(result.promoted).toBe(0);
+    expect(result.candidates[0].promoted).toBeNull();
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("keeps profile and preferences as separate candidates with their own category", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(2);
+    expect(promoteCalls.length).toBe(2);
+    expect(promoteCalls.map((c) => c.category).sort()).toEqual(["preferences", "profile"]);
+  });
+
+  it("skips entries that have no vector available", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+    ];
+    // Only 2 of 3 receive vectors -> the cluster has 2 members < minOccurrences.
+    const vectorMap = new Map<string, number[]>();
+    vectorMap.set(entries[0].id, [1, 0, 0]);
+    vectorMap.set(entries[1].id, [1, 0, 0]);
+    const { deps, promoteCalls } = createScanDeps(entries, { vectorMap });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("ignores events that are not downgraded from profile/preferences", async () => {
+    const plain = (): MemoryEntry =>
+      makeDowngradedEvent("preferences", {
+        metadata: JSON.stringify({
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+          },
+          evolution: ACTIVE_EVOLUTION,
+        }),
+      });
+    const { deps, promoteCalls } = createScanDeps([plain(), plain(), plain()], {
+      defaultVector: [1, 0, 0],
+    });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("skips evidence already promoted (promotedTo marker present)", async () => {
+    const promotedAlready = (): MemoryEntry =>
+      makeDowngradedEvent("preferences", {
+        metadata: JSON.stringify({
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+            originalCategory: "preferences",
+            downgradedFrom: "preferences",
+          },
+          promotedTo: "existing-durable-id",
+          evolution: ACTIVE_EVOLUTION,
+        }),
+      });
+    const { deps, promoteCalls } = createScanDeps(
+      [promotedAlready(), promotedAlready(), promotedAlready()],
+      { defaultVector: [1, 0, 0] },
+    );
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("excludes inactive (superseded) evidence", async () => {
+    const superseded = (): MemoryEntry =>
+      makeDowngradedEvent("preferences", {
+        metadata: JSON.stringify({
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+            originalCategory: "preferences",
+            downgradedFrom: "preferences",
+          },
+          evolution: { ...ACTIVE_EVOLUTION, status: "superseded", supersededBy: "x" },
+        }),
+      });
+    const { deps, promoteCalls } = createScanDeps(
+      [superseded(), superseded(), superseded()],
+      { defaultVector: [1, 0, 0] },
+    );
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("excludes evidence whose text is a bridge-injected prompt (noise)", async () => {
+    const promptText = "# CC Reply Prompt\n\n你是 CC。你不是 Alice 的数字分身，也不是 E 化镜像。";
+    const entries = [
+      makeDowngradedEvent("preferences", { text: promptText }),
+      makeDowngradedEvent("preferences", { text: promptText }),
+      makeDowngradedEvent("preferences", { text: promptText }),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+});

package/src/__tests__/migrate-dedup-ids.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "bun:test";
+
+import { planDedup, type DedupRow } from "../../scripts/migrate-dedup-ids.js";
+
+/**
+ * P0-2 迁移脚本核心逻辑回归。LanceDB I/O 部分（main）由「在备份副本上实跑」做集成验证；
+ * 这里只测纯 keeper 选择：同 id 保留最新 timestamp（importance 破平）、每 id 唯一、不丢文本。
+ */
+
+function row(id: string, ts: number, importance: number, text: string): DedupRow {
+  return { id, timestamp: ts, importance, text, vector: [0, 0, 0], scope: "s", category: "entities", metadata: "{}" };
+}
+
+describe("planDedup (dup-id migration keeper selection)", () => {
+  it("keeps the latest-timestamp row per dup id and counts extras", () => {
+    const rows = [
+      row("dup1", 100, 0.5, "A"),
+      row("dup1", 300, 0.6, "A"), // latest ts → keeper
+      row("dup1", 200, 0.9, "A"),
+      row("uniq", 150, 0.5, "B"),
+    ];
+    const plan = planDedup(rows);
+    expect(plan.dupIds).toEqual(["dup1"]);
+    expect(plan.extraRows).toBe(2);
+    expect(plan.keepers.length).toBe(1);
+    expect(Number(plan.keepers[0].timestamp)).toBe(300);
+  });
+
+  it("breaks timestamp ties by highest importance", () => {
+    const rows = [
+      row("d", 500, 0.4, "X"),
+      row("d", 500, 0.95, "X"), // same ts, higher importance → keeper
+      row("d", 500, 0.7, "X"),
+    ];
+    const plan = planDedup(rows);
+    expect(plan.keepers.length).toBe(1);
+    expect(plan.keepers[0].importance).toBe(0.95);
+  });
+
+  it("leaves unique ids untouched and preserves every distinct id's text", () => {
+    const rows = [
+      row("a", 1, 0.5, "textA"),
+      row("a", 2, 0.5, "textA"), // dup
+      row("b", 1, 0.5, "textB"), // unique
+      row("c", 1, 0.5, "textC"),
+      row("c", 5, 0.5, "textC"), // dup
+    ];
+    const plan = planDedup(rows);
+    const dupSet = new Set(plan.dupIds);
+    // Simulate the migration result: untouched unique-id rows + one keeper per dup id.
+    const survivors = [...rows.filter((r) => !dupSet.has(r.id)), ...plan.keepers];
+    const ids = survivors.map((r) => r.id).sort();
+    expect(ids).toEqual(["a", "b", "c"]); // every distinct id survives
+    expect(new Set(ids).size).toBe(ids.length); // each id appears exactly once
+    expect(survivors.find((r) => r.id === "a")!.text).toBe("textA");
+    expect(survivors.find((r) => r.id === "c")!.text).toBe("textC");
+  });
+
+  it("returns an empty plan when there are no duplicates", () => {
+    const plan = planDedup([row("x", 1, 0.5, "1"), row("y", 2, 0.5, "2")]);
+    expect(plan.dupIds).toEqual([]);
+    expect(plan.extraRows).toBe(0);
+    expect(plan.keepers).toEqual([]);
+  });
+});

package/src/__tests__/multi-vector.test.ts

@@ -0,0 +1,242 @@
+import { describe, expect, it, beforeEach, afterEach } from "bun:test";
+
+import {
+  blendMultiVectorScores,
+  cosineSimilarity,
+  DEFAULT_BLEND_CONFIG,
+  embedMultiVector,
+  extractMultiVectorText,
+  isMultiVectorEnabled,
+  tokenize,
+  textOverlapScore,
+  adaptiveBlendConfig,
+} from "../multi-vector.js";
+
+describe("isMultiVectorEnabled", () => {
+  const original = process.env.RECALLNEST_MULTI_VECTOR;
+
+  afterEach(() => {
+    if (original === undefined) {
+      delete process.env.RECALLNEST_MULTI_VECTOR;
+    } else {
+      process.env.RECALLNEST_MULTI_VECTOR = original;
+    }
+  });
+
+  it("returns false by default", () => {
+    delete process.env.RECALLNEST_MULTI_VECTOR;
+    expect(isMultiVectorEnabled()).toBe(false);
+  });
+
+  it("returns true when set to 'true'", () => {
+    process.env.RECALLNEST_MULTI_VECTOR = "true";
+    expect(isMultiVectorEnabled()).toBe(true);
+  });
+});
+
+describe("extractMultiVectorText", () => {
+  it("extracts L0 and L1 from valid metadata", () => {
+    const meta = JSON.stringify({
+      l0_abstract: "User prefers TypeScript",
+      l1_overview: "User consistently chooses TypeScript over JavaScript for type safety",
+    });
+    const result = extractMultiVectorText(meta);
+    expect(result.l0).toBe("User prefers TypeScript");
+    expect(result.l1).toContain("TypeScript");
+  });
+
+  it("returns empty for missing fields", () => {
+    const meta = JSON.stringify({ source: "manual" });
+    const result = extractMultiVectorText(meta);
+    expect(result.l0).toBeUndefined();
+    expect(result.l1).toBeUndefined();
+  });
+
+  it("returns empty for undefined metadata", () => {
+    const result = extractMultiVectorText(undefined);
+    expect(result.l0).toBeUndefined();
+  });
+
+  it("skips very short L0/L1 text (<=5 chars)", () => {
+    const meta = JSON.stringify({ l0_abstract: "abc", l1_overview: "short" });
+    const result = extractMultiVectorText(meta);
+    expect(result.l0).toBeUndefined();
+    expect(result.l1).toBeUndefined();
+  });
+});
+
+describe("embedMultiVector", () => {
+  const mockEmbedder = {
+    async embedPassage(text: string) {
+      return [text.length, 0.5, 0.1];
+    },
+  };
+
+  it("returns null vectors when disabled", async () => {
+    delete process.env.RECALLNEST_MULTI_VECTOR;
+    const result = await embedMultiVector(mockEmbedder, JSON.stringify({
+      l0_abstract: "Some abstract text here",
+    }));
+    expect(result.vector_l0).toBeNull();
+    expect(result.vector_l1).toBeNull();
+  });
+
+  it("generates vectors when enabled and text available", async () => {
+    process.env.RECALLNEST_MULTI_VECTOR = "true";
+    const result = await embedMultiVector(mockEmbedder, JSON.stringify({
+      l0_abstract: "User prefers TypeScript",
+      l1_overview: "Detailed overview of TypeScript preference with reasoning",
+    }));
+    expect(result.vector_l0).not.toBeNull();
+    expect(result.vector_l1).not.toBeNull();
+    expect(result.vector_l0![0]).toBe("User prefers TypeScript".length);
+    // Cleanup
+    delete process.env.RECALLNEST_MULTI_VECTOR;
+  });
+
+  it("returns null for missing L0/L1 text even when enabled", async () => {
+    process.env.RECALLNEST_MULTI_VECTOR = "true";
+    const result = await embedMultiVector(mockEmbedder, JSON.stringify({ source: "manual" }));
+    expect(result.vector_l0).toBeNull();
+    expect(result.vector_l1).toBeNull();
+    delete process.env.RECALLNEST_MULTI_VECTOR;
+  });
+});
+
+describe("blendMultiVectorScores", () => {
+  it("returns main score when no L0/L1 available", () => {
+    expect(blendMultiVectorScores(0.8, null, null)).toBe(0.8);
+  });
+
+  it("blends with L0 only", () => {
+    const blended = blendMultiVectorScores(0.8, 0.9, null);
+    // (0.8 * 0.65 + 0.9 * 0.20) / (0.65 + 0.20) = (0.52 + 0.18) / 0.85
+    expect(blended).toBeCloseTo(0.824, 2);
+  });
+
+  it("blends with both L0 and L1", () => {
+    const blended = blendMultiVectorScores(0.8, 0.9, 0.7);
+    // (0.8*0.65 + 0.9*0.20 + 0.7*0.15) / (0.65+0.20+0.15) = (0.52+0.18+0.105)/1.0
+    expect(blended).toBeCloseTo(0.805, 2);
+  });
+
+  it("supports custom weights", () => {
+    const config = { vectorWeight: 0.5, l0Weight: 0.3, l1Weight: 0.2 };
+    const blended = blendMultiVectorScores(0.8, 0.9, 0.7, config);
+    expect(blended).toBeCloseTo(0.81, 2);
+  });
+});
+
+describe("cosineSimilarity", () => {
+  it("returns 1 for identical vectors", () => {
+    expect(cosineSimilarity([1, 0, 0], [1, 0, 0])).toBeCloseTo(1.0, 5);
+  });
+
+  it("returns 0 for orthogonal vectors", () => {
+    expect(cosineSimilarity([1, 0, 0], [0, 1, 0])).toBeCloseTo(0.0, 5);
+  });
+
+  it("returns -1 for opposite vectors", () => {
+    expect(cosineSimilarity([1, 0, 0], [-1, 0, 0])).toBeCloseTo(-1.0, 5);
+  });
+
+  it("returns 0 for empty vectors", () => {
+    expect(cosineSimilarity([], [])).toBe(0);
+  });
+
+  it("returns 0 for mismatched dimensions", () => {
+    expect(cosineSimilarity([1, 0], [1, 0, 0])).toBe(0);
+  });
+});
+
+// ===========================================================================
+// Tier 4.3: Text-overlap multi-vector scoring
+// ===========================================================================
+
+describe("tokenize", () => {
+  it("tokenizes English text, removes stop words", () => {
+    const tokens = tokenize("The user prefers TypeScript for projects");
+    expect(tokens).toContain("user");
+    expect(tokens).toContain("prefers");
+    expect(tokens).toContain("typescript");
+    expect(tokens).toContain("projects");
+    expect(tokens).not.toContain("the");
+    expect(tokens).not.toContain("for");
+  });
+
+  it("tokenizes Chinese text into unigrams, removes stop words", () => {
+    const tokens = tokenize("用户喜欢使用TypeScript编程");
+    expect(tokens).toContain("用");
+    expect(tokens).toContain("户");
+    expect(tokens).toContain("喜");
+    expect(tokens).toContain("欢");
+    expect(tokens).toContain("typescript");
+    expect(tokens).not.toContain("的");
+  });
+
+  it("handles mixed CJK and English", () => {
+    const tokens = tokenize("RecallNest 记忆服务 is great");
+    expect(tokens).toContain("recallnest");
+    expect(tokens).toContain("记");
+    expect(tokens).toContain("忆");
+    expect(tokens).toContain("服");
+    expect(tokens).toContain("务");
+    expect(tokens).toContain("great");
+  });
+
+  it("returns empty for empty/whitespace input", () => {
+    expect(tokenize("")).toEqual([]);
+    expect(tokenize("   ")).toEqual([]);
+  });
+});
+
+describe("textOverlapScore", () => {
+  it("returns 1.0 when query fully matches a concise abstract", () => {
+    const score = textOverlapScore("TypeScript preference", "TypeScript preference");
+    expect(score).toBeGreaterThan(0.8);
+  });
+
+  it("returns higher score for L0 abstract than L2 full text on topic query", () => {
+    const query = "TypeScript projects";
+    const l0 = "User prefers TypeScript for all projects";
+    const l2 = "The user mentioned that they have been working with React and JavaScript for three years. They recently started using TypeScript for their projects because of type safety. They also mentioned Python for data science work.";
+
+    const l0Score = textOverlapScore(query, l0);
+    const l2Score = textOverlapScore(query, l2);
+    expect(l0Score).toBeGreaterThan(l2Score);
+  });
+
+  it("returns 0 for completely unrelated texts", () => {
+    const score = textOverlapScore("TypeScript projects", "天气很好适合出去玩");
+    expect(score).toBe(0);
+  });
+
+  it("returns 0 for empty query", () => {
+    expect(textOverlapScore("", "some text")).toBe(0);
+  });
+
+  it("returns 0 for empty candidate", () => {
+    expect(textOverlapScore("some query", "")).toBe(0);
+  });
+});
+
+describe("adaptiveBlendConfig", () => {
+  it("boosts L0/L1 for short queries (≤3 tokens)", () => {
+    const config = adaptiveBlendConfig(2);
+    expect(config.l0Weight).toBe(0.25);
+    expect(config.l1Weight).toBe(0.25);
+    expect(config.vectorWeight).toBe(0.50);
+  });
+
+  it("uses default for medium queries (4-10 tokens)", () => {
+    const config = adaptiveBlendConfig(7);
+    expect(config).toEqual(DEFAULT_BLEND_CONFIG);
+  });
+
+  it("favors L2 for long queries (>10 tokens)", () => {
+    const config = adaptiveBlendConfig(15);
+    expect(config.vectorWeight).toBe(0.80);
+    expect(config.l0Weight).toBe(0.10);
+    expect(config.l1Weight).toBe(0.10);
+  });
+});