recallnest 2.6.1 → 3.0.1

package/src/__tests__/babel-integration.test.ts

@@ -1,144 +0,0 @@
-import { describe, expect, test, beforeAll, afterAll } from "bun:test";
-import { MemoryStore } from "../store.js";
-import { rmSync, mkdirSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-
-// babel-memory is an optional dependency — skip integration tests if not installed
-let babelMemory: typeof import("babel-memory") | null = null;
-try {
-  babelMemory = await import("babel-memory");
-} catch {
-  // not installed — tests will be skipped
-}
-
-const describeIfBabel = babelMemory ? describe : describe.skip;
-
-beforeAll(async () => {
-  if (babelMemory) {
-    await babelMemory.initTokenizer();
-  }
-});
-
-describeIfBabel("babel-memory integration", () => {
-  test("Chinese memory: language detected + fts_text tokenized", () => {
-    const text = "RecallNest 是一个基于 LanceDB 的 AI 记忆系统";
-    const language = babelMemory!.detectLanguage(text);
-    const fts_text = babelMemory!.tokenizeForFts(text, language);
-
-    expect(language).toBe("zh");
-    expect(fts_text).toContain("RecallNest");
-    expect(fts_text.includes(" ")).toBe(true);
-  });
-
-  test("English memory: language detected + fts_text unchanged", () => {
-    const text = "RecallNest is an AI memory system built on LanceDB";
-    const language = babelMemory!.detectLanguage(text);
-    const fts_text = babelMemory!.tokenizeForFts(text, language);
-
-    expect(language).toBe("en");
-    expect(fts_text).toBe(text);
-  });
-
-  test("BM25 query symmetry: tokenized query matches tokenized stored text", () => {
-    const storedText = "机器学习在自然语言处理中的应用";
-    const storedLang = babelMemory!.detectLanguage(storedText);
-    const storedFts = babelMemory!.tokenizeForFts(storedText, storedLang);
-
-    const query = "机器学习";
-    const queryLang = babelMemory!.detectLanguage(query);
-    const queryFts = babelMemory!.tokenizeForFts(query, queryLang);
-
-    const queryTerms = queryFts
-      .split(" ")
-      .filter((t) => t.length > 0);
-    const matchedTerms = queryTerms.filter((term) =>
-      storedFts.includes(term),
-    );
-    expect(matchedTerms.length).toBeGreaterThan(0);
-  });
-
-  test("KG prompt routes correctly for Chinese input", () => {
-    const text = "RecallNest 使用 LanceDB 存储记忆";
-    const lang = babelMemory!.detectLanguage(text);
-    const { system } = babelMemory!.getKgPrompt(lang);
-    expect(system).toContain("知识图谱");
-  });
-
-  test("Session prompt routes correctly for Chinese input", () => {
-    const text = "用户今天讨论了多语言支持方案";
-    const lang = babelMemory!.detectLanguage(text);
-    const { dimensionLabels } = babelMemory!.getSessionPrompt(lang);
-    expect(dimensionLabels.user_intent).toContain("用户意图");
-  });
-
-  test("backward compat: missing language defaults to en", () => {
-    const entry = { text: "some old text", language: undefined };
-    const lang = entry.language || "en";
-    const fts = babelMemory!.tokenizeForFts(entry.text, lang);
-    expect(lang).toBe("en");
-    expect(fts).toBe(entry.text);
-  });
-});
-
-describe("store.update() preserves language + fts_text", () => {
-  const dbPath = join(tmpdir(), `recallnest-update-lang-test-${Date.now()}`);
-  let store: MemoryStore;
-
-  beforeAll(async () => {
-    mkdirSync(dbPath, { recursive: true });
-    store = new MemoryStore({ dbPath, vectorDim: 3 });
-    await (store as any).ensureInitialized();
-  });
-
-  afterAll(() => {
-    try { rmSync(dbPath, { recursive: true, force: true }); } catch {}
-  });
-
-  test("update preserves language and fts_text when not in updates", async () => {
-    const entry = await store.store({
-      text: "这是一条中文记忆",
-      vector: [0.1, 0.2, 0.3],
-      category: "events",
-      scope: "test:lang",
-      importance: 0.7,
-      metadata: "{}",
-      language: "zh",
-      fts_text: "这是 一条 中文 记忆",
-    });
-
-    // Update only text — language and fts_text should be preserved from old row
-    const updated = await store.update(entry.id, {
-      importance: 0.9,
-    });
-
-    expect(updated).not.toBeNull();
-    expect(updated!.language).toBe("zh");
-    expect(updated!.fts_text).toBe("这是 一条 中文 记忆");
-    expect(updated!.importance).toBe(0.9);
-  });
-
-  test("update overwrites language and fts_text when provided", async () => {
-    const entry = await store.store({
-      text: "old english text",
-      vector: [0.4, 0.5, 0.6],
-      category: "events",
-      scope: "test:lang",
-      importance: 0.6,
-      metadata: "{}",
-      language: "en",
-      fts_text: "old english text",
-    });
-
-    const updated = await store.update(entry.id, {
-      text: "新的中文文本",
-      language: "zh",
-      fts_text: "新的 中文 文本",
-    });
-
-    expect(updated).not.toBeNull();
-    expect(updated!.language).toBe("zh");
-    expect(updated!.fts_text).toBe("新的 中文 文本");
-    expect(updated!.text).toBe("新的中文文本");
-  });
-});

package/src/__tests__/batch-dedup.test.ts

@@ -1,65 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import { batchInternalDedup } from "../ingest.js";
-
-describe("batchInternalDedup", () => {
-  it("keeps all entries when vectors are dissimilar", () => {
-    const vectors = [
-      [1, 0, 0],
-      [0, 1, 0],
-      [0, 0, 1],
-    ];
-    const importances = [0.8, 0.7, 0.6];
-    const kept = batchInternalDedup(vectors, importances);
-    expect(kept).toEqual([0, 1, 2]);
-  });
-
-  it("drops lower-importance duplicate when vectors are near-identical", () => {
-    const vectors = [
-      [1, 0, 0],    // importance 0.6
-      [1, 0.01, 0], // importance 0.9 — nearly identical to [0]
-      [0, 1, 0],    // importance 0.7 — dissimilar
-    ];
-    const importances = [0.6, 0.9, 0.7];
-    const kept = batchInternalDedup(vectors, importances);
-    // [1] has higher importance and is near-identical to [0], so [0] gets dropped
-    expect(kept).toContain(1);
-    expect(kept).toContain(2);
-    expect(kept).not.toContain(0);
-  });
-
-  it("returns single-item batch unchanged", () => {
-    expect(batchInternalDedup([[1, 0, 0]], [0.5])).toEqual([0]);
-  });
-
-  it("returns empty for empty input", () => {
-    expect(batchInternalDedup([], [])).toEqual([]);
-  });
-
-  it("preserves original order in output", () => {
-    const vectors = [
-      [0, 1, 0],    // idx 0, imp 0.9
-      [1, 0, 0],    // idx 1, imp 0.5
-      [0, 0, 1],    // idx 2, imp 0.7
-    ];
-    const importances = [0.9, 0.5, 0.7];
-    const kept = batchInternalDedup(vectors, importances);
-    // All dissimilar, all kept, in original order
-    expect(kept).toEqual([0, 1, 2]);
-  });
-
-  it("respects custom threshold", () => {
-    const vectors = [
-      [1, 0, 0],
-      [0.95, 0.31, 0], // cosine ~0.95 with [0]
-    ];
-    const importances = [0.8, 0.7];
-
-    // Default threshold 0.85 → should dedup
-    const strictKept = batchInternalDedup(vectors, importances, 0.85);
-    expect(strictKept).toHaveLength(1);
-
-    // Loose threshold 0.99 → should keep both
-    const looseKept = batchInternalDedup(vectors, importances, 0.99);
-    expect(looseKept).toHaveLength(2);
-  });
-});

package/src/__tests__/belief-history.test.ts

@@ -1,220 +0,0 @@
-import { describe, expect, it } from "bun:test";
-
-import { archiveBeliefVersion } from "../belief-history.js";
-import { persistMemory } from "../capture-engine.js";
-import { isActiveMemory, parseEvolution, traceEvolution } from "../memory-evolution.js";
-
-const TEST_SCOPE = "project:test";
-const CANONICAL_KEY = "user.reply.style";
-
-/**
- * Store double that mirrors the real MemoryStore where belief history depends on it:
- * store() stamps its own id/timestamp, upsert() honours both, and list() strips vectors
- * (which is why archiving has to re-read or re-embed).
- */
-function createDeps(options: { withUpsert?: boolean; withGetById?: boolean } = {}) {
-  const { withUpsert = true, withGetById = true } = options;
-  const storedEntries: any[] = [];
-  const embedCalls: string[] = [];
-  let seq = 1;
-
-  const store: Record<string, unknown> = {
-    async store(entry: any) {
-      const stored = {
-        ...entry,
-        id: entry.id ?? `00000000-0000-0000-0000-${String(seq).padStart(12, "0")}`,
-        timestamp: 1_700_000_000_000 + seq,
-      };
-      seq += 1;
-      // The real store() routes through upsert (mergeInsert on id), so writing a known id
-      // replaces that row rather than adding a second one with the same id.
-      const index = storedEntries.findIndex((item) => item.id === stored.id);
-      if (index >= 0) storedEntries[index] = stored;
-      else storedEntries.push(stored);
-      return stored;
-    },
-    async list(_scopeFilter?: string[], category?: string, limit = 20, offset = 0) {
-      return storedEntries
-        .filter((entry) => !category || entry.category === category)
-        // The real store drops vectors from list results for performance.
-        .map((entry) => ({ ...entry, vector: [] }))
-        .slice(offset, offset + limit);
-    },
-    async update(id: string, updates: any) {
-      const index = storedEntries.findIndex((entry) => entry.id === id);
-      if (index < 0) return null;
-      storedEntries[index] = {
-        ...storedEntries[index],
-        ...updates,
-        timestamp: updates.timestamp ?? storedEntries[index].timestamp,
-      };
-      return storedEntries[index];
-    },
-  };
-
-  if (withUpsert) {
-    store.upsert = async (entry: any) => {
-      const index = storedEntries.findIndex((item) => item.id === entry.id);
-      if (index >= 0) storedEntries[index] = { ...entry };
-      else storedEntries.push({ ...entry });
-      return entry;
-    };
-  }
-  if (withGetById) {
-    store.getById = async (id: string) => storedEntries.find((entry) => entry.id === id) || null;
-  }
-
-  return {
-    storedEntries,
-    embedCalls,
-    deps: {
-      store,
-      embedder: {
-        async embedPassage(text: string) {
-          embedCalls.push(text);
-          return [text.length, 1, 0];
-        },
-      },
-    },
-  };
-}
-
-async function writeBelief(deps: unknown, text: string) {
-  return await persistMemory(deps as any, {
-    text,
-    category: "preferences",
-    scope: TEST_SCOPE,
-    source: "manual",
-    canonicalKey: CANONICAL_KEY,
-  });
-}
-
-describe("belief history", () => {
-  it("keeps every superseded version when a belief changes repeatedly", async () => {
-    const { deps, storedEntries } = createDeps();
-
-    const first = await writeBelief(deps, "User prefers long, thorough replies");
-    await writeBelief(deps, "User prefers concise replies");
-    await writeBelief(deps, "User prefers concise, code-first replies");
-
-    // One live canonical row + two archived versions — nothing was overwritten away.
-    expect(storedEntries).toHaveLength(3);
-
-    const canonical = storedEntries.find((entry) => entry.id === first.id);
-    expect(canonical.text).toBe("User prefers concise, code-first replies");
-    expect(parseEvolution(canonical.metadata).status).toBe("active");
-
-    const archived = storedEntries.filter((entry) => entry.id !== first.id);
-    expect(archived.map((entry) => entry.text).sort()).toEqual([
-      "User prefers concise replies",
-      "User prefers long, thorough replies",
-    ].sort());
-
-    for (const row of archived) {
-      const evo = parseEvolution(row.metadata);
-      expect(evo.status).toBe("superseded");
-      expect(evo.supersededBy).toBe(first.id);
-      // A closed interval is what makes an as-of query possible at all.
-      expect(evo.validUntil).toBeGreaterThan(0);
-      expect(evo.validUntil).toBeGreaterThanOrEqual(evo.validFrom);
-    }
-  });
-
-  it("links versions into a chain that traceEvolution can walk back through", async () => {
-    const { deps, storedEntries } = createDeps();
-
-    const first = await writeBelief(deps, "Belief version one about reply style");
-    await writeBelief(deps, "Belief version two about reply style");
-    await writeBelief(deps, "Belief version three about reply style");
-
-    const byId = new Map(storedEntries.map((entry) => [entry.id, entry]));
-    const trace = await traceEvolution(first.id, async (id) => byId.get(id) ?? null);
-
-    const texts = trace.map((entry) => byId.get(entry.id)?.text);
-    expect(texts).toEqual([
-      "Belief version one about reply style",
-      "Belief version two about reply style",
-      "Belief version three about reply style",
-    ]);
-    expect(trace[trace.length - 1].direction).toBe("self");
-  });
-
-  it("keeps archived versions out of default retrieval", async () => {
-    const { deps, storedEntries } = createDeps();
-
-    await writeBelief(deps, "User is based in Guangzhou");
-    await writeBelief(deps, "User is based in Singapore");
-
-    const visibleByDefault = storedEntries.filter((entry) => isActiveMemory(entry.metadata));
-    expect(visibleByDefault).toHaveLength(1);
-    expect(visibleByDefault[0].text).toBe("User is based in Singapore");
-  });
-
-  it("treats a revert to an earlier wording as a real change, not a duplicate", async () => {
-    const { deps, storedEntries } = createDeps();
-
-    await writeBelief(deps, "User prefers dark mode");
-    await writeBelief(deps, "User prefers light mode");
-    // Changing back: the archived row still carries the same canonicalKey, so without a
-    // status filter on canonical matching this would be misread as a duplicate write.
-    const third = await writeBelief(deps, "User prefers dark mode");
-
-    expect(third.disposition).toBe("updated");
-
-    const canonical = storedEntries.find((entry) => isActiveMemory(entry.metadata));
-    expect(canonical.text).toBe("User prefers dark mode");
-    expect(parseEvolution(canonical.metadata).version).toBe(3);
-  });
-
-  it("archives even when the canonical row has fallen out of the scan window", async () => {
-    // findCanonicalMatches only sees the most recent CANONICAL_SCAN_LIMIT rows. Past that,
-    // the write falls through to store(), which upserts on the deterministic canonical id —
-    // historically an unguarded overwrite. Simulate the miss by blinding list().
-    const { deps, storedEntries } = createDeps();
-    await writeBelief(deps, "User works from the Guangzhou office");
-
-    (deps.store as any).list = async () => [];
-
-    await writeBelief(deps, "User works from the Singapore office");
-
-    expect(storedEntries).toHaveLength(2);
-    const archived = storedEntries.find((entry) => !isActiveMemory(entry.metadata));
-    expect(archived).toBeDefined();
-    expect(archived.text).toBe("User works from the Guangzhou office");
-    expect(parseEvolution(archived.metadata).validUntil).toBeGreaterThan(0);
-  });
-
-  it("re-embeds the archived text when the store cannot return the original vector", async () => {
-    const { deps, embedCalls, storedEntries } = createDeps({ withGetById: false });
-
-    await writeBelief(deps, "Vectorless original");
-    await writeBelief(deps, "Replacement belief");
-
-    const archived = storedEntries.find((entry) => !isActiveMemory(entry.metadata));
-    expect(archived.vector.length).toBeGreaterThan(0);
-    expect(embedCalls).toContain("Vectorless original");
-  });
-
-  it("fails loudly rather than silently dropping history it cannot archive", async () => {
-    const { deps } = createDeps();
-    const failing = {
-      store: {
-        ...(deps.store as Record<string, unknown>),
-        upsert: async () => { throw new Error("disk full"); },
-        store: async () => { throw new Error("disk full"); },
-      },
-      embedder: deps.embedder,
-    };
-
-    await expect(archiveBeliefVersion(failing as any, {
-      id: "00000000-0000-0000-0000-000000000001",
-      text: "Belief that must not vanish",
-      vector: [1, 2, 3],
-      category: "preferences",
-      scope: TEST_SCOPE,
-      importance: 0.8,
-      timestamp: 1_700_000_000_001,
-      metadata: "{}",
-    })).rejects.toThrow("disk full");
-  });
-});

package/src/__tests__/canary-eval.test.ts

@@ -1,114 +0,0 @@
-import { describe, expect, it } from "bun:test";
-
-import { scoreCanaryCase, type CanaryEvalCase } from "../eval.js";
-
-function res(id: string, text = "", scope = "cc", metadata = "") {
-  return { entry: { id, text, scope, metadata } };
-}
-
-describe("scoreCanaryCase", () => {
-  it("A 类 top1 命中给满目标分并 pass", () => {
-    const c: CanaryEvalCase = { name: "a", query: "q", targets: ["m1"], expectTopK: 3 };
-    const r = scoreCanaryCase(c, [res("m1", "hit"), res("m2")]);
-    expect(r.top1Hit).toBe(true);
-    expect(r.top3Hit).toBe(true);
-    expect(r.targetRanks).toEqual([{ id: "m1", rank: 1 }]);
-    expect(r.score).toBeCloseTo(1, 5);
-    expect(r.passed).toBe(true);
-  });
-
-  it("A 类命中但排第 3 给 top3 分", () => {
-    const c: CanaryEvalCase = { name: "a", query: "q", targets: ["m3"], expectTopK: 3 };
-    const r = scoreCanaryCase(c, [res("a"), res("b"), res("m3")]);
-    expect(r.top1Hit).toBe(false);
-    expect(r.top3Hit).toBe(true);
-    expect(r.targetRanks).toEqual([{ id: "m3", rank: 3 }]);
-    expect(r.score).toBeCloseTo(0.7, 5);
-    expect(r.passed).toBe(true);
-  });
-
-  it("A 类 miss 给 0 并 fail", () => {
-    const c: CanaryEvalCase = { name: "a", query: "q", targets: ["zzz"] };
-    const r = scoreCanaryCase(c, [res("a"), res("b")]);
-    expect(r.targetRanks).toEqual([{ id: "zzz", rank: null }]);
-    expect(r.score).toBe(0);
-    expect(r.passed).toBe(false);
-  });
-
-  it("B 类新旧序正确 orderOk=true 并 pass", () => {
-    const c: CanaryEvalCase = { name: "b", query: "q", expectOrder: ["new", "old"] };
-    const r = scoreCanaryCase(c, [res("new"), res("old")]);
-    expect(r.orderOk).toBe(true);
-    expect(r.score).toBeCloseTo(1, 5);
-    expect(r.passed).toBe(true);
-  });
-
-  it("B 类新旧序颠倒 orderOk=false 并 fail", () => {
-    const c: CanaryEvalCase = { name: "b", query: "q", expectOrder: ["new", "old"] };
-    const r = scoreCanaryCase(c, [res("old"), res("new")]);
-    expect(r.orderOk).toBe(false);
-    expect(r.score).toBe(0);
-    expect(r.passed).toBe(false);
-  });
-
-  it("B 类 newer 未召回 orderOk=false", () => {
-    const c: CanaryEvalCase = { name: "b", query: "q", expectOrder: ["new", "old"] };
-    const r = scoreCanaryCase(c, [res("old"), res("other")]);
-    expect(r.orderOk).toBe(false);
-  });
-
-  it("C 类干扰项进 TopK 直接判 fail（即便目标命中）", () => {
-    const c: CanaryEvalCase = { name: "c", query: "q", targets: ["m1"], hardNegatives: ["bad"], limit: 8 };
-    const r = scoreCanaryCase(c, [res("m1"), res("bad")]);
-    expect(r.top1Hit).toBe(true);
-    expect(r.forbiddenIdMatches).toEqual(["bad"]);
-    expect(r.passed).toBe(false);
-  });
-
-  it("C 类干扰项在 limit 之外不算违规", () => {
-    const c: CanaryEvalCase = { name: "c", query: "q", targets: ["m1"], hardNegatives: ["bad"], limit: 1 };
-    const r = scoreCanaryCase(c, [res("m1"), res("bad")]);
-    expect(r.forbiddenIdMatches).toEqual([]);
-    expect(r.passed).toBe(true);
-  });
-
-  it("D 类内容全命中给满分", () => {
-    const c: CanaryEvalCase = { name: "d", query: "q", expectContentAny: ["口语化", "剧评腔"] };
-    const r = scoreCanaryCase(c, [res("x", "她要口语化，带点剧评腔", "cc")]);
-    expect(r.matchedContentAny).toEqual(["口语化", "剧评腔"]);
-    expect(r.score).toBeCloseTo(1, 5);
-    expect(r.passed).toBe(true);
-  });
-
-  it("D 类内容部分命中按比例", () => {
-    const c: CanaryEvalCase = { name: "d", query: "q", expectContentAny: ["口语化", "剧评腔"] };
-    const r = scoreCanaryCase(c, [res("x", "只提了口语化", "cc")]);
-    expect(r.matchedContentAny).toEqual(["口语化"]);
-    expect(r.score).toBeCloseTo(0.5, 5);
-    expect(r.passed).toBe(false);
-  });
-
-  it("文本 forbid 命中判 fail", () => {
-    const c: CanaryEvalCase = { name: "f", query: "q", targets: ["m1"], forbid: ["机密"] };
-    const r = scoreCanaryCase(c, [res("m1", "这是机密内容")]);
-    expect(r.forbiddenMatches).toEqual(["机密"]);
-    expect(r.passed).toBe(false);
-  });
-
-  it("空召回 score 0", () => {
-    const c: CanaryEvalCase = { name: "e", query: "q", targets: ["m1"] };
-    const r = scoreCanaryCase(c, []);
-    expect(r.hitCount).toBe(0);
-    expect(r.score).toBe(0);
-    expect(r.passed).toBe(false);
-  });
-
-  it("A+B 复合：目标命中且新旧序正确，综合 pass", () => {
-    const c: CanaryEvalCase = { name: "ab", query: "q", targets: ["new"], expectOrder: ["new", "old"], expectTopK: 3 };
-    const r = scoreCanaryCase(c, [res("new"), res("old")]);
-    expect(r.top1Hit).toBe(true);
-    expect(r.orderOk).toBe(true);
-    expect(r.score).toBeCloseTo(1, 5);
-    expect(r.passed).toBe(true);
-  });
-});