recallnest 2.3.0 → 2.5.4

package/src/__tests__/dedup-actions.test.ts

@@ -0,0 +1,58 @@
+import { describe, it, expect } from "bun:test";
+import type { DedupAction, DedupCheckResult } from "../ingest.js";
+
+describe("DedupAction types", () => {
+  it("DedupCheckResult supports secondaryDeletes", () => {
+    const result: DedupCheckResult = {
+      action: "store",
+      reason: "unique",
+      secondaryDeletes: [
+        { id: "abc12345", action: "delete", reason: "outdated by candidate" },
+      ],
+    };
+    expect(result.secondaryDeletes).toHaveLength(1);
+    expect(result.secondaryDeletes![0].action).toBe("delete");
+    expect(result.secondaryDeletes![0].id).toBe("abc12345");
+  });
+
+  it("DedupCheckResult without secondaryDeletes is backward-compatible", () => {
+    const result: DedupCheckResult = {
+      action: "skip",
+      reason: "hard",
+      existingText: "existing memory text",
+    };
+    expect(result.secondaryDeletes).toBeUndefined();
+  });
+
+  it("DedupAction only supports delete", () => {
+    const action: DedupAction = {
+      id: "test-id",
+      action: "delete",
+      reason: "superseded by newer info",
+    };
+    expect(action.action).toBe("delete");
+  });
+
+  it("multiple secondary deletes are valid", () => {
+    const result: DedupCheckResult = {
+      action: "store",
+      reason: "llm-merge",
+      secondaryDeletes: [
+        { id: "id1", action: "delete", reason: "outdated" },
+        { id: "id2", action: "delete", reason: "duplicate of id1" },
+      ],
+    };
+    expect(result.secondaryDeletes).toHaveLength(2);
+  });
+
+  it("empty secondaryDeletes array treated as no actions", () => {
+    const result: DedupCheckResult = {
+      action: "store",
+      reason: "unique",
+      secondaryDeletes: [],
+    };
+    expect(result.secondaryDeletes).toHaveLength(0);
+    // In practice, we check .length before executing
+    expect((result.secondaryDeletes?.length ?? 0) > 0).toBe(false);
+  });
+});

package/src/__tests__/dedup-guard.test.ts

@@ -0,0 +1,39 @@
+import { describe, it, expect } from "bun:test";
+import { isTemporalOrActionContent } from "../ingest.js";
+
+describe("Dedup guard: isTemporalOrActionContent", () => {
+  describe("date-stamped events (must NOT be deduped)", () => {
+    it("ISO date", () => expect(isTemporalOrActionContent("2026-03-25 部署了新版本")).toBe(true));
+    it("今天", () => expect(isTemporalOrActionContent("今天跑了 benchmark")).toBe(true));
+    it("昨天", () => expect(isTemporalOrActionContent("昨天讨论了架构")).toBe(true));
+    it("上次", () => expect(isTemporalOrActionContent("上次的方案不行")).toBe(true));
+    it("刚才", () => expect(isTemporalOrActionContent("刚才改了配置")).toBe(true));
+  });
+
+  describe("file operations (must NOT be deduped)", () => {
+    it("file path", () => expect(isTemporalOrActionContent("修改了 /Users/xxx/src/foo.ts")).toBe(true));
+    it("home path", () => expect(isTemporalOrActionContent("保存到 ~/recallnest/data/")).toBe(true));
+    it("md file", () => expect(isTemporalOrActionContent("更新了 README.md")).toBe(true));
+    it("mv command", () => expect(isTemporalOrActionContent("mv old.json new.json")).toBe(true));
+    it("归档", () => expect(isTemporalOrActionContent("把旧文件归档了")).toBe(true));
+  });
+
+  describe("memory instructions (must NOT be deduped)", () => {
+    it("记住", () => expect(isTemporalOrActionContent("记住不要用 rm")).toBe(true));
+    it("remember", () => expect(isTemporalOrActionContent("remember to check tests")).toBe(true));
+    it("别忘了", () => expect(isTemporalOrActionContent("别忘了跑测试")).toBe(true));
+    it("务必", () => expect(isTemporalOrActionContent("务必先备份")).toBe(true));
+  });
+
+  describe("feedback/corrections (must NOT be deduped)", () => {
+    it("不要再", () => expect(isTemporalOrActionContent("不要再用这个方法")).toBe(true));
+    it("stop doing", () => expect(isTemporalOrActionContent("stop doing that")).toBe(true));
+    it("don't", () => expect(isTemporalOrActionContent("don't mock the database")).toBe(true));
+  });
+
+  describe("generic content (CAN be deduped normally)", () => {
+    it("plain description", () => expect(isTemporalOrActionContent("RecallNest 是一个记忆服务")).toBe(false));
+    it("technical fact", () => expect(isTemporalOrActionContent("LanceDB 支持向量搜索")).toBe(false));
+    it("preference without instruction marker", () => expect(isTemporalOrActionContent("我喜欢用 dark mode")).toBe(false));
+  });
+});

package/src/__tests__/deterministic-id.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "bun:test";
+
+import { deterministicId } from "../store.js";
+
+describe("deterministicId", () => {
+  it("same scope+text always produce the same ID", () => {
+    const id1 = deterministicId("project:test", "I prefer TypeScript");
+    const id2 = deterministicId("project:test", "I prefer TypeScript");
+    expect(id1).toBe(id2);
+  });
+
+  it("different text produces different ID", () => {
+    const id1 = deterministicId("project:test", "I prefer TypeScript");
+    const id2 = deterministicId("project:test", "I prefer JavaScript");
+    expect(id1).not.toBe(id2);
+  });
+
+  it("different scope produces different ID", () => {
+    const id1 = deterministicId("project:alpha", "same text");
+    const id2 = deterministicId("project:beta", "same text");
+    expect(id1).not.toBe(id2);
+  });
+
+  it("returns UUID-shaped string (8-4-4-4-12)", () => {
+    const id = deterministicId("project:test", "hello world");
+    expect(id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/);
+  });
+
+  it("handles empty scope gracefully", () => {
+    const id = deterministicId("", "some text");
+    expect(id).toMatch(/^[0-9a-f]{8}-/);
+  });
+
+  it("handles CJK text", () => {
+    const id1 = deterministicId("project:test", "我喜欢用Bun");
+    const id2 = deterministicId("project:test", "我喜欢用Bun");
+    expect(id1).toBe(id2);
+  });
+});

package/src/__tests__/distill-lock.test.ts

@@ -0,0 +1,305 @@
+import { describe, expect, it, beforeEach, afterEach } from "bun:test";
+import { existsSync, writeFileSync, readFileSync, mkdirSync, statSync, utimesSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { tmpdir } from "node:os";
+
+import {
+  acquireLock,
+  releaseLock,
+  rollbackLock,
+  shouldDistill,
+  getLastDistillTime,
+  withLock,
+  withWriteLock,
+  lockPathForKey,
+  stampLock,
+  type DistillLockConfig,
+} from "../distill-lock.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+let tempDir: string;
+let lockPath: string;
+
+function cfg(overrides?: Partial<DistillLockConfig>): Partial<DistillLockConfig> {
+  return { lockPath, ...overrides };
+}
+
+beforeEach(() => {
+  tempDir = join(tmpdir(), `distill-lock-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+  mkdirSync(tempDir, { recursive: true });
+  lockPath = join(tempDir, "distill.lock");
+});
+
+afterEach(() => {
+  // Clean up lock file if it exists
+  if (existsSync(lockPath)) {
+    try {
+      const { unlinkSync } = require("node:fs");
+      unlinkSync(lockPath);
+    } catch { /* ignore */ }
+  }
+});
+
+// ---------------------------------------------------------------------------
+// acquireLock
+// ---------------------------------------------------------------------------
+
+describe("acquireLock", () => {
+  it("creates lock file and returns true when no lock exists", () => {
+    const result = acquireLock(cfg());
+    expect(result).toBe(true);
+    expect(existsSync(lockPath)).toBe(true);
+  });
+
+  it("writes current PID into lock file", () => {
+    acquireLock(cfg());
+    const { readFileSync } = require("node:fs");
+    const content = readFileSync(lockPath, "utf-8").trim();
+    // Lock body is now `<pid>:<ownership-token>` (P2), so assert the PID prefix.
+    expect(content.startsWith(`${process.pid}:`)).toBe(true);
+  });
+
+  it("returns false when lock held by alive PID and not expired", () => {
+    // Write current process PID (which is alive)
+    writeFileSync(lockPath, String(process.pid), "utf-8");
+
+    const result = acquireLock(cfg());
+    expect(result).toBe(false);
+  });
+
+  it("overwrites lock when PID is dead", () => {
+    // PID 999999 is almost certainly not running
+    writeFileSync(lockPath, "999999", "utf-8");
+
+    const result = acquireLock(cfg());
+    expect(result).toBe(true);
+
+    const { readFileSync } = require("node:fs");
+    const content = readFileSync(lockPath, "utf-8").trim();
+    // Reclaimed lock records our `<pid>:<token>` now (P2 ownership token).
+    expect(content.startsWith(`${process.pid}:`)).toBe(true);
+  });
+
+  it("overwrites lock when mtime is expired even if PID alive", () => {
+    writeFileSync(lockPath, String(process.pid), "utf-8");
+    // Set mtime to 2 hours ago
+    const twoHoursAgo = new Date(Date.now() - 7_200_000);
+    utimesSync(lockPath, twoHoursAgo, twoHoursAgo);
+
+    const result = acquireLock(cfg({ expireMs: 3_600_000 }));
+    expect(result).toBe(true);
+  });
+
+  it("creates parent directories if missing", () => {
+    const deepPath = join(tempDir, "nested", "dir", "distill.lock");
+    const result = acquireLock(cfg({ lockPath: deepPath }));
+    expect(result).toBe(true);
+    expect(existsSync(deepPath)).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// releaseLock
+// ---------------------------------------------------------------------------
+
+describe("releaseLock", () => {
+  it("deletes lock file", () => {
+    writeFileSync(lockPath, String(process.pid), "utf-8");
+    expect(existsSync(lockPath)).toBe(true);
+
+    releaseLock(cfg());
+    expect(existsSync(lockPath)).toBe(false);
+  });
+
+  it("is a no-op when lock file does not exist", () => {
+    expect(() => releaseLock(cfg())).not.toThrow();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// rollbackLock
+// ---------------------------------------------------------------------------
+
+describe("rollbackLock", () => {
+  it("sets lock mtime to the given previous value", () => {
+    writeFileSync(lockPath, String(process.pid), "utf-8");
+    const pastDate = new Date(Date.now() - 600_000); // 10 min ago
+
+    rollbackLock(pastDate, cfg());
+
+    const st = statSync(lockPath);
+    // Allow 1 second tolerance for filesystem granularity
+    expect(Math.abs(st.mtimeMs - pastDate.getTime())).toBeLessThan(1000);
+  });
+
+  it("is a no-op when lock file does not exist", () => {
+    expect(() => rollbackLock(new Date(), cfg())).not.toThrow();
+  });
+});
+
+// ---------------------------------------------------------------------------
+// shouldDistill
+// ---------------------------------------------------------------------------
+
+describe("shouldDistill", () => {
+  it("returns true when checkpoint count >= minCheckpoints (default 3)", () => {
+    expect(shouldDistill(3, cfg())).toBe(true);
+    expect(shouldDistill(5, cfg())).toBe(true);
+  });
+
+  it("returns false when checkpoint count < minCheckpoints", () => {
+    expect(shouldDistill(0, cfg())).toBe(false);
+    expect(shouldDistill(1, cfg())).toBe(false);
+    expect(shouldDistill(2, cfg())).toBe(false);
+  });
+
+  it("respects custom minCheckpoints", () => {
+    expect(shouldDistill(4, cfg({ minCheckpoints: 5 }))).toBe(false);
+    expect(shouldDistill(5, cfg({ minCheckpoints: 5 }))).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// getLastDistillTime
+// ---------------------------------------------------------------------------
+
+describe("getLastDistillTime", () => {
+  it("returns lock file mtime when lock exists", () => {
+    writeFileSync(lockPath, String(process.pid), "utf-8");
+    const st = statSync(lockPath);
+
+    const result = getLastDistillTime(cfg());
+    expect(Math.abs(result - st.mtimeMs)).toBeLessThan(100);
+  });
+
+  it("returns 0 when lock file does not exist", () => {
+    const result = getLastDistillTime(cfg());
+    expect(result).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// P0-1: acquireLock atomicity + named withLock / withWriteLock
+// ---------------------------------------------------------------------------
+
+describe("acquireLock atomicity (O_EXCL)", () => {
+  it("is exclusive: second acquire on a live-held lock fails", () => {
+    expect(acquireLock(cfg({ expireMs: 60_000 }))).toBe(true);
+    expect(acquireLock(cfg({ expireMs: 60_000 }))).toBe(false);
+  });
+});
+
+describe("withLock", () => {
+  it("runs fn and releases the lock afterwards", async () => {
+    const out = await withLock("wl-run", async () => 42, { lockDir: tempDir });
+    expect(out.ran).toBe(true);
+    if (out.ran) expect(out.result).toBe(42);
+    expect(existsSync(lockPathForKey("wl-run", tempDir))).toBe(false);
+  });
+
+  it("skips (ran:false) when a live process already holds the lock", async () => {
+    const p = lockPathForKey("wl-skip", tempDir);
+    mkdirSync(dirname(p), { recursive: true });
+    writeFileSync(p, String(process.pid), "utf-8"); // current pid = alive
+    let ran = false;
+    const out = await withLock(
+      "wl-skip",
+      async () => { ran = true; return 1; },
+      { lockDir: tempDir, expireMs: 60_000 },
+    );
+    expect(out.ran).toBe(false);
+    if (!out.ran) expect(out.reason).toBe("locked_by_another_process");
+    expect(ran).toBe(false);
+  });
+
+  it("does not block across different lock keys", async () => {
+    const held = lockPathForKey("wl-keyA", tempDir);
+    mkdirSync(dirname(held), { recursive: true });
+    writeFileSync(held, String(process.pid), "utf-8");
+    const out = await withLock("wl-keyB", async () => "ran", { lockDir: tempDir, expireMs: 60_000 });
+    expect(out.ran).toBe(true);
+  });
+
+  it("releases the lock even when fn throws", async () => {
+    await expect(
+      withLock("wl-throw", async () => { throw new Error("boom"); }, { lockDir: tempDir }),
+    ).rejects.toThrow("boom");
+    expect(existsSync(lockPathForKey("wl-throw", tempDir))).toBe(false);
+  });
+});
+
+describe("withWriteLock", () => {
+  it("throws when a live holder never releases (timeout)", async () => {
+    const p = lockPathForKey("ww-timeout", tempDir);
+    mkdirSync(dirname(p), { recursive: true });
+    writeFileSync(p, String(process.pid), "utf-8"); // alive, never released
+    await expect(
+      withWriteLock("ww-timeout", async () => 1, {
+        lockDir: tempDir,
+        expireMs: 60_000,
+        waitTimeoutMs: 150,
+        pollMs: 20,
+      }),
+    ).rejects.toThrow(/timed out/);
+  });
+
+  it("reclaims an expired lock and runs", async () => {
+    const p = lockPathForKey("ww-expired", tempDir);
+    mkdirSync(dirname(p), { recursive: true });
+    writeFileSync(p, String(process.pid), "utf-8");
+    const twoHoursAgo = new Date(Date.now() - 7_200_000);
+    utimesSync(p, twoHoursAgo, twoHoursAgo);
+    const result = await withWriteLock("ww-expired", async () => "done", {
+      lockDir: tempDir,
+      expireMs: 3_600_000,
+    });
+    expect(result).toBe("done");
+  });
+
+  it("returns fn result directly on a free lock", async () => {
+    const result = await withWriteLock("ww-free", async () => 7, { lockDir: tempDir });
+    expect(result).toBe(7);
+  });
+});
+
+describe("stampLock", () => {
+  it("touches an existing lock file's mtime to now", () => {
+    const p = lockPathForKey("stamp", tempDir);
+    mkdirSync(dirname(p), { recursive: true });
+    writeFileSync(p, String(process.pid), "utf-8");
+    utimesSync(p, new Date(Date.now() - 100_000), new Date(Date.now() - 100_000));
+    stampLock({ lockPath: p });
+    const st = statSync(p);
+    expect(Math.abs(st.mtimeMs - Date.now())).toBeLessThan(2000);
+  });
+
+  it("creates the marker file when absent (mtime ~ now)", () => {
+    const p = lockPathForKey("stamp-absent", tempDir);
+    expect(existsSync(p)).toBe(false);
+    stampLock({ lockPath: p });
+    expect(existsSync(p)).toBe(true);
+    expect(Math.abs(statSync(p).mtimeMs - Date.now())).toBeLessThan(2000);
+  });
+});
+
+describe("lock ownership token (release safety)", () => {
+  it("does not delete a lock reclaimed by another owner after expiry", () => {
+    const p = lockPathForKey("own-reclaimed", tempDir);
+    expect(acquireLock({ lockPath: p, expireMs: 60_000 })).toBe(true);
+    // Simulate: our lock expired mid-run, another process reclaimed it (its own token).
+    writeFileSync(p, "99999:other-owner-token", "utf-8");
+    releaseLock({ lockPath: p });
+    expect(existsSync(p)).toBe(true); // new owner's lock must survive our release
+    expect(readFileSync(p, "utf-8")).toContain("other-owner-token");
+  });
+
+  it("removes a lock we still own", () => {
+    const p = lockPathForKey("own-normal", tempDir);
+    expect(acquireLock({ lockPath: p, expireMs: 60_000 })).toBe(true);
+    releaseLock({ lockPath: p });
+    expect(existsSync(p)).toBe(false);
+  });
+});

package/src/__tests__/distill-session-argshape.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect } from "bun:test";
+import { distillSession, type ConversationMessage } from "../session-distiller.js";
+
+/**
+ * Regression test for mcp-server.ts:1894 P0 bug (fixed 2026-05-13).
+ *
+ * The bug: mcp-server.ts passed { messages, scope, preserveRecent, keepRecentTools, persist }
+ * as first arg, but distillSession signature is (messages: ConversationMessage[], deps, opts).
+ *
+ * Result before fix: microcompact internally saw undefined.length, silently skipped all
+ * loops, returned empty result. distill_session MCP appeared to work but produced 0 output.
+ *
+ * Fix: split call into 3 args: messages (array), deps, opts.
+ *
+ * These tests verify the fix works (microcompact actually clears tools, tokens freed)
+ * and document the bug shape (calling with object as first arg returns silent noop, not error).
+ */
+describe("distillSession parameter shape regression", () => {
+  const buildMessages = (): ConversationMessage[] => [
+    { role: "user", content: "start task" },
+    { role: "assistant", content: "calling tool" },
+    { role: "tool", content: "tool result 1 ".repeat(50), tool_name: "Read" },
+    { role: "assistant", content: "more work" },
+    { role: "tool", content: "tool result 2 ".repeat(50), tool_name: "Read" },
+    { role: "assistant", content: "more work" },
+    { role: "tool", content: "tool result 3 ".repeat(50), tool_name: "Read" },
+    { role: "user", content: "preserved recent" },
+  ];
+
+  // Mock llm.chatLong returns empty string → summarize falls through to {text:"",dimensions:{}}
+  // → persist block skipped (Object.keys(dimensions).length === 0)
+  const mockDeps = {
+    llm: { chatLong: async () => "" } as any,
+    persistMemory: async () => ({ disposition: "stored" as const, id: "test-id" }),
+  };
+
+  it("microcompact actually processes messages when called with correct (array, deps, opts) shape", async () => {
+    const messages = buildMessages();
+    // preserveRecent=1 → cutoff = 8 - 1 = 7 → scan first 7 messages, find 3 Read tools (idx 2/4/6)
+    // keepRecentTools=1 → clear 3-1 = 2 tools (idx 2,4)
+    const result = await distillSession(messages, mockDeps, {
+      scope: "test:regression-2026-05-13",
+      preserveRecent: 1,
+      keepRecentTools: 1,
+      persist: false,
+    });
+    expect(result).toBeDefined();
+    expect(result.compacted_messages).toBeDefined();
+    expect(result.compacted_messages.length).toBe(messages.length);
+    expect(result.microcompact.tools_cleared).toBe(2);
+    expect(result.microcompact.tokens_freed).toBeGreaterThan(0);
+    // Verify the cleared tool messages have been replaced with placeholder
+    const clearedMsg = result.compacted_messages[2];
+    expect(clearedMsg.role).toBe("tool");
+    expect(clearedMsg.content).toContain("[Cleared:");
+  });
+
+  it("documents bug shape: object as first arg = silent noop (no error, empty result)", async () => {
+    // BUG REPRODUCTION (pre-fix mcp-server.ts:1894 behavior)
+    // distillSession expects messages: ConversationMessage[]
+    // but received an object — microcompact internally does messages.length / .forEach
+    // → on object, .length === undefined → loops skip → empty result returned silently
+    const wrongShape = {
+      messages: buildMessages(),
+      scope: "test:bug-shape",
+      preserveRecent: 1,
+    };
+    const result = await distillSession(
+      // @ts-expect-error: intentional wrong shape to verify silent noop characteristic
+      wrongShape,
+      mockDeps,
+      {},
+    );
+    // The bug signature: silent noop — no exception, but no work either
+    expect(result.compacted_messages).toEqual([]);
+    expect(result.microcompact.tools_cleared).toBe(0);
+    expect(result.microcompact.tokens_freed).toBe(0);
+  });
+});