recallnest 2.3.0 → 2.6.0

package/src/__tests__/hotness-blend.test.ts

@@ -0,0 +1,283 @@
+import { describe, it, expect } from "bun:test";
+import { computeHotnessScore, parseAccessMetadata } from "../access-tracker.js";
+import { TraceCollector } from "../retrieval-trace.js";
+import {
+  createRetriever,
+  DEFAULT_CATEGORY_MIN_SCORES,
+} from "../retriever.js";
+
+function applyHotnessBlend(
+  score: number,
+  accessCount: number,
+  lastAccessedAt: number,
+): number {
+  const retriever = createRetriever({} as any, {} as any, { hotnessWeight: 0.15 });
+  retriever.setAccessTracker({} as any);
+  const results = [{
+    entry: {
+      id: "hotness-test",
+      text: "hotness test",
+      vector: [],
+      category: "events",
+      scope: "test:hotness",
+      importance: 0.5,
+      timestamp: Date.now(),
+      metadata: JSON.stringify({ accessCount, lastAccessedAt }),
+    },
+    score,
+    sources: {},
+  }];
+  return (retriever as any).applyHotnessBlend(results)[0].score;
+}
+
+describe("computeHotnessScore", () => {
+  it("returns 0 for zero accesses", () => {
+    expect(computeHotnessScore(0, Date.now())).toBe(0);
+  });
+
+  it("returns positive score for accessed memories", () => {
+    const score = computeHotnessScore(5, Date.now());
+    expect(score).toBeGreaterThan(0);
+    expect(score).toBeLessThanOrEqual(1);
+  });
+
+  it("higher access count yields higher score", () => {
+    const now = Date.now();
+    const low = computeHotnessScore(1, now);
+    const mid = computeHotnessScore(5, now);
+    const high = computeHotnessScore(50, now);
+    expect(low).toBeLessThan(mid);
+    expect(mid).toBeLessThan(high);
+  });
+
+  it("recent access yields higher score than old access", () => {
+    const now = Date.now();
+    const recent = computeHotnessScore(5, now);
+    const weekAgo = computeHotnessScore(5, now - 7 * 86_400_000);
+    const monthAgo = computeHotnessScore(5, now - 30 * 86_400_000);
+    expect(recent).toBeGreaterThan(weekAgo);
+    expect(weekAgo).toBeGreaterThan(monthAgo);
+  });
+
+  it("decays to near-zero for very old accesses", () => {
+    const score = computeHotnessScore(5, Date.now() - 365 * 86_400_000);
+    expect(score).toBeLessThan(0.01);
+  });
+
+  it("caps at 1.0 even with extreme access counts", () => {
+    const score = computeHotnessScore(10_000, Date.now());
+    expect(score).toBeLessThanOrEqual(1.0);
+  });
+
+  it("respects custom decay rate", () => {
+    const now = Date.now();
+    const fast = computeHotnessScore(5, now - 7 * 86_400_000, 0.5);
+    const slow = computeHotnessScore(5, now - 7 * 86_400_000, 0.01);
+    expect(slow).toBeGreaterThan(fast);
+  });
+});
+
+describe("parseAccessMetadata", () => {
+  it("parses valid metadata", () => {
+    const meta = JSON.stringify({ accessCount: 5, lastAccessedAt: 1234567890 });
+    const result = parseAccessMetadata(meta);
+    expect(result.accessCount).toBe(5);
+    expect(result.lastAccessedAt).toBe(1234567890);
+  });
+
+  it("returns defaults for missing fields", () => {
+    const result = parseAccessMetadata("{}");
+    expect(result.accessCount).toBe(0);
+    expect(result.lastAccessedAt).toBe(0);
+  });
+
+  it("handles undefined input", () => {
+    const result = parseAccessMetadata(undefined);
+    expect(result.accessCount).toBe(0);
+    expect(result.lastAccessedAt).toBe(0);
+  });
+
+  it("handles malformed JSON", () => {
+    const result = parseAccessMetadata("not-json");
+    expect(result.accessCount).toBe(0);
+    expect(result.lastAccessedAt).toBe(0);
+  });
+});
+
+describe("applyHotnessBlend", () => {
+  it("does not penalize a zero-access row", () => {
+    expect(applyHotnessBlend(0.6, 0, 0)).toBe(0.6);
+  });
+
+  it("preserves the existing blend formula for an accessed row", () => {
+    const futureAccess = Date.now() + 60_000;
+    const hotness = computeHotnessScore(5, futureAccess);
+    expect(applyHotnessBlend(0.6, 5, futureAccess)).toBeCloseTo(
+      0.6 * 0.85 + hotness * 0.15,
+      12,
+    );
+  });
+
+  it("still applies the existing penalty when an accessed row's hotness decays to zero", () => {
+    expect(computeHotnessScore(1, 1)).toBe(0);
+    expect(applyHotnessBlend(0.6, 1, 1)).toBeCloseTo(0.6 * 0.85, 12);
+  });
+});
+
+describe("cold-start hotness pipeline regression", () => {
+  it("recalls a78ad478 without dropping control dd05b422", async () => {
+    const query = "owner 模型 任务锁 并发 谁在干活 责任层 资源层";
+    const now = Date.now();
+    const text = `${query} `.repeat(80).slice(0, 1900);
+
+    expect(text).toHaveLength(1900);
+
+    function metadata(
+      source: "agent" | "manual",
+      confidence: number,
+      accessCount: number,
+    ): string {
+      const lastAccessedAt = accessCount > 0 ? now : 0;
+
+      return JSON.stringify({
+        source,
+        confidence: {
+          score: confidence,
+          reliability: source === "manual" ? "direct" : "inferred",
+        },
+        accessCount,
+        lastAccessedAt,
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "patterns",
+        },
+        evolution: {
+          status: "active",
+          version: 1,
+          accessCount,
+          lastAccessedAt: accessCount > 0 ? lastAccessedAt : null,
+          validFrom: now,
+          validUntil: null,
+        },
+      });
+    }
+
+    const candidates = [
+      {
+        entry: {
+          id: "a78ad478",
+          text,
+          vector: [1, 0, 0],
+          category: "patterns",
+          scope: "project:hippo-wiki",
+          importance: 0.8,
+          timestamp: now,
+          metadata: metadata("agent", 0.7, 0),
+        },
+        score: 0.8628,
+      },
+      {
+        entry: {
+          id: "dd05b422",
+          text,
+          vector: [0.99, 0.01, 0],
+          category: "patterns",
+          scope: "project:hippo-wiki",
+          importance: 0.8,
+          timestamp: now,
+          metadata: metadata("manual", 0.9, 4),
+        },
+        score: 0.8579,
+      },
+    ];
+
+    let vectorSearchCalls = 0;
+
+    const store = {
+      hasFtsSupport: false,
+      async vectorSearch(
+        _queryVector: number[],
+        limit: number,
+        minScore: number,
+        scopeFilter?: string[],
+      ) {
+        vectorSearchCalls += 1;
+        return candidates
+          .filter(candidate => candidate.score >= minScore)
+          .filter(candidate =>
+            !scopeFilter || scopeFilter.includes(candidate.entry.scope),
+          )
+          .slice(0, limit);
+      },
+    };
+
+    const embedder = {
+      async embedQuery() {
+        return [1, 0, 0];
+      },
+      async embedPassage() {
+        return [1, 0, 0];
+      },
+    };
+
+    const retriever = createRetriever(
+      store as any,
+      embedder as any,
+      {
+        mode: "vector",
+        rerank: "none",
+        candidatePoolSize: 5,
+        minScore: 0.3,
+        hardMinScore: 0.38,
+        recencyHalfLifeDays: 14,
+        recencyWeight: 0.15,
+        lengthNormAnchor: 700,
+        timeDecayHalfLifeDays: 60,
+        hotnessWeight: 0.15,
+        filterNoise: false,
+        enableRIF: false,
+        sourceDiversity: 0,
+        multiHop: false,
+      },
+    );
+
+    retriever.setAccessTracker({
+      computeEffectiveHalfLife(baseHalfLife: number) {
+        return baseHalfLife;
+      },
+      recordAccess() {
+        throw new Error("pipeline regression must not record access");
+      },
+    } as any);
+
+    const trace = new TraceCollector();
+    const results = await retriever.retrieve({
+      query,
+      limit: 5,
+      scopeFilter: ["project:hippo-wiki"],
+      category: "patterns",
+      source: "auto-recall",
+      trace,
+    });
+
+    const ids = results.map(result => result.entry.id);
+    const cold = results.find(result => result.entry.id === "a78ad478");
+    const hardMin = trace
+      .finalize(query, "vector")
+      .stages.find(stage => stage.name === "hard_min_score");
+
+    expect(ids).toContain("a78ad478");
+    expect(ids).toContain("dd05b422");
+    expect(cold?.score).toBeGreaterThanOrEqual(
+      DEFAULT_CATEGORY_MIN_SCORES.patterns,
+    );
+    expect(hardMin).toMatchObject({
+      inputCount: 2,
+      outputCount: 2,
+      droppedCount: 0,
+    });
+    expect(vectorSearchCalls).toBe(1);
+  });
+});

package/src/__tests__/ingest-chunking.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, it } from "bun:test";
+
+import { groupTurnsIntoChunks } from "../ingest.js";
+import { isNoise } from "../noise-filter.js";
+
+// P1 defence-in-depth: groupTurnsIntoChunks pre-filters whole turns via
+// filterNoiseTurns BEFORE chunking, but long turns then get split — and the
+// split fragments were never re-checked. A long turn whose tail is orphaned
+// context-JSON therefore leaks severed JSON field-lines as standalone chunks.
+// This guards that no chunk survives as noise, without over-deleting real talk.
+
+function userTurn(text: string) {
+  return { role: "user" as const, text, timestamp: "2026-05-21T00:00:00Z", sessionId: "s1" };
+}
+function assistantTurn(text: string) {
+  return { role: "assistant" as const, text, timestamp: "2026-05-21T00:00:01Z", sessionId: "s1" };
+}
+
+describe("groupTurnsIntoChunks — post-split noise filtering (P1)", () => {
+  it("drops severed context-JSON fragments produced by splitting a long turn", () => {
+    // Real-content opener (passes turn-level filterNoiseTurns), then a long tail
+    // of orphaned JSON field-lines that chunking peels into standalone chunks.
+    const opener = "我们今天把记忆系统这周的进展过一遍，重点是噪声治理和召回质量的提升。".repeat(8);
+    const jsonTail = Array.from(
+      { length: 60 },
+      (_, i) => `"time": "2026-05-21T15:${String(i).padStart(2, "0")}:03.796Z",`,
+    ).join("\n");
+    const chunks = groupTurnsIntoChunks([userTurn(`${opener}\n${jsonTail}`)]);
+
+    expect(chunks.length).toBeGreaterThan(0); // real content survives
+    // No chunk may survive as a recognizable noise fragment.
+    expect(chunks.every((c) => !isNoise(c.text))).toBe(true);
+  });
+
+  it("keeps a normal long Q&A conversation intact (no over-deletion)", () => {
+    const longQuestion = "你帮我想想这个方案怎么落地，我担心几个点没考虑周全，得一起捋一捋。".repeat(20);
+    const longAnswer = "可以分三步走，先做最小验证看效果，再逐步扩展覆盖面，最后补回归测试兜底。".repeat(20);
+    const chunks = groupTurnsIntoChunks([userTurn(longQuestion), assistantTurn(longAnswer)]);
+
+    expect(chunks.length).toBeGreaterThan(0);
+    expect(chunks.every((c) => !isNoise(c.text))).toBe(true); // real talk not deleted
+    expect(chunks.some((c) => c.text.includes("最小验证"))).toBe(true); // content retained
+  });
+});

package/src/__tests__/ingest.test.ts

@@ -0,0 +1,360 @@
+import { describe, expect, it } from "bun:test";
+
+import { dedupCheck } from "../ingest.js";
+
+function buildSearchResult(text: string, score: number) {
+  return {
+    score,
+    entry: {
+      id: `memory-${score}`,
+      text,
+      vector: [score],
+      category: "events",
+      scope: "cc:test-session",
+      importance: 0.6,
+      timestamp: 1_700_000_000_000,
+      metadata: "{}",
+    },
+  };
+}
+
+describe("dedupCheck", () => {
+  it("stores a new same-brand item preference instead of treating it as a duplicate topic", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("喜欢吃麦当劳的麦旋风", 0.92),
+          buildSearchResult("喜欢吃麦当劳的板烧鸡腿堡", 0.89),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "same topic" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "我喜欢吃麦当劳的麦辣鸡翅",
+      llm as any,
+    );
+
+    expect(result.action).toBe("store");
+    expect(result.reason).toBe("unique");
+    expect(result.existingText).toBe("喜欢吃麦当劳的板烧鸡腿堡");
+    expect(llmCalls).toBe(0);
+  });
+
+  it("stores a new atomic preference even when the closest match is an aggregate summary", async () => {
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("喜欢吃麦当劳的麦旋风、板烧鸡腿堡和藤椒鸡派", 0.94),
+        ];
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "我喜欢吃麦当劳的麦辣鸡翅",
+    );
+
+    expect(result.action).toBe("store");
+    expect(result.reason).toBe("unique");
+    expect(result.existingText).toContain("麦旋风");
+  });
+
+  it("still skips when an exact atomic preference already exists among the candidates", async () => {
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("喜欢吃麦当劳的麦旋风、板烧鸡腿堡和藤椒鸡派", 0.94),
+          buildSearchResult("我喜欢吃麦当劳的麦辣鸡翅", 0.82),
+        ];
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "我喜欢吃麦当劳的麦辣鸡翅",
+    );
+
+    expect(result.action).toBe("skip");
+    expect(result.reason).toBe("exact");
+    expect(result.existingText).toBe("我喜欢吃麦当劳的麦辣鸡翅");
+  });
+
+  it("does not force-create when the same brand-item preference is only rephrased", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("我喜欢吃麦当劳的麦辣鸡翅", 0.75),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "same item, rephrased" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "用户喜欢麦当劳的麦辣鸡翅",
+      llm as any,
+    );
+
+    expect(result.action).toBe("skip");
+    expect(result.reason).toBe("llm-skip");
+    expect(result.existingText).toBe("我喜欢吃麦当劳的麦辣鸡翅");
+    expect(llmCalls).toBe(1);
+  });
+
+  it("does not trigger the brand-item guard for non-preference text that mentions the same brand and item", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("我喜欢喝星巴克的抹茶拿铁", 0.75),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "same context, not a new preference" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "我们刚讨论过星巴克的抹茶拿铁做法",
+      llm as any,
+    );
+
+    expect(result.action).toBe("skip");
+    expect(result.reason).toBe("llm-skip");
+    expect(result.existingText).toBe("我喜欢喝星巴克的抹茶拿铁");
+    expect(llmCalls).toBe(1);
+  });
+
+  it("stores a new reply-style preference instead of collapsing different style traits into one topic", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("User prefers concise, direct replies.", 0.91),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "same topic" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "User prefers colloquial, grounded replies.",
+      llm as any,
+    );
+
+    expect(result.action).toBe("store");
+    expect(result.reason).toBe("unique");
+    expect(result.existingText).toBe("User prefers concise, direct replies.");
+    expect(llmCalls).toBe(0);
+  });
+
+  it("does not trigger the reply-style guard for descriptive text about a draft", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("User prefers colloquial, grounded replies.", 0.75),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "draft note, not a user preference" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "这段文案简洁直接，先别改。",
+      llm as any,
+    );
+
+    expect(result.action).toBe("skip");
+    expect(result.reason).toBe("llm-skip");
+    expect(result.existingText).toBe("User prefers colloquial, grounded replies.");
+    expect(llmCalls).toBe(1);
+  });
+
+  it("stores a new tool-choice preference instead of collapsing different tool choices into one topic", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("Uses Bun over Node.", 0.91),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "same topic" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "Prefers rg over grep.",
+      llm as any,
+    );
+
+    expect(result.action).toBe("store");
+    expect(result.reason).toBe("unique");
+    expect(result.existingText).toBe("Uses Bun over Node.");
+    expect(llmCalls).toBe(0);
+  });
+
+  it("does not trigger the tool-choice guard for narrative migration text", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult("Prefers rg over grep.", 0.75),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "SKIP" as const, reason: "migration note, not a tool preference" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "文档里写了 uses Bun over Node 的迁移说明。",
+      llm as any,
+    );
+
+    expect(result.action).toBe("skip");
+    expect(result.reason).toBe("llm-skip");
+    expect(result.existingText).toBe("Prefers rg over grep.");
+    expect(llmCalls).toBe(1);
+  });
+
+  it("stores a borderline chunk when LLM says it should CREATE", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult(
+            "A2A gateway 之前是先本地 smoke，再补 Claude SDK 配置。",
+            0.74,
+          ),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "CREATE" as const, reason: "new implementation branch" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "A2A gateway 升级后要补 MCP transport 配置和 LaunchAgent 环境变量，不是原来的 smoke 流程。",
+      llm as any,
+    );
+
+    expect(result.action).toBe("store");
+    expect(result.reason).toBe("unique");
+    expect(llmCalls).toBe(1);
+  });
+
+  it("stores a borderline chunk when LLM says it adds new information and should MERGE", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult(
+            "A2A 调 Claude SDK 时，可以先看 `adapters/claude.js` 里的权限相关配置。",
+            0.77,
+          ),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "MERGE" as const, reason: "same topic but new implementation detail" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "给 A2A 调用传 permissionMode: \"dontAsk\" + allowedTools，避免 Claude SDK 在后台 HTTP handler 里卡死等权限确认。",
+      llm as any,
+    );
+
+    expect(result.action).toBe("store");
+    expect(result.reason).toBe("llm-merge");
+    expect(result.existingText).toContain("权限相关配置");
+    expect(llmCalls).toBe(1);
+  });
+
+  it("skips a hard-threshold duplicate without calling the LLM", async () => {
+    let llmCalls = 0;
+    const store = {
+      async vectorSearch() {
+        return [
+          buildSearchResult(
+            "OpenClaw provider 配置里已经写过 `ANTHROPIC_BASE_URL` 和 `ANTHROPIC_AUTH_TOKEN` 的设置方法。",
+            0.95,
+          ),
+        ];
+      },
+    };
+    const llm = {
+      async dedupDecision() {
+        llmCalls += 1;
+        return { action: "MERGE" as const, reason: "should never be called" };
+      },
+    };
+
+    const result = await dedupCheck(
+      store as any,
+      [1, 0, 0],
+      "OpenClaw provider 配置里已经写过 `ANTHROPIC_BASE_URL` 和 `ANTHROPIC_AUTH_TOKEN` 的设置方法，但这里重复了一遍。",
+      llm as any,
+    );
+
+    expect(result.action).toBe("skip");
+    expect(result.reason).toBe("hard");
+    expect(llmCalls).toBe(0);
+  });
+});