recallnest 2.3.0 → 2.5.4

package/src/__tests__/memory-output.test.ts

@@ -0,0 +1,215 @@
+import { describe, expect, it } from "bun:test";
+
+import { formatExplainResults, formatSearchResults } from "../memory-output.js";
+import type { RetrievalResult } from "../retriever.js";
+
+function buildResult(id: string, metadata: Record<string, unknown>): RetrievalResult {
+  return {
+    entry: {
+      id,
+      text: "User prefers concise, direct replies.",
+      vector: [],
+      category: "preferences",
+      scope: "memory:agent",
+      importance: 0.8,
+      timestamp: Date.parse("2026-03-16T00:00:00.000Z"),
+      metadata: JSON.stringify(metadata),
+    },
+    score: 0.91,
+    sources: {
+      vector: { score: 0.9, rank: 1 },
+      bm25: { score: 0.8, rank: 2 },
+      fused: { score: 0.91 },
+    },
+  };
+}
+
+describe("memory output", () => {
+  it("includes provenance in search results", () => {
+    const output = formatSearchResults([
+      buildResult("abcd1234-0000-0000-0000-000000000001", {
+        source: "agent",
+        canonicalKey: "user-reply-style",
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "preferences",
+        },
+        promotedFrom: {
+          memoryId: "feedface-0000-0000-0000-000000000001",
+          scope: "cc:session1",
+          category: "events",
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+            originalCategory: "preferences",
+          },
+        },
+        provenanceHistory: [
+          {
+            memoryId: "feedface-0000-0000-0000-000000000001",
+            scope: "cc:session1",
+            category: "events",
+            source: "cc",
+          },
+          {
+            memoryId: "deadbeef-0000-0000-0000-000000000002",
+            scope: "cc:session2",
+            category: "events",
+            source: "cc",
+            observedAt: "2026-03-17T04:30:00.000Z",
+            boundary: {
+              layer: "evidence",
+              authority: "transcript-ingest",
+              conflictPolicy: "append-only",
+              originalCategory: "preferences",
+            },
+          },
+        ],
+        provenanceHistoryCount: 2,
+        preferenceSlot: {
+          type: "brand-item",
+          brand: "麦当劳",
+          item: "麦辣鸡翅",
+        },
+      }),
+    ], {
+      query: "reply style",
+      profile: "default",
+    });
+
+    expect(output).toContain("prov : durable/structured-memory");
+    expect(output).toContain("key:user-reply-style");
+    expect(output).toContain("promoted:feedface<-evidence/transcript-ingest");
+    expect(output).toContain("history:2");
+    expect(output).toContain("observed:deadbeef@2026-03-17");
+    expect(output).toContain("slot:brand-item:麦当劳:麦辣鸡翅");
+  });
+
+  it("includes provenance in explain results", () => {
+    const output = formatExplainResults([
+      buildResult("abcd1234-0000-0000-0000-000000000001", {
+        source: "cc",
+        boundary: {
+          layer: "evidence",
+          authority: "transcript-ingest",
+          conflictPolicy: "append-only",
+          originalCategory: "preferences",
+          downgradedFrom: "preferences",
+        },
+      }),
+    ], {
+      query: "reply style",
+      profile: "writing",
+    });
+
+    expect(output).toContain("prov    : evidence/transcript-ingest");
+    expect(output).toContain("downgraded:preferences");
+  });
+
+  it("renders reply-style slots in provenance summaries", () => {
+    const output = formatSearchResults([
+      buildResult("abcd1234-0000-0000-0000-000000000002", {
+        source: "agent",
+        canonicalKey: "preferences:reply-style:concise:direct",
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "preferences",
+        },
+        preferenceSlot: {
+          type: "reply-style",
+          traits: ["concise", "direct"],
+        },
+      }),
+    ], {
+      query: "reply style",
+      profile: "default",
+    });
+
+    expect(output).toContain("slot:reply-style:concise:direct");
+  });
+
+  it("renders tool-choice slots in provenance summaries", () => {
+    const output = formatSearchResults([
+      buildResult("abcd1234-0000-0000-0000-000000000003", {
+        source: "agent",
+        canonicalKey: "preferences:tool-choice:bun:over:node",
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "preferences",
+        },
+        preferenceSlot: {
+          type: "tool-choice",
+          preferredTool: "bun",
+          avoidedTool: "node",
+        },
+      }),
+    ], {
+      query: "tool choice",
+      profile: "default",
+    });
+
+    expect(output).toContain("slot:tool-choice:bun:over:node");
+  });
+
+  it("does not render slot provenance for plain preferences canonical keys", () => {
+    const output = formatSearchResults([
+      buildResult("abcd1234-0000-0000-0000-000000000004", {
+        source: "agent",
+        canonicalKey: "preferences:这段文案简洁直接-先别改",
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "preferences",
+        },
+      }),
+      buildResult("abcd1234-0000-0000-0000-000000000005", {
+        source: "agent",
+        canonicalKey: "preferences:文档里写了-uses-bun-over-node-的迁移说明",
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "preferences",
+        },
+      }),
+    ], {
+      query: "preferences",
+      profile: "default",
+    });
+
+    expect(output).toContain("key:preferences:这段文案简洁直接-先别改");
+    expect(output).toContain("key:preferences:文档里写了-uses-bun-over-node-的迁移说明");
+    expect(output).not.toContain("slot:reply-style:");
+    expect(output).not.toContain("slot:tool-choice:");
+  });
+
+  it("does not render slot provenance in explain output for plain preferences canonical keys", () => {
+    const output = formatExplainResults([
+      buildResult("abcd1234-0000-0000-0000-000000000006", {
+        source: "agent",
+        canonicalKey: "preferences:这段文案简洁直接-先别改",
+        boundary: {
+          layer: "durable",
+          authority: "structured-memory",
+          conflictPolicy: "latest-wins",
+          originalCategory: "preferences",
+        },
+      }),
+    ], {
+      query: "draft note",
+      profile: "default",
+    });
+
+    expect(output).toContain("key:preferences:这段文案简洁直接-先别改");
+    expect(output).not.toContain("slot:reply-style:");
+    expect(output).not.toContain("slot:tool-choice:");
+  });
+});

package/src/__tests__/memory-promotion.test.ts

@@ -0,0 +1,315 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+  scanMemoryPromotions,
+  type PromoteScanDeps,
+  type PromoteRequest,
+} from "../memory-promotion.js";
+import type { MemoryEntry } from "../store.js";
+import type { StoredPromotedMemoryRecord } from "../memory-schema.js";
+
+// ---------------------------------------------------------------------------
+// Mock helpers
+// ---------------------------------------------------------------------------
+
+let idSeq = 0;
+
+const ACTIVE_EVOLUTION = {
+  status: "active",
+  version: 1,
+  accessCount: 0,
+  lastAccessedAt: null,
+  supersededBy: null,
+  consolidatedInto: null,
+  sourceMemories: [],
+  validFrom: 1_700_000_000_000,
+  validUntil: null,
+};
+
+/**
+ * Build a transcript-downgraded evidence entry (category=events) with an EMPTY
+ * vector, faithfully reproducing store.list — which never returns vectors.
+ */
+function makeDowngradedEvent(
+  downgradedFrom: "profile" | "preferences",
+  overrides: Partial<MemoryEntry> = {},
+): MemoryEntry {
+  idSeq += 1;
+  return {
+    id: `ev-${String(idSeq).padStart(4, "0")}`,
+    text: `${downgradedFrom} hint #${idSeq}`,
+    vector: [], // list() never returns vectors
+    category: "events",
+    scope: "cc:project:test",
+    importance: 0.7,
+    timestamp: 1_700_000_000_000 + idSeq,
+    metadata: JSON.stringify({
+      boundary: {
+        layer: "evidence",
+        authority: "transcript-ingest",
+        conflictPolicy: "append-only",
+        originalCategory: downgradedFrom,
+        downgradedFrom,
+      },
+      evolution: ACTIVE_EVOLUTION,
+    }),
+    ...overrides,
+  };
+}
+
+interface MockOptions {
+  /** Per-id vector overrides; ids absent from the map get no vector. */
+  vectorMap?: Map<string, number[]>;
+  /** Fallback vector handed to every listed id when vectorMap is omitted. */
+  defaultVector?: number[];
+}
+
+function createScanDeps(entries: MemoryEntry[], options: MockOptions = {}) {
+  const promoteCalls: PromoteRequest[] = [];
+  const deps: PromoteScanDeps = {
+    store: {
+      async list(
+        _scopeFilter?: string[],
+        category?: string,
+        _limit?: number,
+        _offset?: number,
+      ): Promise<MemoryEntry[]> {
+        return category ? entries.filter((e) => e.category === category) : entries;
+      },
+      async getVectors(ids: string[]): Promise<Map<string, number[]>> {
+        const map = new Map<string, number[]>();
+        for (const id of ids) {
+          if (options.vectorMap?.has(id)) {
+            map.set(id, options.vectorMap.get(id)!);
+          } else if (options.defaultVector && !options.vectorMap) {
+            map.set(id, options.defaultVector);
+          }
+        }
+        return map;
+      },
+    },
+    async promote(req: PromoteRequest): Promise<StoredPromotedMemoryRecord> {
+      promoteCalls.push(req);
+      return {
+        text: "promoted text",
+        category: req.category,
+        importance: req.importance,
+        scope: req.scope,
+        source: "agent",
+        tags: [],
+        canonicalKey: `${req.category}:k${promoteCalls.length}`,
+        id: `durable-${req.memoryId}`,
+        resolvedScope: req.scope,
+        storedAt: "2026-05-29T00:00:00.000Z",
+        disposition: "promoted",
+        sourceMemoryId: req.memoryId,
+        sourceCategory: "events",
+      };
+    },
+  };
+  return { deps, promoteCalls };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("scanMemoryPromotions", () => {
+  it("promotes a cluster once it has >= minOccurrences similar members", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(1);
+    expect(result.candidates[0].occurrences).toBe(3);
+    expect(result.candidates[0].downgradedFrom).toBe("preferences");
+    expect(result.promoted).toBe(1);
+    expect(promoteCalls.length).toBe(1);
+    expect(promoteCalls[0].category).toBe("preferences");
+    expect(promoteCalls[0].scope).toBe("cc:project:test");
+  });
+
+  it("does not promote when occurrences are below minOccurrences", async () => {
+    const entries = [makeDowngradedEvent("preferences"), makeDowngradedEvent("preferences")];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("does not promote a cluster whose average importance is below minImportance", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences", { importance: 0.4 }),
+      makeDowngradedEvent("preferences", { importance: 0.4 }),
+      makeDowngradedEvent("preferences", { importance: 0.4 }),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("promotes with the target category equal to downgradedFrom (profile)", async () => {
+    const entries = [
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(promoteCalls.length).toBe(1);
+    expect(promoteCalls[0].category).toBe("profile");
+  });
+
+  it("finds candidates but does NOT call promote in dryRun mode", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: true });
+
+    expect(result.candidates.length).toBe(1);
+    expect(result.dryRun).toBe(true);
+    expect(result.promoted).toBe(0);
+    expect(result.candidates[0].promoted).toBeNull();
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("keeps profile and preferences as separate candidates with their own category", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+      makeDowngradedEvent("profile"),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(2);
+    expect(promoteCalls.length).toBe(2);
+    expect(promoteCalls.map((c) => c.category).sort()).toEqual(["preferences", "profile"]);
+  });
+
+  it("skips entries that have no vector available", async () => {
+    const entries = [
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+      makeDowngradedEvent("preferences"),
+    ];
+    // Only 2 of 3 receive vectors -> the cluster has 2 members < minOccurrences.
+    const vectorMap = new Map<string, number[]>();
+    vectorMap.set(entries[0].id, [1, 0, 0]);
+    vectorMap.set(entries[1].id, [1, 0, 0]);
+    const { deps, promoteCalls } = createScanDeps(entries, { vectorMap });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("ignores events that are not downgraded from profile/preferences", async () => {
+    const plain = (): MemoryEntry =>
+      makeDowngradedEvent("preferences", {
+        metadata: JSON.stringify({
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+          },
+          evolution: ACTIVE_EVOLUTION,
+        }),
+      });
+    const { deps, promoteCalls } = createScanDeps([plain(), plain(), plain()], {
+      defaultVector: [1, 0, 0],
+    });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("skips evidence already promoted (promotedTo marker present)", async () => {
+    const promotedAlready = (): MemoryEntry =>
+      makeDowngradedEvent("preferences", {
+        metadata: JSON.stringify({
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+            originalCategory: "preferences",
+            downgradedFrom: "preferences",
+          },
+          promotedTo: "existing-durable-id",
+          evolution: ACTIVE_EVOLUTION,
+        }),
+      });
+    const { deps, promoteCalls } = createScanDeps(
+      [promotedAlready(), promotedAlready(), promotedAlready()],
+      { defaultVector: [1, 0, 0] },
+    );
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("excludes inactive (superseded) evidence", async () => {
+    const superseded = (): MemoryEntry =>
+      makeDowngradedEvent("preferences", {
+        metadata: JSON.stringify({
+          boundary: {
+            layer: "evidence",
+            authority: "transcript-ingest",
+            conflictPolicy: "append-only",
+            originalCategory: "preferences",
+            downgradedFrom: "preferences",
+          },
+          evolution: { ...ACTIVE_EVOLUTION, status: "superseded", supersededBy: "x" },
+        }),
+      });
+    const { deps, promoteCalls } = createScanDeps(
+      [superseded(), superseded(), superseded()],
+      { defaultVector: [1, 0, 0] },
+    );
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+
+  it("excludes evidence whose text is a bridge-injected prompt (noise)", async () => {
+    const promptText = "# CC Reply Prompt\n\n你是 CC。你不是 Alice 的数字分身，也不是 E 化镜像。";
+    const entries = [
+      makeDowngradedEvent("preferences", { text: promptText }),
+      makeDowngradedEvent("preferences", { text: promptText }),
+      makeDowngradedEvent("preferences", { text: promptText }),
+    ];
+    const { deps, promoteCalls } = createScanDeps(entries, { defaultVector: [1, 0, 0] });
+
+    const result = await scanMemoryPromotions(deps, "cc:project:test", { dryRun: false });
+
+    expect(result.candidates.length).toBe(0);
+    expect(promoteCalls.length).toBe(0);
+  });
+});

package/src/__tests__/migrate-dedup-ids.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "bun:test";
+
+import { planDedup, type DedupRow } from "../../scripts/migrate-dedup-ids.js";
+
+/**
+ * P0-2 迁移脚本核心逻辑回归。LanceDB I/O 部分（main）由「在备份副本上实跑」做集成验证；
+ * 这里只测纯 keeper 选择：同 id 保留最新 timestamp（importance 破平）、每 id 唯一、不丢文本。
+ */
+
+function row(id: string, ts: number, importance: number, text: string): DedupRow {
+  return { id, timestamp: ts, importance, text, vector: [0, 0, 0], scope: "s", category: "entities", metadata: "{}" };
+}
+
+describe("planDedup (dup-id migration keeper selection)", () => {
+  it("keeps the latest-timestamp row per dup id and counts extras", () => {
+    const rows = [
+      row("dup1", 100, 0.5, "A"),
+      row("dup1", 300, 0.6, "A"), // latest ts → keeper
+      row("dup1", 200, 0.9, "A"),
+      row("uniq", 150, 0.5, "B"),
+    ];
+    const plan = planDedup(rows);
+    expect(plan.dupIds).toEqual(["dup1"]);
+    expect(plan.extraRows).toBe(2);
+    expect(plan.keepers.length).toBe(1);
+    expect(Number(plan.keepers[0].timestamp)).toBe(300);
+  });
+
+  it("breaks timestamp ties by highest importance", () => {
+    const rows = [
+      row("d", 500, 0.4, "X"),
+      row("d", 500, 0.95, "X"), // same ts, higher importance → keeper
+      row("d", 500, 0.7, "X"),
+    ];
+    const plan = planDedup(rows);
+    expect(plan.keepers.length).toBe(1);
+    expect(plan.keepers[0].importance).toBe(0.95);
+  });
+
+  it("leaves unique ids untouched and preserves every distinct id's text", () => {
+    const rows = [
+      row("a", 1, 0.5, "textA"),
+      row("a", 2, 0.5, "textA"), // dup
+      row("b", 1, 0.5, "textB"), // unique
+      row("c", 1, 0.5, "textC"),
+      row("c", 5, 0.5, "textC"), // dup
+    ];
+    const plan = planDedup(rows);
+    const dupSet = new Set(plan.dupIds);
+    // Simulate the migration result: untouched unique-id rows + one keeper per dup id.
+    const survivors = [...rows.filter((r) => !dupSet.has(r.id)), ...plan.keepers];
+    const ids = survivors.map((r) => r.id).sort();
+    expect(ids).toEqual(["a", "b", "c"]); // every distinct id survives
+    expect(new Set(ids).size).toBe(ids.length); // each id appears exactly once
+    expect(survivors.find((r) => r.id === "a")!.text).toBe("textA");
+    expect(survivors.find((r) => r.id === "c")!.text).toBe("textC");
+  });
+
+  it("returns an empty plan when there are no duplicates", () => {
+    const plan = planDedup([row("x", 1, 0.5, "1"), row("y", 2, 0.5, "2")]);
+    expect(plan.dupIds).toEqual([]);
+    expect(plan.extraRows).toBe(0);
+    expect(plan.keepers).toEqual([]);
+  });
+});