recallnest 1.4.1

package/src/__tests__/access-tracker-novelty.test.ts

@@ -0,0 +1,228 @@
+/**
+ * Tests for Tier 3.2: Conditional access reinforcement (prediction error gating).
+ *
+ * Validates that:
+ * 1. High-similarity (low-novelty) results are NOT reinforced
+ * 2. Low-similarity (high-novelty) results ARE reinforced
+ * 3. Cooldown prevents duplicate reinforcement within the window
+ * 4. When no scores provided, all entries are reinforced (backward compat)
+ * 5. shouldReinforce() logic is correct in isolation
+ */
+import { describe, expect, it, beforeEach } from "bun:test";
+import { AccessTracker, DEFAULT_ACCESS_TRACKER_CONFIG } from "../access-tracker.js";
+import type { MemoryEntry } from "../store.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeEntry(id: string, overrides: Partial<MemoryEntry> = {}): MemoryEntry {
+  return {
+    id,
+    text: `memory-${id}`,
+    vector: [1, 0, 0],
+    category: "events",
+    scope: "project:test",
+    importance: 0.7,
+    timestamp: Date.now(),
+    metadata: JSON.stringify({ accessCount: 0, lastAccessedAt: 0 }),
+    ...overrides,
+  };
+}
+
+function createMockStore(entries: MemoryEntry[]) {
+  const data = new Map(entries.map(e => [e.id, { ...e }]));
+  const updates: Array<{ id: string; metadata?: string }> = [];
+
+  return {
+    updates,
+    data,
+    async getById(id: string) {
+      return data.get(id) ?? null;
+    },
+    async update(id: string, upd: { metadata?: string }) {
+      const entry = data.get(id);
+      if (!entry) return null;
+      if (upd.metadata) {
+        entry.metadata = upd.metadata;
+        updates.push({ id, metadata: upd.metadata });
+      }
+      return entry;
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// shouldReinforce() unit tests
+// ---------------------------------------------------------------------------
+
+describe("shouldReinforce", () => {
+  let tracker: AccessTracker;
+
+  beforeEach(() => {
+    tracker = new AccessTracker(
+      createMockStore([]) as any,
+      { ...DEFAULT_ACCESS_TRACKER_CONFIG, noveltyThreshold: 0.35, cooldownMs: 300_000 },
+    );
+  });
+
+  it("rejects high-similarity (low-novelty) results", () => {
+    // similarity 0.90 → novelty 0.10 < threshold 0.35 → reject
+    expect(tracker.shouldReinforce("a", 0.90)).toBe(false);
+  });
+
+  it("rejects exact match (similarity 1.0)", () => {
+    expect(tracker.shouldReinforce("a", 1.0)).toBe(false);
+  });
+
+  it("accepts low-similarity (high-novelty) results", () => {
+    // similarity 0.50 → novelty 0.50 > threshold 0.35 → accept
+    expect(tracker.shouldReinforce("a", 0.50)).toBe(true);
+  });
+
+  it("accepts borderline novelty (exactly at threshold)", () => {
+    // similarity 0.65 → novelty 0.35 = threshold 0.35 → accept (not strictly less)
+    expect(tracker.shouldReinforce("a", 0.65)).toBe(true);
+  });
+
+  it("rejects same entry within cooldown window", () => {
+    // First call: accepted
+    expect(tracker.shouldReinforce("a", 0.50)).toBe(true);
+    // Simulate recording to set cooldown
+    tracker.recordAccess(["a"], [0.50]);
+    // Second call within cooldown: rejected
+    expect(tracker.shouldReinforce("a", 0.50)).toBe(false);
+  });
+
+  it("accepts different entry even if another is on cooldown", () => {
+    tracker.recordAccess(["a"], [0.50]);
+    expect(tracker.shouldReinforce("b", 0.50)).toBe(true);
+  });
+
+  it("bypasses novelty gate when threshold is 0", () => {
+    const noGateTracker = new AccessTracker(
+      createMockStore([]) as any,
+      { ...DEFAULT_ACCESS_TRACKER_CONFIG, noveltyThreshold: 0, cooldownMs: 0 },
+    );
+    // Even high similarity passes
+    expect(noGateTracker.shouldReinforce("a", 0.99)).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// recordAccess() with scores
+// ---------------------------------------------------------------------------
+
+describe("recordAccess with novelty gating", () => {
+  it("only queues novel entries for flush", async () => {
+    const entries = [makeEntry("a"), makeEntry("b"), makeEntry("c")];
+    const store = createMockStore(entries);
+    const tracker = new AccessTracker(store as any, {
+      ...DEFAULT_ACCESS_TRACKER_CONFIG,
+      flushIntervalMs: 0, // immediate flush for testing
+      noveltyThreshold: 0.35,
+      cooldownMs: 0,
+    });
+
+    // scores: a=0.90 (low novelty, reject), b=0.50 (novel, accept), c=0.40 (novel, accept)
+    tracker.recordAccess(["a", "b", "c"], [0.90, 0.50, 0.40]);
+
+    await tracker.flush();
+
+    // Only b and c should have been updated
+    const updatedIds = store.updates.map(u => u.id);
+    expect(updatedIds).not.toContain("a");
+    expect(updatedIds).toContain("b");
+    expect(updatedIds).toContain("c");
+  });
+
+  it("reinforces all entries when no scores provided (backward compat)", async () => {
+    const entries = [makeEntry("a"), makeEntry("b")];
+    const store = createMockStore(entries);
+    const tracker = new AccessTracker(store as any, {
+      ...DEFAULT_ACCESS_TRACKER_CONFIG,
+      flushIntervalMs: 0,
+      noveltyThreshold: 0.35,
+      cooldownMs: 0,
+    });
+
+    // No scores → all reinforced
+    tracker.recordAccess(["a", "b"]);
+    await tracker.flush();
+
+    const updatedIds = store.updates.map(u => u.id);
+    expect(updatedIds).toContain("a");
+    expect(updatedIds).toContain("b");
+  });
+
+  it("respects cooldown across multiple recordAccess calls", async () => {
+    const entries = [makeEntry("x")];
+    const store = createMockStore(entries);
+    const tracker = new AccessTracker(store as any, {
+      ...DEFAULT_ACCESS_TRACKER_CONFIG,
+      flushIntervalMs: 0,
+      noveltyThreshold: 0.35,
+      cooldownMs: 600_000, // 10 min cooldown
+    });
+
+    // First call: novel enough → queued
+    tracker.recordAccess(["x"], [0.50]);
+    await tracker.flush();
+    expect(store.updates.length).toBe(1);
+
+    // Second call: same entry within cooldown → not queued
+    tracker.recordAccess(["x"], [0.50]);
+    await tracker.flush();
+    // Still only 1 update (no new flush)
+    expect(store.updates.length).toBe(1);
+  });
+
+  it("skips flush scheduling when all entries are gated out", () => {
+    const store = createMockStore([]);
+    const tracker = new AccessTracker(store as any, {
+      ...DEFAULT_ACCESS_TRACKER_CONFIG,
+      flushIntervalMs: 5000,
+      noveltyThreshold: 0.35,
+      cooldownMs: 0,
+    });
+
+    // All scores too high (low novelty) → nothing queued
+    tracker.recordAccess(["a", "b"], [0.95, 0.90]);
+    expect(tracker.pendingCount).toBe(0);
+  });
+
+  it("accessCount increments correctly after novelty-gated flush", async () => {
+    const entries = [makeEntry("m1")];
+    const store = createMockStore(entries);
+    const tracker = new AccessTracker(store as any, {
+      ...DEFAULT_ACCESS_TRACKER_CONFIG,
+      flushIntervalMs: 0,
+      noveltyThreshold: 0.35,
+      cooldownMs: 0,
+    });
+
+    tracker.recordAccess(["m1"], [0.40]); // novel → accepted
+    await tracker.flush();
+
+    const updated = store.data.get("m1")!;
+    const meta = JSON.parse(updated.metadata);
+    expect(meta.accessCount).toBe(1);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// destroy() cleans cooldown
+// ---------------------------------------------------------------------------
+
+describe("destroy cleans cooldown", () => {
+  it("clears cooldown map on destroy", () => {
+    const tracker = new AccessTracker(
+      createMockStore([]) as any,
+      { ...DEFAULT_ACCESS_TRACKER_CONFIG, cooldownMs: 600_000 },
+    );
+    tracker.recordAccess(["a"], [0.40]);
+    tracker.destroy();
+    // After destroy, same entry should be reinforceable again
+    expect(tracker.shouldReinforce("a", 0.40)).toBe(true);
+  });
+});

package/src/__tests__/auto-consolidation.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, it, beforeEach } from "bun:test";
+
+import {
+  maybeConsolidate,
+  resetConsolidationState,
+  DEFAULT_AUTO_CONSOLIDATION_CONFIG,
+  type AutoConsolidationConfig,
+} from "../auto-consolidation.js";
+import type { MemoryEntry, MemorySearchResult } from "../store.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeEntry(id: string, text: string, overrides: Partial<MemoryEntry> = {}): MemoryEntry {
+  return {
+    id,
+    text,
+    vector: [1, 0, 0],
+    category: "events",
+    scope: "project:test",
+    importance: 0.7,
+    timestamp: Date.now(),
+    metadata: "{}",
+    ...overrides,
+  };
+}
+
+function createMockStore(entries: MemoryEntry[]) {
+  const data = new Map(entries.map(e => [e.id, { ...e }]));
+  return {
+    async list(scopeFilter?: string[], _category?: string, limit = 500, _offset = 0) {
+      return [...data.values()]
+        .filter(e => !scopeFilter || scopeFilter.some(s => e.scope === s))
+        .slice(0, limit);
+    },
+    async getById(id: string) {
+      return data.get(id) ?? null;
+    },
+    async vectorSearch(_vector: number[], _limit = 5, _minScore = 0.3, _scopeFilter?: string[]) {
+      return [] as MemorySearchResult[];
+    },
+    async update(id: string, updates: any, _scopeFilter?: string[]) {
+      const entry = data.get(id);
+      if (!entry) return null;
+      Object.assign(entry, updates);
+      return entry;
+    },
+    async stats(scopeFilter?: string[]) {
+      const filtered = [...data.values()].filter(
+        e => !scopeFilter || scopeFilter.some(s => e.scope === s),
+      );
+      return { total: filtered.length, byCategory: {}, byScope: {} };
+    },
+    hasFtsSupport: false,
+  } as any;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("auto-consolidation", () => {
+  beforeEach(() => {
+    resetConsolidationState();
+  });
+
+  it("skips when not enough new memories", async () => {
+    const entries = Array.from({ length: 10 }, (_, i) =>
+      makeEntry(`id-${i}`, `memory ${i}`),
+    );
+    const store = createMockStore(entries);
+
+    const result = await maybeConsolidate(store, "project:test");
+    expect(result.triggered).toBe(false);
+    expect(result.reason).toBe("insufficient_new_memories");
+  });
+
+  it("skips when too soon since last run", async () => {
+    // Start with 60 entries, trigger first run, then grow to 120 so
+    // the memory-count gate passes but the time gate blocks.
+    const data: MemoryEntry[] = Array.from({ length: 60 }, (_, i) =>
+      makeEntry(`id-${i}`, `memory ${i}`),
+    );
+    const store = createMockStore(data);
+
+    const config: AutoConsolidationConfig = {
+      minNewMemories: 50,
+      minHoursSinceLastRun: 12,
+      consolidation: DEFAULT_AUTO_CONSOLIDATION_CONFIG.consolidation,
+    };
+
+    // First run: triggers (never run before, 60 > 50 new memories)
+    const first = await maybeConsolidate(store, "project:test", config);
+    expect(first.triggered).toBe(true);
+
+    // Grow the store so memory-count gate would pass
+    for (let i = 60; i < 120; i++) {
+      data.push(makeEntry(`id-${i}`, `memory ${i}`));
+    }
+    const bigStore = createMockStore(data);
+
+    // Second run immediately: time gate blocks (< 12 hours)
+    const second = await maybeConsolidate(bigStore, "project:test", config);
+    expect(second.triggered).toBe(false);
+    expect(second.reason).toBe("too_soon");
+  });
+
+  it("triggers when both conditions met", async () => {
+    const entries = Array.from({ length: 60 }, (_, i) =>
+      makeEntry(`id-${i}`, `memory ${i}`),
+    );
+    const store = createMockStore(entries);
+
+    const config: AutoConsolidationConfig = {
+      minNewMemories: 50,
+      minHoursSinceLastRun: 0, // disable time gate for test
+      consolidation: {
+        clusterThreshold: 0.82,
+        mergeThreshold: 0.92,
+        maxEntriesPerRun: 500,
+      },
+    };
+
+    const result = await maybeConsolidate(store, "project:test", config);
+    expect(result.triggered).toBe(true);
+    expect(result.consolidation).toBeDefined();
+    expect(result.consolidation!.scope).toBe("project:test");
+  });
+
+  it("returns consolidation result when triggered", async () => {
+    const entries = Array.from({ length: 60 }, (_, i) =>
+      makeEntry(`id-${i}`, `memory ${i}`),
+    );
+    const store = createMockStore(entries);
+
+    const config: AutoConsolidationConfig = {
+      minNewMemories: 50,
+      minHoursSinceLastRun: 0,
+      consolidation: {
+        clusterThreshold: 0.82,
+        mergeThreshold: 0.92,
+        maxEntriesPerRun: 500,
+      },
+    };
+
+    const result = await maybeConsolidate(store, "project:test", config);
+    expect(result.triggered).toBe(true);
+    expect(result.consolidation!.originalCount).toBe(60);
+  });
+
+  it("respects custom thresholds", async () => {
+    const entries = Array.from({ length: 200 }, (_, i) =>
+      makeEntry(`id-${i}`, `memory ${i}`),
+    );
+    const store = createMockStore(entries);
+
+    const config: AutoConsolidationConfig = {
+      minNewMemories: 300, // higher than available
+      minHoursSinceLastRun: 0,
+      consolidation: DEFAULT_AUTO_CONSOLIDATION_CONFIG.consolidation,
+    };
+
+    const result = await maybeConsolidate(store, "project:test", config);
+    expect(result.triggered).toBe(false);
+    expect(result.reason).toBe("insufficient_new_memories");
+  });
+});

package/src/__tests__/auto-recall.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, it } from "bun:test";
+
+import { runAutoRecall } from "../auto-recall.js";
+import type { RetrievalContext, RetrievalResult } from "../retriever.js";
+import type { SessionCheckpointRecord } from "../session-schema.js";
+
+function buildResult(id: string, category: "profile" | "preferences" | "entities" | "patterns" | "cases" | "events", text: string, scope = "project:test"): RetrievalResult {
+  return {
+    entry: {
+      id,
+      text,
+      vector: [],
+      category,
+      scope,
+      importance: 0.8,
+      timestamp: Date.parse("2026-03-17T00:00:00.000Z"),
+      metadata: "{}",
+    },
+    score: 0.9,
+    sources: {
+      fused: { score: 0.9 },
+    },
+  };
+}
+
+describe("runAutoRecall", () => {
+  it("reuses an explicit scope for focused recall after composing resume context", async () => {
+    const calls: RetrievalContext[] = [];
+    const retriever = {
+      async retrieve(context: RetrievalContext): Promise<RetrievalResult[]> {
+        calls.push(context);
+        if (context.category === "profile") {
+          return [buildResult("profile-1", "profile", "Profile: RecallNest maintainer.", "project:alpha")];
+        }
+        if (!context.category) {
+          return [buildResult("event-1", "events", "RecallNest uses Weibull decay.", "project:alpha")];
+        }
+        return [];
+      },
+    };
+
+    const response = await runAutoRecall({
+      retriever,
+      checkpointStore: { async getLatest() { return null; } },
+      listPins: () => [],
+    }, {
+      message: "那个衰减曲线用的什么公式来着",
+      scope: "project:alpha",
+      limit: 5,
+      profile: "default",
+      operation: "test:auto-recall",
+    });
+
+    expect(response.mode).toBe("resume+search");
+    expect(response.resolvedScope).toBe("project:alpha");
+    expect(response.results[0]?.entry.id).toBe("event-1");
+    const searchCall = calls.find((call) => !call.category);
+    expect(searchCall?.scopeFilter).toEqual(["project:alpha"]);
+    expect(searchCall?.source).toBe("auto-recall");
+  });
+
+  it("uses the checkpoint-resolved scope when the request only carries sessionId", async () => {
+    const calls: RetrievalContext[] = [];
+    const checkpoint: SessionCheckpointRecord = {
+      checkpointId: "checkpoint-1",
+      sessionId: "session-123",
+      resolvedScope: "project:recallnest",
+      summary: "Continue proactive recall work.",
+      task: "Implement auto recall route",
+      decisions: [],
+      openLoops: [],
+      nextActions: [],
+      entities: [],
+      files: [],
+      updatedAt: "2026-03-17T10:00:00.000Z",
+    };
+
+    const retriever = {
+      async retrieve(context: RetrievalContext): Promise<RetrievalResult[]> {
+        calls.push(context);
+        if (!context.category) {
+          return [buildResult("event-1", "events", "Add a true auto recall route.", "project:recallnest")];
+        }
+        return [];
+      },
+    };
+
+    const response = await runAutoRecall({
+      retriever,
+      checkpointStore: {
+        async getLatest(query?: { sessionId?: string }) {
+          return query?.sessionId === "session-123" ? checkpoint : null;
+        },
+      },
+      listPins: () => [],
+    }, {
+      message: "继续做主动 recall",
+      sessionId: "session-123",
+      operation: "test:auto-recall",
+    });
+
+    expect(response.resolvedScope).toBe("project:recallnest");
+    expect(response.resume.latestCheckpoint?.resolvedScope).toBe("project:recallnest");
+    const searchCall = calls.find((call) => !call.category);
+    expect(searchCall?.scopeFilter).toEqual(["project:recallnest"]);
+  });
+
+  it("falls back to resume-only mode when no scope can be inferred for focused search", async () => {
+    const calls: RetrievalContext[] = [];
+    const retriever = {
+      async retrieve(context: RetrievalContext): Promise<RetrievalResult[]> {
+        calls.push(context);
+        return [];
+      },
+    };
+
+    const response = await runAutoRecall({
+      retriever,
+      checkpointStore: { async getLatest() { return null; } },
+      listPins: () => [],
+    }, {
+      message: "what do you remember about this",
+      operation: "test:auto-recall",
+    });
+
+    expect(response.mode).toBe("resume-only");
+    expect(response.results).toHaveLength(0);
+    expect(response.searchSkippedReason).toContain("No explicit or inferred scope");
+    expect(calls.some((call) => !call.category)).toBe(false);
+  });
+});

package/src/__tests__/batch-dedup.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from "bun:test";
+import { batchInternalDedup } from "../ingest.js";
+
+describe("batchInternalDedup", () => {
+  it("keeps all entries when vectors are dissimilar", () => {
+    const vectors = [
+      [1, 0, 0],
+      [0, 1, 0],
+      [0, 0, 1],
+    ];
+    const importances = [0.8, 0.7, 0.6];
+    const kept = batchInternalDedup(vectors, importances);
+    expect(kept).toEqual([0, 1, 2]);
+  });
+
+  it("drops lower-importance duplicate when vectors are near-identical", () => {
+    const vectors = [
+      [1, 0, 0],    // importance 0.6
+      [1, 0.01, 0], // importance 0.9 — nearly identical to [0]
+      [0, 1, 0],    // importance 0.7 — dissimilar
+    ];
+    const importances = [0.6, 0.9, 0.7];
+    const kept = batchInternalDedup(vectors, importances);
+    // [1] has higher importance and is near-identical to [0], so [0] gets dropped
+    expect(kept).toContain(1);
+    expect(kept).toContain(2);
+    expect(kept).not.toContain(0);
+  });
+
+  it("returns single-item batch unchanged", () => {
+    expect(batchInternalDedup([[1, 0, 0]], [0.5])).toEqual([0]);
+  });
+
+  it("returns empty for empty input", () => {
+    expect(batchInternalDedup([], [])).toEqual([]);
+  });
+
+  it("preserves original order in output", () => {
+    const vectors = [
+      [0, 1, 0],    // idx 0, imp 0.9
+      [1, 0, 0],    // idx 1, imp 0.5
+      [0, 0, 1],    // idx 2, imp 0.7
+    ];
+    const importances = [0.9, 0.5, 0.7];
+    const kept = batchInternalDedup(vectors, importances);
+    // All dissimilar, all kept, in original order
+    expect(kept).toEqual([0, 1, 2]);
+  });
+
+  it("respects custom threshold", () => {
+    const vectors = [
+      [1, 0, 0],
+      [0.95, 0.31, 0], // cosine ~0.95 with [0]
+    ];
+    const importances = [0.8, 0.7];
+
+    // Default threshold 0.85 → should dedup
+    const strictKept = batchInternalDedup(vectors, importances, 0.85);
+    expect(strictKept).toHaveLength(1);
+
+    // Loose threshold 0.99 → should keep both
+    const looseKept = batchInternalDedup(vectors, importances, 0.99);
+    expect(looseKept).toHaveLength(2);
+  });
+});