recallnest 1.4.1 → 2.2.0

package/src/__tests__/post-retrieval-filter.test.ts

@@ -0,0 +1,178 @@
+import { describe, expect, it } from "bun:test";
+
+import { filterByRelevance, type FilterConfig } from "../post-retrieval-filter.js";
+import type { LLMClient } from "../llm-client.js";
+import type { RetrievalResult } from "../retriever.js";
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeResult(id: string, text: string, score: number): RetrievalResult {
+  return {
+    entry: {
+      id,
+      text,
+      vector: [],
+      category: "events",
+      scope: "test",
+      importance: 0.5,
+      timestamp: Date.now(),
+    },
+    score,
+    sources: {},
+  };
+}
+
+function makeMockLLM(response: boolean[] | null, shouldThrow = false): LLMClient {
+  return {
+    chatJson: async () => {
+      if (shouldThrow) throw new Error("LLM failure");
+      return response;
+    },
+  } as unknown as LLMClient;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("filterByRelevance", () => {
+  it("returns empty array for empty results", async () => {
+    const llm = makeMockLLM([]);
+    const out = await filterByRelevance([], "some query", llm);
+    expect(out).toEqual([]);
+  });
+
+  it("returns original results when query is empty", async () => {
+    const results = [makeResult("a", "hello", 0.8)];
+    const llm = makeMockLLM([true]);
+    const out = await filterByRelevance(results, "", llm);
+    expect(out).toEqual(results);
+  });
+
+  it("returns all when LLM marks all as relevant", async () => {
+    const results = [
+      makeResult("a", "TypeScript project setup", 0.9),
+      makeResult("b", "Docker deployment notes", 0.85),
+    ];
+    const llm = makeMockLLM([true, true]);
+    const out = await filterByRelevance(results, "project setup", llm);
+    expect(out).toHaveLength(2);
+    expect(out[0].entry.id).toBe("a");
+    expect(out[1].entry.id).toBe("b");
+  });
+
+  it("filters out items LLM marks as irrelevant", async () => {
+    const results = [
+      makeResult("a", "TypeScript project setup", 0.9),
+      makeResult("b", "Favorite pizza recipe", 0.85),
+      makeResult("c", "Docker deployment notes", 0.8),
+    ];
+    const llm = makeMockLLM([true, false, true]);
+    const out = await filterByRelevance(results, "project deployment", llm);
+    expect(out).toHaveLength(2);
+    expect(out[0].entry.id).toBe("a");
+    expect(out[1].entry.id).toBe("c");
+  });
+
+  it("drops items below minScoreForFilter without LLM call", async () => {
+    const results = [
+      makeResult("a", "High score item", 0.9),
+      makeResult("b", "Low score item", 0.3),
+      makeResult("c", "Very low score", 0.1),
+    ];
+    // LLM only sees 1 item (the one above threshold)
+    const llm = makeMockLLM([true]);
+    const out = await filterByRelevance(results, "test query", llm);
+    expect(out).toHaveLength(1);
+    expect(out[0].entry.id).toBe("a");
+  });
+
+  it("returns all candidates when all are below minScoreForFilter", async () => {
+    const results = [
+      makeResult("a", "Low score A", 0.3),
+      makeResult("b", "Low score B", 0.2),
+    ];
+    const llm = makeMockLLM([]);
+    const out = await filterByRelevance(results, "test query", llm);
+    expect(out).toHaveLength(0);
+  });
+
+  it("truncates to maxItems", async () => {
+    const results = Array.from({ length: 15 }, (_, i) =>
+      makeResult(`item-${i}`, `Memory item ${i}`, 0.9 - i * 0.01),
+    );
+    // LLM receives only 5 items, marks all as relevant
+    const cfg: Partial<FilterConfig> = { maxItems: 5 };
+    const llm = makeMockLLM([true, true, true, true, true]);
+    const out = await filterByRelevance(results, "test query", llm, cfg);
+    expect(out).toHaveLength(5);
+    expect(out[0].entry.id).toBe("item-0");
+    expect(out[4].entry.id).toBe("item-4");
+  });
+
+  it("returns candidates on LLM failure (graceful fallback)", async () => {
+    const results = [
+      makeResult("a", "Important memory", 0.9),
+      makeResult("b", "Another memory", 0.85),
+    ];
+    const llm = makeMockLLM(null, true);
+    const out = await filterByRelevance(results, "test query", llm);
+    // Should return all candidates, not lose data
+    expect(out).toHaveLength(2);
+    expect(out[0].entry.id).toBe("a");
+  });
+
+  it("returns candidates when LLM returns non-array", async () => {
+    const results = [
+      makeResult("a", "Important memory", 0.9),
+    ];
+    const llm = makeMockLLM(null);
+    const out = await filterByRelevance(results, "test query", llm);
+    expect(out).toHaveLength(1);
+  });
+
+  it("uses l0_abstract from metadata when available", async () => {
+    const result = makeResult("a", "Very long raw text that should not be used", 0.9);
+    result.entry.metadata = JSON.stringify({ l0_abstract: "Short summary" });
+    const calls: string[] = [];
+    const llm = {
+      chatJson: async (_sys: string, user: string) => {
+        calls.push(user);
+        return [true];
+      },
+    } as unknown as LLMClient;
+
+    await filterByRelevance([result], "test", llm);
+    expect(calls[0]).toContain("Short summary");
+    expect(calls[0]).not.toContain("Very long raw text");
+  });
+
+  it("handles custom minScoreForFilter", async () => {
+    const results = [
+      makeResult("a", "High", 0.9),
+      makeResult("b", "Medium", 0.6),
+      makeResult("c", "Low", 0.4),
+    ];
+    const cfg: Partial<FilterConfig> = { minScoreForFilter: 0.65 };
+    const llm = makeMockLLM([true]);
+    const out = await filterByRelevance(results, "test", llm, cfg);
+    expect(out).toHaveLength(1);
+    expect(out[0].entry.id).toBe("a");
+  });
+
+  it("handles shorter LLM verdict array than candidates", async () => {
+    const results = [
+      makeResult("a", "Memory A", 0.9),
+      makeResult("b", "Memory B", 0.85),
+      makeResult("c", "Memory C", 0.8),
+    ];
+    // LLM returns fewer verdicts than items
+    const llm = makeMockLLM([true, false]);
+    const out = await filterByRelevance(results, "test", llm);
+    // Only first item passes (index 0 = true, index 1 = false, index 2 = out of bounds)
+    expect(out).toHaveLength(1);
+    expect(out[0].entry.id).toBe("a");
+  });
+});

package/src/__tests__/prediction-engine.test.ts

@@ -0,0 +1,349 @@
+/**
+ * Tests for HP-predictive Phase 5: Prediction Engine
+ *
+ * Validates:
+ * 1. collectSignals() — each signal type is correctly extracted
+ * 2. scorePredictions() — scoring, filtering by threshold, ranking
+ * 3. Edge cases — empty inputs, boundary conditions
+ */
+import { describe, expect, it } from "bun:test";
+import {
+  collectSignals,
+  scorePredictions,
+  _testConstants,
+  type PredictionContext,
+  type PredictionSignal,
+} from "../prediction-engine.js";
+import type { SessionCheckpointRecord } from "../session-schema.js";
+import type { WorkflowObservationRecord } from "../workflow-observation-schema.js";
+
+const { CONFIDENCE_THRESHOLD, STALE_OPEN_LOOP_HOURS, STALE_MEMORY_DAYS, MAX_PREDICTIONS, SIGNAL_WEIGHTS } = _testConstants;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeCheckpoint(overrides: Partial<SessionCheckpointRecord> & { openLoops?: string[]; updatedAt: string }): SessionCheckpointRecord {
+  return {
+    checkpointId: `cp-${Math.random().toString(36).slice(2, 8)}`,
+    sessionId: "test-session",
+    resolvedScope: "test",
+    scope: "test",
+    summary: "test checkpoint",
+    task: "test task",
+    decisions: [],
+    openLoops: overrides.openLoops ?? [],
+    nextActions: [],
+    entities: [],
+    files: [],
+    updatedAt: overrides.updatedAt,
+    ...overrides,
+  };
+}
+
+function makeWorkflowObservation(overrides: Partial<WorkflowObservationRecord>): WorkflowObservationRecord {
+  return {
+    observationId: `obs-${Math.random().toString(36).slice(2, 8)}`,
+    workflowId: "test-workflow",
+    outcome: "corrected",
+    summary: "Fixed a workflow issue",
+    scope: "test",
+    resolvedScope: "test",
+    source: "manual",
+    tags: [],
+    tools: [],
+    recordedAt: new Date().toISOString(),
+    ...overrides,
+  };
+}
+
+function emptyContext(overrides?: Partial<PredictionContext>): PredictionContext {
+  return {
+    checkpoints: [],
+    workflowObservations: [],
+    frequentMemories: [],
+    uncoveredTopics: [],
+    now: new Date("2026-04-11T12:00:00Z"),
+    ...overrides,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// collectSignals
+// ---------------------------------------------------------------------------
+
+describe("collectSignals", () => {
+  describe("stale open loop signals", () => {
+    it("detects stale open loops from checkpoints older than 24h", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const staleCheckpoint = makeCheckpoint({
+        openLoops: ["Fix CI pipeline", "Review PR #42"],
+        updatedAt: new Date(now.getTime() - 48 * 60 * 60 * 1000).toISOString(), // 48h ago
+      });
+
+      const signals = collectSignals(emptyContext({
+        checkpoints: [staleCheckpoint],
+        now,
+      }));
+
+      const openLoopSignals = signals.filter(s => s.type === "stale_open_loop");
+      expect(openLoopSignals.length).toBe(2);
+      expect(openLoopSignals[0].trigger).toBe("Fix CI pipeline");
+      expect(openLoopSignals[0].weight).toBeGreaterThan(0.6);
+      expect(openLoopSignals[0].evidence.length).toBeGreaterThan(0);
+    });
+
+    it("ignores fresh open loops (< 24h)", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const freshCheckpoint = makeCheckpoint({
+        openLoops: ["In-progress task"],
+        updatedAt: new Date(now.getTime() - 12 * 60 * 60 * 1000).toISOString(), // 12h ago
+      });
+
+      const signals = collectSignals(emptyContext({
+        checkpoints: [freshCheckpoint],
+        now,
+      }));
+
+      expect(signals.filter(s => s.type === "stale_open_loop").length).toBe(0);
+    });
+
+    it("deduplicates same open loop across checkpoints", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const cp1 = makeCheckpoint({
+        openLoops: ["Fix CI pipeline"],
+        updatedAt: new Date(now.getTime() - 48 * 60 * 60 * 1000).toISOString(),
+      });
+      const cp2 = makeCheckpoint({
+        openLoops: ["fix ci pipeline"], // same but different case
+        updatedAt: new Date(now.getTime() - 72 * 60 * 60 * 1000).toISOString(),
+      });
+
+      const signals = collectSignals(emptyContext({
+        checkpoints: [cp1, cp2],
+        now,
+      }));
+
+      expect(signals.filter(s => s.type === "stale_open_loop").length).toBe(1);
+    });
+
+    it("weights increase with staleness", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const cp24h = makeCheckpoint({
+        openLoops: ["Task A"],
+        updatedAt: new Date(now.getTime() - 25 * 60 * 60 * 1000).toISOString(),
+      });
+      const cp72h = makeCheckpoint({
+        openLoops: ["Task B"],
+        updatedAt: new Date(now.getTime() - 72 * 60 * 60 * 1000).toISOString(),
+      });
+
+      const signals = collectSignals(emptyContext({
+        checkpoints: [cp24h, cp72h],
+        now,
+      }));
+
+      const signalA = signals.find(s => s.trigger === "Task A")!;
+      const signalB = signals.find(s => s.trigger === "Task B")!;
+      expect(signalB.weight).toBeGreaterThan(signalA.weight);
+    });
+  });
+
+  describe("stale high-frequency memory signals", () => {
+    it("detects memories accessed frequently but not recently", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const signals = collectSignals(emptyContext({
+        frequentMemories: [{
+          text: "Auth migration plan",
+          topicTag: "auth",
+          lastAccessedAt: new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString(), // 30 days ago
+          accessCount: 5,
+        }],
+        now,
+      }));
+
+      const staleSignals = signals.filter(s => s.type === "stale_high_frequency");
+      expect(staleSignals.length).toBe(1);
+      expect(staleSignals[0].trigger).toBe("auth");
+      expect(staleSignals[0].evidence).toContain("Accessed 5 times but not in 30 days");
+    });
+
+    it("ignores recently accessed memories", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const signals = collectSignals(emptyContext({
+        frequentMemories: [{
+          text: "Recent memory",
+          lastAccessedAt: new Date(now.getTime() - 3 * 24 * 60 * 60 * 1000).toISOString(), // 3 days ago
+          accessCount: 10,
+        }],
+        now,
+      }));
+
+      expect(signals.filter(s => s.type === "stale_high_frequency").length).toBe(0);
+    });
+
+    it("ignores infrequently accessed memories (< 3 accesses)", () => {
+      const now = new Date("2026-04-11T12:00:00Z");
+      const signals = collectSignals(emptyContext({
+        frequentMemories: [{
+          text: "Rarely accessed",
+          lastAccessedAt: new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000).toISOString(),
+          accessCount: 2,
+        }],
+        now,
+      }));
+
+      expect(signals.filter(s => s.type === "stale_high_frequency").length).toBe(0);
+    });
+  });
+
+  describe("workflow issue signals", () => {
+    it("detects corrected/missed workflow observations", () => {
+      const observations = [
+        makeWorkflowObservation({ workflowId: "deploy", outcome: "corrected", summary: "Forgot to run tests" }),
+        makeWorkflowObservation({ workflowId: "deploy", outcome: "missed", summary: "Missed the changelog update" }),
+        makeWorkflowObservation({ workflowId: "deploy", outcome: "success", summary: "All good" }),
+      ];
+
+      const signals = collectSignals(emptyContext({ workflowObservations: observations }));
+      const issueSignals = signals.filter(s => s.type === "workflow_issue");
+      expect(issueSignals.length).toBe(1);
+      expect(issueSignals[0].trigger).toBe("deploy");
+      expect(issueSignals[0].evidence[0]).toContain("2 corrected/missed");
+    });
+
+    it("ignores workflows with only successes", () => {
+      const observations = [
+        makeWorkflowObservation({ workflowId: "deploy", outcome: "success" }),
+      ];
+
+      const signals = collectSignals(emptyContext({ workflowObservations: observations }));
+      expect(signals.filter(s => s.type === "workflow_issue").length).toBe(0);
+    });
+  });
+
+  describe("uncovered topic signals", () => {
+    it("creates signals for uncovered topics", () => {
+      const signals = collectSignals(emptyContext({
+        uncoveredTopics: ["kubernetes", "migration"],
+      }));
+
+      const topicSignals = signals.filter(s => s.type === "uncovered_topic");
+      expect(topicSignals.length).toBe(2);
+      expect(topicSignals[0].trigger).toBe("kubernetes");
+      expect(topicSignals[0].weight).toBe(0.65);
+    });
+  });
+
+  describe("empty inputs", () => {
+    it("returns empty array for empty context", () => {
+      const signals = collectSignals(emptyContext());
+      expect(signals).toEqual([]);
+    });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// scorePredictions
+// ---------------------------------------------------------------------------
+
+describe("scorePredictions", () => {
+  it("scores signals and filters by confidence threshold", () => {
+    const signals: PredictionSignal[] = [
+      {
+        type: "stale_open_loop",
+        trigger: "Fix CI",
+        action: "Unresolved: Fix CI",
+        weight: 0.9,
+        evidence: ["48h old"],
+      },
+      {
+        type: "uncovered_topic",
+        trigger: "obscure",
+        action: "No memories for obscure",
+        weight: 0.5, // Below threshold after scoring
+        evidence: ["No coverage"],
+      },
+    ];
+
+    const predictions = scorePredictions(signals);
+
+    // stale_open_loop: 0.9 * 0.9 = 0.81 (above 0.6)
+    // uncovered_topic: 0.5 * 0.65 = 0.325 (below 0.6)
+    expect(predictions.length).toBe(1);
+    expect(predictions[0].trigger).toBe("Fix CI");
+    expect(predictions[0].confidence).toBe(0.81);
+  });
+
+  it("sorts by confidence descending", () => {
+    const signals: PredictionSignal[] = [
+      { type: "uncovered_topic", trigger: "low", action: "low", weight: 1.0, evidence: [] },
+      { type: "stale_open_loop", trigger: "high", action: "high", weight: 1.0, evidence: [] },
+      { type: "workflow_issue", trigger: "mid", action: "mid", weight: 1.0, evidence: [] },
+    ];
+
+    const predictions = scorePredictions(signals);
+    // stale_open_loop: 1.0 * 0.9 = 0.9
+    // workflow_issue: 1.0 * 0.85 = 0.85
+    // uncovered_topic: 1.0 * 0.65 = 0.65
+    expect(predictions.length).toBe(3);
+    expect(predictions[0].trigger).toBe("high");
+    expect(predictions[1].trigger).toBe("mid");
+    expect(predictions[2].trigger).toBe("low");
+  });
+
+  it("limits to MAX_PREDICTIONS", () => {
+    const signals: PredictionSignal[] = Array.from({ length: 10 }, (_, i) => ({
+      type: "stale_open_loop" as const,
+      trigger: `task-${i}`,
+      action: `action-${i}`,
+      weight: 1.0,
+      evidence: [],
+    }));
+
+    const predictions = scorePredictions(signals);
+    expect(predictions.length).toBe(MAX_PREDICTIONS);
+  });
+
+  it("returns empty for all below threshold", () => {
+    const signals: PredictionSignal[] = [
+      { type: "uncovered_topic", trigger: "x", action: "x", weight: 0.3, evidence: [] },
+    ];
+
+    const predictions = scorePredictions(signals);
+    expect(predictions.length).toBe(0);
+  });
+
+  it("returns empty for empty signals", () => {
+    expect(scorePredictions([])).toEqual([]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Constants sanity
+// ---------------------------------------------------------------------------
+
+describe("prediction engine constants", () => {
+  it("confidence threshold is 0.6", () => {
+    expect(CONFIDENCE_THRESHOLD).toBe(0.6);
+  });
+
+  it("stale open loop hours is 24", () => {
+    expect(STALE_OPEN_LOOP_HOURS).toBe(24);
+  });
+
+  it("stale memory days is 14", () => {
+    expect(STALE_MEMORY_DAYS).toBe(14);
+  });
+
+  it("max predictions is 5", () => {
+    expect(MAX_PREDICTIONS).toBe(5);
+  });
+
+  it("signal weights are all between 0 and 1", () => {
+    for (const [, weight] of Object.entries(SIGNAL_WEIGHTS)) {
+      expect(weight).toBeGreaterThanOrEqual(0);
+      expect(weight).toBeLessThanOrEqual(1);
+    }
+  });
+});