recallnest 2.3.0 → 2.5.4

package/src/__tests__/noise-filter.test.ts

@@ -0,0 +1,205 @@
+import { describe, expect, it } from "bun:test";
+
+import { isNoise } from "../noise-filter.js";
+
+// Real samples drawn from production cc: scopes — CC bridge injects these system
+// prompts as `user` messages, which transcript ingest then mis-extracts as
+// profile/preferences. They are CC's own instructions, never user content.
+const CC_REPLY_PROMPT = `# CC Reply Prompt
+
+你是 CC。Alice 刚给你发了消息。
+
+你不是 Alice 的数字分身，也不是 E 化镜像。不要模仿她的公众号语料、身份、口头禅或写作人格。你要以 CC 自己的主体感回复她：清楚、稳、可靠，有温度但不油。
+
+## 当前 context
+
+\`\`\`json
+{ "time_now": "2026-05-28T11:11:37.034Z", "hour_of_day": 19 }
+\`\`\`
+
+## 你的输出
+
+直接回复 Alice。`;
+
+const CC_SELF_DECISION_PROMPT = `# CC Self-Decision Prompt
+
+你是 CC。现在你周期性醒来一次，看当前状态，自己决定要不要 ping Alice。`;
+
+describe("isNoise — CC bridge injected prompts", () => {
+  it("filters the CC Reply Prompt template", () => {
+    expect(isNoise(CC_REPLY_PROMPT)).toBe(true);
+  });
+
+  it("filters the CC Self-Decision Prompt template", () => {
+    expect(isNoise(CC_SELF_DECISION_PROMPT)).toBe(true);
+  });
+
+  it("filters by the bridge persona signature alone", () => {
+    expect(isNoise("你不是 Alice 的数字分身，也不是 E 化镜像。要清楚、稳、可靠。")).toBe(true);
+  });
+
+  it("filters heading variants of the prompt title", () => {
+    expect(isNoise("## CC Reply Prompt\n\n你是 CC。")).toBe(true);
+  });
+
+  it("filters the bridge prompt output-section fragment", () => {
+    expect(isNoise("## 你的输出\n\n直接回复 Alice。用双换行把回复切成自然段落。")).toBe(true);
+  });
+
+  it("filters the bridge prompt rhythm-section fragment", () => {
+    expect(isNoise("节奏要求：\n\n- 短话题 1-2 条；\n- 长话题 3-5 条；\n- 每条一两句话；")).toBe(true);
+  });
+
+  // Guards: must NOT over-filter real conversation
+  it("keeps a real message that happens to use the words 节奏要求", () => {
+    expect(isNoise("这周节奏要求挺紧的，周五前要交稿，你帮我盯一下进度")).toBe(false);
+  });
+
+  it("keeps a normal message that merely mentions CC", () => {
+    expect(isNoise("CC 帮我看下这个 bug，promote-scan 跑出来全是噪声")).toBe(false);
+  });
+
+  it("keeps a normal technical message", () => {
+    expect(isNoise("我觉得用 Redis 做缓存这个方案不错，比 memcached 省事")).toBe(false);
+  });
+
+  it("keeps a message that starts with 你是 but is not a bridge prompt", () => {
+    expect(isNoise("你是我见过最靠谱的搭子，这事就交给你了")).toBe(false);
+  });
+
+  it("respects filterAgentPrompts=false opt-out", () => {
+    expect(isNoise(CC_REPLY_PROMPT, { filterAgentPrompts: false })).toBe(false);
+  });
+});
+
+// Real samples drawn from the promote-scan --min-occurrences 2 candidate dump:
+// long bridge self-decision / distill payloads embed structured JSON (decision
+// logs, conversation history, stats). Transcript chunking splits them, so a
+// JSON field-line survives as a standalone chunk after losing its { } wrapper —
+// it then looks like a preference. The defining trait is an orphaned field-line
+// (starts as "key":, no brace) or a severed structural token (leading , } ]).
+describe("isNoise — bridge/distill context JSON fragments", () => {
+  it("filters an orphan self-decision action field-line", () => {
+    expect(isNoise(`"action": "silent",`)).toBe(true);
+  });
+  it("filters an orphan reasoning field-line", () => {
+    expect(isNoise(`"reasoning": "现在是凌晨4点，在静默窗口内。她应该在睡觉。",`)).toBe(true);
+  });
+  it("filters an orphan conversation-history role field-line", () => {
+    expect(isNoise(`"role": "assistant",`)).toBe(true);
+  });
+  it("filters a field-line whose leading quote was severed by chunking", () => {
+    expect(isNoise(`content": "头发乱一点，比我那张更随意，有点文艺感。",`)).toBe(true);
+  });
+  it("filters an isolated ISO-timestamp field-line", () => {
+    expect(isNoise(`"time": "2026-05-21T15:42:03.796Z"`)).toBe(true);
+  });
+  it("filters the bridge proprietary next_check_hint key", () => {
+    expect(isNoise(`"next_check_hint": "tomorrow_morning"`)).toBe(true);
+  });
+  it("filters the distill proprietary distilled_at key", () => {
+    expect(isNoise(`"distilled_at": "2026-05-22T05:12:13.309Z"`)).toBe(true);
+  });
+  it("filters the bridge proprietary alice_interaction_stats key", () => {
+    expect(isNoise(`"alice_interaction_stats_7d": {`)).toBe(true);
+  });
+  it("filters a fragment that starts with a dangling comma (severed mid-object)", () => {
+    expect(
+      isNoise(`,\n  {\n    "role": "user",\n    "content": "对，就是看这个",\n    "time": "2026-05-22T12:46:19.937Z"`),
+    ).toBe(true);
+  });
+  it("filters a multi-line self-decision JSON block", () => {
+    expect(
+      isNoise(`"action": "silent",\n    "message": "",\n    "reasoning": "凌晨2点，静默窗口，她在睡觉。",\n    "next_check_hint": "tomorrow_morning"`),
+    ).toBe(true);
+  });
+
+  // Guards: must NOT over-filter real conversation or legitimate JSON discussion
+  it("keeps a real message discussing JSON inline", () => {
+    expect(isNoise(`这个配置 {"action": "silent"} 你看对不对？`)).toBe(false);
+  });
+  it("keeps a complete brace-wrapped JSON object as possible real discussion", () => {
+    expect(isNoise(`{"action": "silent", "reasoning": "test"}`)).toBe(false);
+  });
+  it("keeps natural language that merely mentions a json-ish word", () => {
+    expect(isNoise(`我觉得 time 管理这块你得帮我盯紧一点`)).toBe(false);
+  });
+  it("keeps a message that opens with an English quoted phrase", () => {
+    expect(isNoise(`"hello world" 是我们的开场白，你记一下`)).toBe(false);
+  });
+  it("keeps a real reply that opens with a Chinese quote", () => {
+    expect(isNoise(`"去上班，乖" 这句你昨天说过，我记得`)).toBe(false);
+  });
+
+  // Guards against over-filtering real code discussion — the cc bot reviews
+  // code, so split chunks routinely open with closing braces/brackets.
+  it("keeps a code fragment opening with a closing brace", () => {
+    expect(isNoise(`} catch (e) {\n  console.error(e);\n}`)).toBe(false);
+  });
+  it("keeps a code fragment opening with });", () => {
+    expect(isNoise(`});\n\nfunction next() { return 1; }`)).toBe(false);
+  });
+  it("keeps a code fragment opening with a closing bracket", () => {
+    expect(isNoise(`]\n  return result;`)).toBe(false);
+  });
+  it("keeps an unquoted-key object/type line (no JSON string key)", () => {
+    expect(isNoise(`name: string; // 字段定义`)).toBe(false);
+  });
+  it("keeps a diff hunk that opens with a closing brace", () => {
+    expect(isNoise(`}\n\n=== DIFF: src/cli.ts ===\n@@ -1361,8 +1361,10 @@`)).toBe(false);
+  });
+
+  // Guards from Codex review: real memories that DISCUSS RecallNest's own JSON
+  // schema fields inline, or paste config with object-valued keys, must survive.
+  it("keeps a real memory that mentions a proprietary field name inline", () => {
+    expect(isNoise(`我们刚才讨论 "distilled_at": 这个字段要不要保留`)).toBe(false);
+  });
+  it("keeps a config chunk opening with an object-valued key", () => {
+    expect(isNoise(`"scripts": {`)).toBe(false);
+  });
+  it("still filters an orphan line that LEADS with a proprietary key", () => {
+    expect(isNoise(`"next_check_hint": "tomorrow_morning"`)).toBe(true);
+  });
+});
+
+// Regression (2026-07-10): the English greeting pattern lacked a word boundary,
+// so `^hi` matched any text starting with "hi…" — hippo-wiki, history, highlight,
+// hidden state, Hippocampus. Alice's core project is named hippo; every memory
+// opening with it was silently rejected. Fixed by anchoring with \b.
+describe("isNoise — greeting word boundary regression", () => {
+  it("keeps text starting with hippo-wiki", () => {
+    expect(isNoise("hippo-wiki 的索引层已清零，8 个现役工具正文补齐")).toBe(false);
+  });
+  it("keeps text starting with history", () => {
+    expect(isNoise("history 记录显示这个 commit 有问题，需要 revert")).toBe(false);
+  });
+  it("keeps text starting with highlight", () => {
+    expect(isNoise("highlight 一下：这里有个 bug，修复方案是加词边界")).toBe(false);
+  });
+  it("keeps text starting with hidden state", () => {
+    expect(isNoise("hidden state 是 agent 的核心，不能只看输出")).toBe(false);
+  });
+  it("keeps text starting with Hippocampus", () => {
+    expect(isNoise("Hippocampus 海马体的记忆巩固机制值得借鉴")).toBe(false);
+  });
+  it("keeps text starting with heyday", () => {
+    expect(isNoise("heyday 时期的架构决策现在看仍然成立")).toBe(false);
+  });
+
+  // Real greetings must still be filtered after the fix
+  it("still filters hey there", () => {
+    expect(isNoise("hey there")).toBe(true);
+  });
+  it("still filters hello world", () => {
+    expect(isNoise("hello world")).toBe(true);
+  });
+  it("still filters greetings from mini", () => {
+    expect(isNoise("greetings from mini")).toBe(true);
+  });
+  it("still filters hi followed by a comma", () => {
+    expect(isNoise("hi, 能帮我看下吗")).toBe(true);
+  });
+  it("still filters good morning with trailing words", () => {
+    expect(isNoise("good morning everyone")).toBe(true);
+  });
+});

package/src/__tests__/obsidian-connector.test.ts

@@ -0,0 +1,363 @@
+import { describe, expect, it, beforeAll, afterAll } from "bun:test";
+import { mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+import {
+  isObsidianVault,
+  parseFrontmatter,
+  extractWikiLinks,
+  contentHash,
+  scanVault,
+} from "../obsidian-connector.js";
+
+// ---------------------------------------------------------------------------
+// Temp vault fixture
+// ---------------------------------------------------------------------------
+
+const VAULT_ROOT = join(tmpdir(), `recallnest-obsidian-test-${Date.now()}`);
+
+beforeAll(() => {
+  // Create vault structure
+  mkdirSync(join(VAULT_ROOT, ".obsidian"), { recursive: true });
+  mkdirSync(join(VAULT_ROOT, "projects"), { recursive: true });
+  mkdirSync(join(VAULT_ROOT, "daily"), { recursive: true });
+  mkdirSync(join(VAULT_ROOT, ".trash"), { recursive: true });
+
+  // Root-level note with frontmatter + wikilinks
+  writeFileSync(
+    join(VAULT_ROOT, "My Note.md"),
+    `---
+tags: [ai, memory]
+aliases: [note-1]
+status: active
+---
+
+# My Note
+
+This is a note about [[RecallNest]] and [[LanceDB|vector database]].
+
+Some more content here to make it long enough.
+`,
+  );
+
+  // Nested note in projects/
+  writeFileSync(
+    join(VAULT_ROOT, "projects", "Project Alpha.md"),
+    `---
+tags:
+  - project
+  - alpha
+priority: high
+---
+
+# Project Alpha
+
+Working on [[Feature X]] for [[Project Alpha]].
+
+Details about the project implementation and design decisions.
+`,
+  );
+
+  // Daily note without frontmatter
+  writeFileSync(
+    join(VAULT_ROOT, "daily", "2026-04-13.md"),
+    `# 2026-04-13
+
+- Met with team about [[Q2 Goals]]
+- Reviewed [[PR #123]]
+- Need to follow up on [[Budget Review]]
+
+Some additional notes about the day and what happened.
+`,
+  );
+
+  // Short note (should be skipped)
+  writeFileSync(join(VAULT_ROOT, "short.md"), "Hi");
+
+  // Trash note (should be skipped)
+  writeFileSync(
+    join(VAULT_ROOT, ".trash", "deleted.md"),
+    "This is deleted content that should not be ingested into the vault.\n",
+  );
+});
+
+afterAll(() => {
+  try {
+    rmSync(VAULT_ROOT, { recursive: true, force: true });
+  } catch {
+    // cleanup best-effort
+  }
+});
+
+// ---------------------------------------------------------------------------
+// isObsidianVault
+// ---------------------------------------------------------------------------
+
+describe("isObsidianVault", () => {
+  it("returns true for a directory with .obsidian/", () => {
+    expect(isObsidianVault(VAULT_ROOT)).toBe(true);
+  });
+
+  it("returns false for a regular directory", () => {
+    expect(isObsidianVault(join(VAULT_ROOT, "projects"))).toBe(false);
+  });
+
+  it("returns false for non-existent directory", () => {
+    expect(isObsidianVault("/nonexistent/path")).toBe(false);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// parseFrontmatter
+// ---------------------------------------------------------------------------
+
+describe("parseFrontmatter", () => {
+  it("parses inline list tags", () => {
+    const content = `---
+tags: [ai, memory]
+status: active
+---
+
+Body content here.`;
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter).not.toBeNull();
+    expect(frontmatter!.tags).toEqual(["ai", "memory"]);
+    expect(frontmatter!.status).toBe("active");
+    expect(body).toBe("Body content here.");
+  });
+
+  it("parses YAML list tags", () => {
+    const content = `---
+tags:
+  - project
+  - alpha
+priority: high
+---
+
+Body here.`;
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter).not.toBeNull();
+    expect(frontmatter!.tags).toEqual(["project", "alpha"]);
+    expect(frontmatter!.priority).toBe("high");
+    expect(body).toBe("Body here.");
+  });
+
+  it("returns null frontmatter when no --- block", () => {
+    const content = "# Just a heading\n\nSome content.";
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter).toBeNull();
+    expect(body).toBe(content);
+  });
+
+  it("handles empty frontmatter", () => {
+    const content = `---
+---
+
+Body.`;
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter).toBeNull();
+    expect(body).toBe("Body.");
+  });
+
+  it("handles quoted values", () => {
+    const content = `---
+title: 'My Title'
+author: "Alice"
+---
+
+Content.`;
+    const { frontmatter } = parseFrontmatter(content);
+    expect(frontmatter).not.toBeNull();
+    expect(frontmatter!.title).toBe("My Title");
+    expect(frontmatter!.author).toBe("Alice");
+  });
+
+  it("handles empty inline list []", () => {
+    const content = `---
+tags: []
+---
+
+Content here is long enough.`;
+    const { frontmatter } = parseFrontmatter(content);
+    // Empty list should not produce tags
+    expect(frontmatter).toBeNull();
+  });
+
+  it("handles Windows-style line endings", () => {
+    const content = "---\r\ntags: [a, b]\r\n---\r\n\r\nBody.";
+    const { frontmatter, body } = parseFrontmatter(content);
+    expect(frontmatter).not.toBeNull();
+    expect(frontmatter!.tags).toEqual(["a", "b"]);
+    expect(body).toBe("Body.");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// extractWikiLinks
+// ---------------------------------------------------------------------------
+
+describe("extractWikiLinks", () => {
+  it("extracts simple wikilinks", () => {
+    const links = extractWikiLinks("Link to [[RecallNest]] and [[LanceDB]].");
+    expect(links).toHaveLength(2);
+    expect(links[0].target).toBe("RecallNest");
+    expect(links[1].target).toBe("LanceDB");
+  });
+
+  it("extracts wikilinks with display text", () => {
+    const links = extractWikiLinks("See [[LanceDB|vector database]] for details.");
+    expect(links).toHaveLength(1);
+    expect(links[0].target).toBe("LanceDB");
+    expect(links[0].displayText).toBe("vector database");
+  });
+
+  it("skips embed syntax ![[embed]]", () => {
+    const links = extractWikiLinks("Normal [[link]] and embed ![[image.png]].");
+    expect(links).toHaveLength(1);
+    expect(links[0].target).toBe("link");
+  });
+
+  it("handles multiple links on same line", () => {
+    const links = extractWikiLinks("[[A]] connects to [[B]] and [[C]]");
+    expect(links).toHaveLength(3);
+  });
+
+  it("returns empty array for no links", () => {
+    const links = extractWikiLinks("Just plain text.");
+    expect(links).toHaveLength(0);
+  });
+
+  it("trims whitespace from targets", () => {
+    const links = extractWikiLinks("[[ Spaced Target ]]");
+    expect(links[0].target).toBe("Spaced Target");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// contentHash
+// ---------------------------------------------------------------------------
+
+describe("contentHash", () => {
+  it("returns a 16-char hex string", () => {
+    const hash = contentHash("hello world");
+    expect(hash).toHaveLength(16);
+    expect(/^[0-9a-f]{16}$/.test(hash)).toBe(true);
+  });
+
+  it("is deterministic", () => {
+    expect(contentHash("same input")).toBe(contentHash("same input"));
+  });
+
+  it("differs for different inputs", () => {
+    expect(contentHash("input A")).not.toBe(contentHash("input B"));
+  });
+});
+
+// ---------------------------------------------------------------------------
+// scanVault
+// ---------------------------------------------------------------------------
+
+describe("scanVault", () => {
+  it("produces valid connector-v1 output", () => {
+    const output = scanVault(VAULT_ROOT);
+    expect(output.version).toBe("connector-v1");
+    expect(output.source).toBe("obsidian");
+    expect(output.scope).toContain("vault:");
+    expect(Array.isArray(output.records)).toBe(true);
+  });
+
+  it("finds all non-short, non-excluded .md files", () => {
+    const output = scanVault(VAULT_ROOT);
+    // Should find: My Note.md, projects/Project Alpha.md, daily/2026-04-13.md
+    // Should NOT find: short.md (too short), .trash/deleted.md (excluded)
+    expect(output.records).toHaveLength(3);
+  });
+
+  it("extracts frontmatter tags", () => {
+    const output = scanVault(VAULT_ROOT);
+    const myNote = output.records.find((r) => r.id === "obsidian:My Note.md");
+    expect(myNote).toBeDefined();
+    expect(myNote!.tags).toContain("ai");
+    expect(myNote!.tags).toContain("memory");
+  });
+
+  it("extracts wikilinks as link: tags", () => {
+    const output = scanVault(VAULT_ROOT);
+    const myNote = output.records.find((r) => r.id === "obsidian:My Note.md");
+    expect(myNote).toBeDefined();
+    expect(myNote!.tags).toContain("link:RecallNest");
+    expect(myNote!.tags).toContain("link:LanceDB");
+  });
+
+  it("maps folder structure to folder: tags", () => {
+    const output = scanVault(VAULT_ROOT);
+    const projectNote = output.records.find((r) => r.id?.includes("projects/"));
+    expect(projectNote).toBeDefined();
+    expect(projectNote!.tags).toContain("folder:projects");
+  });
+
+  it("does not add folder: tag for root-level files", () => {
+    const output = scanVault(VAULT_ROOT);
+    const myNote = output.records.find((r) => r.id === "obsidian:My Note.md");
+    expect(myNote).toBeDefined();
+    const folderTags = myNote!.tags!.filter((t) => t.startsWith("folder:"));
+    expect(folderTags).toHaveLength(0);
+  });
+
+  it("skips .trash and .obsidian directories", () => {
+    const output = scanVault(VAULT_ROOT);
+    const trashNote = output.records.find((r) => r.id?.includes(".trash"));
+    expect(trashNote).toBeUndefined();
+  });
+
+  it("computes contentHash for each record", () => {
+    const output = scanVault(VAULT_ROOT);
+    for (const record of output.records) {
+      expect(record.contentHash).toBeDefined();
+      expect(record.contentHash!).toHaveLength(16);
+    }
+  });
+
+  it("includes sourceMetadata with vault info", () => {
+    const output = scanVault(VAULT_ROOT);
+    const myNote = output.records.find((r) => r.id === "obsidian:My Note.md");
+    expect(myNote!.sourceMetadata).toBeDefined();
+    expect(myNote!.sourceMetadata!.vaultPath).toBe(VAULT_ROOT);
+    expect(myNote!.sourceMetadata!.relativePath).toBe("My Note.md");
+    expect(myNote!.sourceMetadata!.frontmatter).toBeDefined();
+    expect(myNote!.sourceMetadata!.wikiLinks).toBeDefined();
+  });
+
+  it("respects custom scopePrefix", () => {
+    const output = scanVault(VAULT_ROOT, { scopePrefix: "custom:my-vault" });
+    expect(output.scope).toBe("custom:my-vault");
+  });
+
+  it("respects custom excludeDirs", () => {
+    // Exclude 'daily' directory
+    const output = scanVault(VAULT_ROOT, {
+      excludeDirs: [".obsidian", ".trash", ".git", "node_modules", "daily"],
+    });
+    const dailyNote = output.records.find((r) => r.id?.includes("daily/"));
+    expect(dailyNote).toBeUndefined();
+    // Should still find the other 2
+    expect(output.records).toHaveLength(2);
+  });
+
+  it("strips frontmatter from body text", () => {
+    const output = scanVault(VAULT_ROOT);
+    const myNote = output.records.find((r) => r.id === "obsidian:My Note.md");
+    expect(myNote!.text).not.toContain("---");
+    expect(myNote!.text).toContain("# My Note");
+  });
+
+  it("handles note without frontmatter", () => {
+    const output = scanVault(VAULT_ROOT);
+    const dailyNote = output.records.find((r) => r.id?.includes("2026-04-13"));
+    expect(dailyNote).toBeDefined();
+    // Should still extract wikilinks
+    expect(dailyNote!.tags).toContain("link:Q2 Goals");
+    expect(dailyNote!.tags).toContain("link:PR #123");
+  });
+});

package/src/__tests__/omitted-reporting.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, it } from "bun:test";
+
+import { createRetriever, type RetrievalResultSet } from "../retriever.js";
+
+// P-omitted: 验证召回因 limit 上限截断时，对调用方暴露 omitted（提高 limit 可见）。
+// 借鉴 RepoPrompt CE「剪枝必回报」——召回从黑箱变可协商。
+
+function buildCandidate(id: string, score: number) {
+  return {
+    entry: {
+      id,
+      text: `distinct memory content number ${id}`,
+      vector: [1, 0, 0],
+      category: "preferences",
+      scope: "memory:agent",
+      importance: 0.8,
+      timestamp: Date.parse("2026-03-16T00:00:00.000Z"),
+      metadata: JSON.stringify({}),
+    },
+    score,
+  };
+}
+
+// 最小过滤 config：mode=vector 跳过 BM25/rerank；minScore/hardMinScore=0 + 高分候选避免 hard-min 截断；
+// filterNoise=false；RIF/sourceDiversity 字段缺省即不启用；MMR 只重排不删。
+// 于是过滤链净效果 = sort → slice(limit)，omitted = candidates - limit 可精确断言。
+function makeRetriever(
+  candidates: ReturnType<typeof buildCandidate>[],
+  configOverride: Record<string, unknown> = {},
+) {
+  return createRetriever({
+    hasFtsSupport: false,
+    // 模拟真实 MemoryStore.vectorSearch：尊重 k 参数（按 k 截断返回）。
+    // 这样测试能真实反映 over-fetch 是否生效——若 vector-only 不 over-fetch、只取 limit 条，
+    // 候选池 ≤ limit、omitted 必为 0，下面断言会失败（Codex review 暴露的假绿点）。
+    async vectorSearch(_vec: number[], k: number) { return candidates.slice(0, k); },
+  } as any, {
+    async embedQuery() { return [1, 0, 0]; },
+    async embedPassage() { return [1, 0, 0]; },
+  } as any, {
+    mode: "vector",
+    rerank: "none",
+    filterNoise: false,
+    hardMinScore: 0,
+    minScore: 0,
+    recencyWeight: 0,
+    timeDecayHalfLifeDays: 0,
+    ...configOverride,
+  });
+}
+
+describe("retrieval omitted reporting (P-omitted)", () => {
+  it("reports omitted count + reason when candidates exceed limit", async () => {
+    const candidates = Array.from({ length: 8 }, (_, i) => buildCandidate(`m${i}`, 0.9 - i * 0.01));
+    const retriever = makeRetriever(candidates);
+
+    const results = await retriever.retrieve({ query: "user reply style preference detail", limit: 3, category: "preferences" }) as RetrievalResultSet;
+
+    expect(results).toHaveLength(3);
+    expect(results.omitted).toBeDefined();
+    expect(results.omitted?.count).toBe(5); // 8 candidates - limit 3
+    expect(results.omitted?.reason).toBe("limit");
+  });
+
+  it("leaves omitted undefined when candidates fit within limit", async () => {
+    const candidates = Array.from({ length: 2 }, (_, i) => buildCandidate(`m${i}`, 0.9 - i * 0.01));
+    const retriever = makeRetriever(candidates);
+
+    const results = await retriever.retrieve({ query: "user reply style preference detail", limit: 5, category: "preferences" }) as RetrievalResultSet;
+
+    expect(results).toHaveLength(2);
+    expect(results.omitted).toBeUndefined();
+  });
+
+  it("persists omitted through the validity filter (regression: array is rebuilt downstream)", async () => {
+    // retrieve() runs filterByValidity which returns a NEW array, dropping any
+    // property hung on the result array. omitted must survive via the resultSet snapshot.
+    const candidates = Array.from({ length: 6 }, (_, i) => buildCandidate(`m${i}`, 0.9 - i * 0.01));
+    const retriever = makeRetriever(candidates);
+
+    const results = await retriever.retrieve({ query: "user reply style preference detail", limit: 2, category: "preferences" }) as RetrievalResultSet;
+
+    expect(results).toHaveLength(2);
+    expect(results.omitted?.count).toBe(4); // 6 - 2
+  });
+
+  it("reports omitted even when sourceDiversity caps the diversified pool (P3-1)", async () => {
+    // 启用 sourceDiversity 时 applySourceDiversity 先把 diversified 压到 limit；
+    // omitted 必须从 source-diversity 之前的候选池(clusterDeduped)算，否则恒为 0（Codex 复审 P3-1）。
+    const candidates = Array.from({ length: 8 }, (_, i) => buildCandidate(`m${i}`, 0.9 - i * 0.01));
+    const retriever = makeRetriever(candidates, { sourceDiversity: 0.5 });
+
+    const results = await retriever.retrieve({ query: "user reply style preference detail", limit: 3, category: "preferences" }) as RetrievalResultSet;
+
+    expect(results).toHaveLength(3);
+    expect(results.omitted?.count).toBe(5); // 8 - 3，即使 source diversity 压缩了 diversified
+  });
+});