recallnest 2.2.1 → 2.5.4

package/src/error-signature.ts

@@ -0,0 +1,153 @@
+/**
+ * A1 · Error-signature extractor — zero-LLM 结构化错误指纹（debug case 精确召回用）。
+ *
+ * 借鉴 MemOS local-plugin `core/capture/error-signature.ts`（repo-insight 审计 2026-06-13），
+ * 针对 RecallNest 本地化适配（Codex read-only 验证指出的三处不适配）：
+ *   - case 没有 toolCalls，错误指纹藏在 problem/context 自由文本里（MemOS 面向 tool output）
+ *   - 补裸 `X not found` / `No such file or directory` / 中文错误模式（MemOS 原 6 正则抽不到
+ *     裸的 `xmlsec1 not found`，也会被英文 alpha 检查滤掉中文）
+ *   - normaliseFragment 放宽长度/字母检查以保留中文短片段
+ *
+ * **读写对称**：纯函数。写入端（persistCaseMemory）抽指纹存 metadata.error_signature；
+ * 未来 P3-A 检索精确召回通道复用同一个 extractErrorSignatures，保证读写抽取规则一致。
+ * 零 LLM、确定性——每次 case 写入都跑，必须便宜。
+ */
+
+/** 每条 case 最多保留的指纹数（控制 hot-path 边界）。 */
+export const MAX_SIGNATURES = 4;
+const MIN_FRAGMENT_LEN = 6;
+const MAX_FRAGMENT_LEN = 160;
+
+/**
+ * verbatim 抽取模式——顺序即优先级，整段 match（m[0]）作为 fragment。
+ */
+const ERROR_PATTERNS: RegExp[] = [
+  // <Name>Error: / <Name>Exception: body
+  /\b([A-Z][A-Za-z0-9]*(?:Error|Exception)):\s*([^\n]{4,160})/g,
+  // error: / fatal: / ERROR: body
+  /\b(?:error|Error|fatal|FATAL|ERROR)\s*:\s*([^\n]{4,160})/g,
+  // <cmd>: ... not found / no such file / permission denied（带冒号）
+  /\b([A-Za-z0-9_\-./]+):\s*[^\n]{0,40}\b(not found|no such (?:file|directory)|permission denied|undefined reference|command not found)\b[^\n]*/g,
+  // 裸 <thing> [command] not found（无冒号——MemOS 原版抽不到 "xmlsec1 not found"）
+  /\b([A-Za-z0-9_.\-/]{2,60})\s+(?:command )?not found\b/gi,
+  // No such file or directory（独立出现）
+  /\bno such file or directory\b/gi,
+  // <thing> is required / must be / cannot / could not / failed to
+  /\b([A-Za-z0-9_]{3,40})\s+(is required|must be|cannot|could not|failed to)\s+[^\n]{3,120}/g,
+  // exit code / status N
+  /\bexit (?:code|status)\s*[:=]?\s*(\d{1,4})\b[^\n]{0,80}/g,
+  // HTTP 4xx/5xx + 标准状态短语（收紧：避免 "500 ms" / "404 rows" 这类裸数字误抽）
+  /\b(4\d\d|5\d\d)\s+(Bad Request|Unauthorized|Payment Required|Forbidden|Not Found|Method Not Allowed|Not Acceptable|Request Timeout|Conflict|Gone|Payload Too Large|Unprocessable Entity|Too Many Requests|Internal Server Error|Internal|Not Implemented|Bad Gateway|Service Unavailable|Gateway Timeout)\b/g,
+  // 中文错误现象 + 后续短语
+  /(找不到|没有找到|不存在|无法|没有权限|权限被拒|拒绝访问|命令不存在|没有那个文件|连接失败|连接超时|超时|崩溃|段错误|空指针|未定义|报错|异常|失败)[^\n]{0,24}/g,
+  // 中文 退出码/状态码/错误码/返回码 N
+  /(退出码|状态码|错误码|返回码)\s*[:=：]?\s*\d{1,4}/g,
+];
+
+const STOP_WORDS = new Set([
+  "the", "for", "this", "that", "your", "from", "with", "have", "has",
+  "not", "a", "an", "of", "to", "is", "in", "on", "by", "was", "were",
+]);
+
+/**
+ * 过泛中文错误词——单独出现（裸词 + 语气助词、无具体锚点）会在检索端过度匹配。
+ * 具体短语（找不到 / 权限被拒 / 连接失败 / 段错误 / 空指针）不在此列、照常保留。
+ * 已有 cjk<3 长度门挡掉 2 字裸词；本表治的是 3+ 字「裸词+助词」（失败了 / 超时了）。
+ */
+const GENERIC_CN_ERROR = /^(?:报错|出错|错误|异常|失败|超时|崩溃|无法|未定义|不能)[了的过着吧呢啊啦哦呀嘛。，！？!?,\s]*$/;
+
+/** 失败信号词——outcome/solutionSteps 仅在命中时才纳入语料（避免成功态反向噪声）。 */
+const FAILURE_HINT = /(error|fail|failed|failure|exception|not found|cannot|denied|timeout|拒绝|失败|报错|错误|异常|不能|无法|超时|崩溃)/i;
+
+export interface ExtractInput {
+  /** 主信号：case 的 problem 字段（"What problem happened"）。 */
+  problem: string;
+  /** case 的 context（原始 stderr / 触发条件常在此）。 */
+  context?: string;
+  /** case 的 outcome（解决后状态——偏成功态，仅含失败词时参与）。 */
+  outcome?: string;
+  /** solution 步骤（修复命令噪声多，仅含失败词时参与）。 */
+  solutionSteps?: string[];
+}
+
+/**
+ * 抽取至多 MAX_SIGNATURES 条规范化错误指纹，按 specificity 降序。
+ * problem + context 是主语料；outcome / solutionSteps 仅当包含失败词时纳入
+ * （outcome 多为成功态，否则会抽到 "exit code 0 / 不再报错" 这类反向信号）。
+ */
+export function extractErrorSignatures(input: ExtractInput): string[] {
+  const corpus: string[] = [input.problem];
+  if (input.context) corpus.push(input.context);
+  const tail = [input.outcome ?? "", ...(input.solutionSteps ?? [])].join("\n");
+  if (tail.trim() && FAILURE_HINT.test(tail)) corpus.push(tail);
+
+  // dedup by lowercased/collapsed key（重叠正则不产近似重复）；first-seen casing 胜出。
+  const candidates = new Map<string, { frag: string; freq: number }>();
+  for (const text of corpus) {
+    for (const frag of extractFragments(text)) {
+      const normalised = normaliseFragment(frag);
+      if (!normalised) continue;
+      const key = normalised.toLowerCase().replace(/\s+/g, " ");
+      const existing = candidates.get(key);
+      if (existing) existing.freq++;
+      else candidates.set(key, { frag: normalised, freq: 1 });
+    }
+  }
+
+  const scored = Array.from(candidates.values()).map(({ frag, freq }) => ({
+    frag,
+    score: specificityScore(frag, freq),
+  }));
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, MAX_SIGNATURES).map((s) => s.frag);
+}
+
+function extractFragments(text: string): string[] {
+  if (!text) return [];
+  const out: string[] = [];
+  for (const pattern of ERROR_PATTERNS) {
+    pattern.lastIndex = 0;
+    let m: RegExpExecArray | null;
+    while ((m = pattern.exec(text))) {
+      out.push(m[0]);
+      if (m.index === pattern.lastIndex) pattern.lastIndex++; // 防零宽匹配死循环
+      if (out.length >= 32) break; // 每个 pattern 硬上限
+    }
+  }
+  return out;
+}
+
+function normaliseFragment(frag: string): string | null {
+  const collapsed = frag.replace(/\s+/g, " ").trim();
+  const cjk = collapsed.replace(/[^一-鿿]/g, "");
+  // 长度门：英文片段 ≥MIN_FRAGMENT_LEN，或中文 ≥3 字（中文短片段如"找不到文件"也保留）
+  if (collapsed.length < MIN_FRAGMENT_LEN && cjk.length < 3) return null;
+  const truncated = collapsed.length > MAX_FRAGMENT_LEN ? collapsed.slice(0, MAX_FRAGMENT_LEN) : collapsed;
+  // 信息量门：英文字母 ≥4 或 中文 ≥2 字（MemOS 原版只看 alpha≥4，会滤掉纯中文）
+  const alpha = truncated.replace(/[^A-Za-z]/g, "");
+  if (alpha.length < 4 && cjk.length < 2) return null;
+  // 纯英文停用词片段才拒；纯中文（words 为空）不走这条
+  const words = truncated.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
+  if (words.length > 0 && words.every((w) => STOP_WORDS.has(w))) return null;
+  // P3-A 收紧：裸过泛中文词（报错/失败/超时…+语气助词）且无具体锚点（latin≥3 / 多位数字 / 路径）→ 拒，
+  // 避免检索端 error-signature 精确通道被这类词过度匹配成噪声。
+  const hasSpecificAnchor = /[A-Za-z]{3,}|\d{2,}|\//.test(truncated);
+  if (!hasSpecificAnchor && GENERIC_CN_ERROR.test(truncated)) return null;
+  return truncated;
+}
+
+/** specificity：不寻常 token（PascalCase Error / 路径 / 错误码 / 中文错误）得分更高。 */
+function specificityScore(frag: string, freq: number): number {
+  let score = 0;
+  if (/\b[A-Z][a-zA-Z]*Error\b/.test(frag)) score += 3;
+  if (/\b[A-Z][a-zA-Z]*Exception\b/.test(frag)) score += 3;
+  if (/(\b|_)E[A-Z]{3,}\b/.test(frag)) score += 2; // ENOENT, EACCES
+  if (/\/[a-zA-Z0-9._\-/]+/.test(frag)) score += 2; // path
+  if (/\bcode\s*=\s*\d+/.test(frag)) score += 1;
+  if (/\b\d{3}\b/.test(frag)) score += 1; // status
+  if (/_/.test(frag)) score += 1; // snake_case id
+  if (/[一-鿿]/.test(frag)) score += 1; // 中文错误片段值得保留
+  score += Math.min(2, freq - 1); // 重复片段小幅加权
+  if (frag.length > 120) score -= 1; // 过长片段降权
+  return score;
+}

package/src/error-taxonomy.ts

@@ -0,0 +1,115 @@
+/**
+ * MCP 错误分类学 — 借鉴 RepoPrompt CE 的 error taxonomy。
+ *
+ * 问题：RecallNest 43 个 MCP 工具的错误现在三种风格混着（fail-open 空结果 /
+ * 错误当正常 content / 局部 catch 返回文本），agent 拿到错误无法判断该重试、
+ * 改参数还是上报用户。
+ *
+ * 方案：给每种失败标三元组 {reasonCode, retryable, responsibility}，在 registerTool
+ * 外层统一兜底分类，返回带机器可读 block 的 isError result。
+ *
+ * 范围（第一阶段）：只兜底「抛到 handler 顶层、未被内部 catch」的错误。各 handler
+ * 内部的 fail-open / 文本错误风格统一是后续第二阶段，不在此处改（避免破坏客户端兼容）。
+ */
+
+/** 责任方：host=运行环境/配置，app=RecallNest 自身逻辑，peer=调用方输入或上游 API，transport=网络层。 */
+export type ErrorResponsibility = "host" | "app" | "peer" | "transport";
+
+export interface ErrorClassification {
+  /** 稳定错误码（机器可读，跨版本稳定，供 agent 分支判断）。 */
+  reasonCode: string;
+  /** 是否值得重试（transient vs permanent）。 */
+  retryable: boolean;
+  /** 责任方，决定 agent 是重试 / 改参数 / 上报用户。 */
+  responsibility: ErrorResponsibility;
+}
+
+/**
+ * 失败模式表：按 error message 模式分类。
+ * 未匹配 → internal_error / app / non-retryable（保守：不误导 agent 去重试一个未知错误）。
+ */
+export function classifyError(err: unknown): ErrorClassification {
+  const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+
+  // transport：网络/连接层，可重试
+  if (/econnreset|etimedout|enotfound|econnrefused|socket|network|fetch failed|timed out|timeout/.test(msg)) {
+    return { reasonCode: "transport_error", retryable: true, responsibility: "transport" };
+  }
+  // 上游 API 过载/限流，可重试（退避后）
+  if (/rate.?limit|\b429\b|\b529\b|overloaded|too many requests|quota exceeded/.test(msg)) {
+    return { reasonCode: "upstream_overloaded", retryable: true, responsibility: "peer" };
+  }
+  // 上游 API 鉴权/密钥，不可重试（需修 host 配置）
+  if (/\b401\b|\b403\b|unauthorized|invalid api key|authentication failed|missing api key/.test(msg)) {
+    return { reasonCode: "upstream_auth", retryable: false, responsibility: "host" };
+  }
+  // 运行环境/配置缺失（host）：缺 env var、未配置（Codex 复审 P2-3，关联 lazy-init 失败 P2-1）
+  if (/environment variable .* not set|not configured|missing required (config|environment)/.test(msg)) {
+    return { reasonCode: "config_missing", retryable: false, responsibility: "host" };
+  }
+  // 调用方未给 scope（resolveScopeSelection 抛出）
+  if (/requires a scope|scope is required|pass scope/.test(msg)) {
+    return { reasonCode: "scope_required", retryable: false, responsibility: "peer" };
+  }
+  // 调用方给的 memory 引用无效：ID 格式错 / 前缀歧义 / 不在可访问 scope（Codex 复审 P2-3）。
+  // 指引 agent 给有效 ID / 更长前缀 / 正确 scope，而非误以为 RecallNest 内部故障。
+  if (/invalid memory id|ambiguous prefix|invalid .* id format|outside accessible scope|not accessible/.test(msg)) {
+    return { reasonCode: "invalid_memory_ref", retryable: false, responsibility: "peer" };
+  }
+  // 调用方输入非法（zod/validation）
+  if (/validation|invalid input|expected .* received|must be a|is required|not a valid|failed to parse/.test(msg)) {
+    return { reasonCode: "invalid_input", retryable: false, responsibility: "peer" };
+  }
+  // 存储层（LanceDB / 向量 / 索引）
+  if (/lancedb|database|\btable\b|vector ?search|embedding failed|index/.test(msg)) {
+    return { reasonCode: "store_error", retryable: false, responsibility: "app" };
+  }
+  // 默认：保守归为 app 内部错误，不可重试
+  return { reasonCode: "internal_error", retryable: false, responsibility: "app" };
+}
+
+/** MCP tool error content item 类型（与 handler 成功路径的 content 同构）。 */
+interface ErrorResult {
+  content: { type: "text"; text: string }[];
+  isError: true;
+}
+
+/**
+ * 把任意错误转成带机器可读分类 block 的 MCP isError result。
+ * agent 可解析 reason_code / retryable / responsibility 决定下一步。
+ */
+export function toErrorResult(toolName: string, err: unknown): ErrorResult {
+  const cls = classifyError(err);
+  const detail = err instanceof Error ? err.message : String(err);
+  return {
+    content: [{
+      type: "text",
+      text: [
+        `❌ Tool "${toolName}" failed`,
+        `reason_code: ${cls.reasonCode}`,
+        `retryable: ${cls.retryable}`,
+        `responsibility: ${cls.responsibility}`,
+        `detail: ${detail}`,
+      ].join("\n"),
+    }],
+    isError: true,
+  };
+}
+
+/**
+ * 安全执行 tool 逻辑：捕获任意抛错（含 lazy-init / handler 执行），分类 + 结构化
+ * stderr 日志 + 返回 isError result。提取成独立函数以便单测锁定回归（Codex 复审建议）。
+ * registerTool 的 lazyHandler 把「ensureComponents + handler 调用」整体交给它。
+ */
+export async function runToolSafely<T>(toolName: string, fn: () => Promise<T>): Promise<T | ErrorResult> {
+  try {
+    return await fn();
+  } catch (err) {
+    const cls = classifyError(err);
+    console.error(
+      `[recallnest] tool ${toolName} failed [${cls.reasonCode}/${cls.responsibility}/retryable=${cls.retryable}]:`,
+      err instanceof Error ? err.stack || err.message : String(err),
+    );
+    return toErrorResult(toolName, err);
+  }
+}

package/src/eval.ts

@@ -15,7 +15,7 @@ import type { WorkflowObservationInput } from "./workflow-observation-schema.js"
 import { WorkflowObservationStore } from "./workflow-observation-store.js";
 
 export type ProfileName = "default" | "writing" | "debug" | "fact-check";
-export type EvalMode = "retrieval" | "continuity";
+export type EvalMode = "retrieval" | "continuity" | "canary";
 
 export interface RetrievalEvalCase {
   name: string;
@@ -59,6 +59,47 @@ export interface ContinuityEvalCase {
   notes?: string;
 }
 
+export interface CanaryEvalCase {
+  name: string;
+  query: string;
+  profile?: ProfileName;
+  scope?: string;
+  limit?: number;
+  /** A 类：该召回的 memory id（按 target id 算 rank） */
+  targets?: string[];
+  /** C 类：不该进 TopK 的干扰项 memory id */
+  hardNegatives?: string[];
+  /** 目标应进前几（默认 3） */
+  expectTopK?: number;
+  /** B 类新旧序：按此顺序排列，newer 在前，前者 rank 应小于后者 */
+  expectOrder?: string[];
+  /** D 类稳定偏好：召回文本命中任一要点即算对（不钉 id） */
+  expectContentAny?: string[];
+  /** 文本层禁词（沿用 retrieval 语义） */
+  forbid?: string[];
+  notes?: string;
+}
+
+export interface CanaryCaseReport {
+  mode: "canary";
+  name: string;
+  query: string;
+  profile: ProfileName;
+  score: number;
+  passed: boolean;
+  hitCount: number;
+  targetRanks: Array<{ id: string; rank: number | null }>;
+  top1Hit: boolean;
+  top3Hit: boolean;
+  forbiddenIdMatches: string[];
+  orderOk: boolean | null;
+  matchedContentAny: string[];
+  forbiddenMatches: string[];
+  topIds: string[];
+  topSnippet: string;
+  notes?: string;
+}
+
 export interface RetrievalCaseReport {
   mode: "retrieval";
   name: string;
@@ -100,7 +141,7 @@ export interface ContinuityCaseReport {
   notes?: string;
 }
 
-type EvalReport = RetrievalCaseReport | ContinuityCaseReport;
+type EvalReport = RetrievalCaseReport | ContinuityCaseReport | CanaryCaseReport;
 
 interface EvalArgs {
   mode: EvalMode;
@@ -144,7 +185,7 @@ function parseArgs(args: string[]): EvalArgs {
   const observationScopeIdx = args.indexOf("--observation-scope");
   const observationSourceIdx = args.indexOf("--observation-source");
   const modeRaw = modeIdx >= 0 ? args[modeIdx + 1] : "retrieval";
-  const mode = modeRaw === "continuity" ? "continuity" : "retrieval";
+  const mode: EvalMode = modeRaw === "continuity" ? "continuity" : modeRaw === "canary" ? "canary" : "retrieval";
 
   return {
     mode,
@@ -158,9 +199,9 @@ function parseArgs(args: string[]): EvalArgs {
 }
 
 function defaultCasesPath(mode: EvalMode): string {
-  return mode === "continuity"
-    ? resolve(metaDir(import.meta), "../eval/continuity/cases.json")
-    : resolve(metaDir(import.meta), "../eval/cases.json");
+  if (mode === "continuity") return resolve(metaDir(import.meta), "../eval/continuity/cases.json");
+  if (mode === "canary") return resolve(metaDir(import.meta), "../eval/cases-canary.json");
+  return resolve(metaDir(import.meta), "../eval/cases.json");
 }
 
 function loadCases<T>(mode: EvalMode, pathArg?: string): T[] {
@@ -221,6 +262,109 @@ export function scoreRetrievalCase(
   };
 }
 
+export function scoreCanaryCase(
+  evalCase: CanaryEvalCase,
+  results: Array<{ entry: { id: string; text: string; scope: string; metadata?: string } }>,
+): CanaryCaseReport {
+  const profile = evalCase.profile || "default";
+  const ids = results.map((r) => r.entry.id);
+  const topK = evalCase.expectTopK ?? 3;
+  const limit = evalCase.limit ?? results.length;
+  const joined = results
+    .map((r) => `${r.entry.scope}\n${r.entry.text}\n${r.entry.metadata || ""}`)
+    .join("\n")
+    .toLowerCase();
+
+  // memory id 用前缀匹配：targets/expectOrder/hardNegatives 常写 8 位短前缀，
+  // 而召回结果的 entry.id 是完整 UUID（recallnest 惯例：≥8 hex 前缀即可定位）。
+  const idMatches = (fullId: string, target: string): boolean =>
+    fullId === target || fullId.startsWith(target);
+  const rankOf = (target: string): number | null => {
+    const idx = ids.findIndex((fullId) => idMatches(fullId, target));
+    return idx === -1 ? null : idx + 1;
+  };
+
+  // A 类：单目标命中
+  const targets = evalCase.targets || [];
+  const targetRanks = targets.map((id) => ({ id, rank: rankOf(id) }));
+  const top1Hit = targetRanks.some((t) => t.rank === 1);
+  const top3Hit = targetRanks.some((t) => t.rank !== null && t.rank <= Math.min(3, topK));
+
+  // C 类：干扰项进 TopK（limit 内）即违规（同样前缀匹配）
+  const forbiddenIdMatches = (evalCase.hardNegatives || []).filter((neg) => {
+    const rank = rankOf(neg);
+    return rank !== null && rank <= limit;
+  });
+
+  // B 类：新旧序，前者须召回且 rank 小于后者
+  let orderOk: boolean | null = null;
+  const order = evalCase.expectOrder || [];
+  if (order.length >= 2) {
+    orderOk = true;
+    for (let i = 0; i < order.length - 1; i += 1) {
+      const a = rankOf(order[i]);
+      const b = rankOf(order[i + 1]);
+      if (a === null) {
+        orderOk = false;
+        break;
+      }
+      if (b !== null && a >= b) {
+        orderOk = false;
+        break;
+      }
+    }
+  }
+
+  // D 类：内容要点命中
+  const matchedContentAny = matchedTerms(evalCase.expectContentAny, joined);
+
+  // 文本禁词
+  const forbiddenMatches = matchedTerms(evalCase.forbid, joined);
+
+  // 动态加权：只计 case 实际声明的维度
+  const parts: Array<{ weight: number; value: number }> = [];
+  if (targets.length > 0) {
+    const ranks = targetRanks.map((t) => t.rank).filter((r): r is number => r !== null);
+    const bestRank = ranks.length > 0 ? Math.min(...ranks) : null;
+    const targetScore = bestRank === null ? 0 : bestRank === 1 ? 1 : bestRank <= 3 ? 0.7 : 0.4;
+    parts.push({ weight: 0.5, value: targetScore });
+  }
+  if (orderOk !== null) {
+    parts.push({ weight: 0.2, value: orderOk ? 1 : 0 });
+  }
+  const contentAny = evalCase.expectContentAny || [];
+  if (contentAny.length > 0) {
+    parts.push({ weight: 0.3, value: matchedContentAny.length / contentAny.length });
+  }
+  const totalWeight = parts.reduce((sum, p) => sum + p.weight, 0);
+  let score = totalWeight > 0
+    ? parts.reduce((sum, p) => sum + p.weight * p.value, 0) / totalWeight
+    : (results.length > 0 ? 0.5 : 0);
+  if (forbiddenIdMatches.length > 0) score -= 0.3;
+  if (forbiddenMatches.length > 0) score -= 0.3;
+  score = Math.max(0, Math.min(1, score));
+
+  return {
+    mode: "canary",
+    name: evalCase.name,
+    query: evalCase.query,
+    profile,
+    score,
+    passed: score >= 0.7 && forbiddenIdMatches.length === 0 && forbiddenMatches.length === 0,
+    hitCount: results.length,
+    targetRanks,
+    top1Hit,
+    top3Hit,
+    forbiddenIdMatches,
+    orderOk,
+    matchedContentAny,
+    forbiddenMatches,
+    topIds: ids.slice(0, 5),
+    topSnippet: results[0] ? clip(results[0].entry.text) : "-",
+    notes: evalCase.notes,
+  };
+}
+
 function joinResumeSections(response: ResumeContextResponse): string {
   return [
     response.summary,
@@ -438,6 +582,50 @@ function markdownRetrievalReport(reports: RetrievalCaseReport[]): string {
   return lines.join("\n");
 }
 
+function markdownCanaryReport(reports: CanaryCaseReport[]): string {
+  const { passed, average } = summarizeReports(reports);
+  const top1 = reports.filter((r) => r.top1Hit).length;
+  const top3 = reports.filter((r) => r.top3Hit).length;
+
+  const lines = [
+    "# RecallNest Canary Eval",
+    "",
+    `- Generated: ${new Date().toISOString()}`,
+    `- Cases: ${reports.length}`,
+    `- Passed: ${passed}/${reports.length}`,
+    `- Average score: ${formatPercent(average)}`,
+    `- Top1 hit: ${top1}/${reports.length}`,
+    `- Top3 hit: ${top3}/${reports.length}`,
+    "",
+    "| Case | Profile | Score | Pass | Top1 | Top3 | Order | Forbid |",
+    "|------|---------|-------|------|------|------|-------|--------|",
+    ...reports.map((item) =>
+      `| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.top1Hit ? "yes" : "-"} | ${item.top3Hit ? "yes" : "-"} | ${item.orderOk === null ? "-" : item.orderOk ? "ok" : "bad"} | ${item.forbiddenIdMatches.length + item.forbiddenMatches.length || "-"} |`,
+    ),
+    "",
+    "## Case Notes",
+    "",
+  ];
+
+  for (const item of reports) {
+    lines.push(`### ${item.name}`);
+    lines.push(`- Query: ${item.query}`);
+    lines.push(`- Score: ${(item.score * 100).toFixed(0)}% (${item.passed ? "pass" : "fail"})`);
+    lines.push(`- Target ranks: ${item.targetRanks.map((t) => `${t.id}=${t.rank ?? "miss"}`).join(", ") || "-"}`);
+    lines.push(`- Top1 / Top3: ${item.top1Hit ? "yes" : "no"} / ${item.top3Hit ? "yes" : "no"}`);
+    lines.push(`- Order ok: ${item.orderOk === null ? "n/a" : item.orderOk}`);
+    lines.push(`- Matched content: ${item.matchedContentAny.join(", ") || "-"}`);
+    lines.push(`- Forbidden id matches: ${item.forbiddenIdMatches.join(", ") || "-"}`);
+    lines.push(`- Forbidden term matches: ${item.forbiddenMatches.join(", ") || "-"}`);
+    lines.push(`- Top ids: ${item.topIds.join(", ") || "-"}`);
+    lines.push(`- Top snippet: ${item.topSnippet}`);
+    if (item.notes) lines.push(`- Notes: ${item.notes}`);
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
 function markdownContinuityReport(reports: ContinuityCaseReport[]): string {
   const { passed, average } = summarizeReports(reports);
 
@@ -523,6 +711,44 @@ export async function runRetrievalEval(
   return reports;
 }
 
+export async function runCanaryEval(
+  cases: CanaryEvalCase[],
+  deps: {
+    createEvalComponents?: EvalComponentFactory;
+  } = {},
+): Promise<CanaryCaseReport[]> {
+  const config = deps.createEvalComponents ? null : loadConfig();
+  const createEvalComponentsForCase = deps.createEvalComponents || createFreshEvalComponentsFactory(config!);
+  const reports: CanaryCaseReport[] = [];
+
+  for (const [index, evalCase] of cases.entries()) {
+    if (cases.length > 1) {
+      logInfo(`[INFO] canary-eval ${index + 1}/${cases.length}: ${evalCase.name}`);
+    }
+    const profileName = evalCase.profile || "default";
+    const { retriever, accessTracker } = createEvalComponentsForCase(profileName);
+    try {
+      const results = await retriever.retrieve({
+        query: evalCase.query,
+        limit: evalCase.limit || 8,
+        scopeFilter: evalCase.scope ? [evalCase.scope] : undefined,
+        source: "auto-recall",
+      });
+      const report = scoreCanaryCase(evalCase, results);
+      reports.push(report);
+      if (cases.length > 1) {
+        logInfo(
+          `[INFO] canary-eval ${index + 1}/${cases.length} done: ${evalCase.name} ${formatPercent(report.score)} ${report.passed ? "pass" : "fail"}`,
+        );
+      }
+    } finally {
+      accessTracker.destroy();
+    }
+  }
+
+  return reports;
+}
+
 export async function runContinuityEval(
   cases: ContinuityEvalCase[],
   options: { recordObservations?: boolean; observationScope?: string; observationSource?: string } = {},
@@ -610,6 +836,25 @@ async function main() {
     return;
   }
 
+  if (args.mode === "canary") {
+    const cases = loadCases<CanaryEvalCase>("canary", args.casesPath);
+    const reports = await runCanaryEval(cases);
+    if (args.jsonMode) {
+      const payload = JSON.stringify({
+        mode: "canary",
+        generatedAt: new Date().toISOString(),
+        reports,
+      }, null, 2);
+      writeOutput(args.outputPath, payload);
+      console.log(payload);
+      return;
+    }
+    const output = markdownCanaryReport(reports);
+    writeOutput(args.outputPath, output);
+    console.log(output);
+    return;
+  }
+
   const cases = loadCases<RetrievalEvalCase>("retrieval", args.casesPath);
   const reports = await runRetrievalEval(cases);
   if (args.jsonMode) {

package/src/idempotency.ts

@@ -0,0 +1,8 @@
+import { createHash } from "node:crypto";
+
+export function buildIdempotentRecordId(prefix: string, key?: string): string | undefined {
+  const normalized = key?.replace(/\s+/g, " ").trim();
+  if (!normalized) return undefined;
+  const digest = createHash("sha256").update(normalized).digest("hex").slice(0, 32);
+  return `${prefix}-${digest}`;
+}