recallnest 2.3.0 → 2.6.0

package/src/eval.ts

@@ -8,6 +8,8 @@ import { buildSessionCheckpointRecord, normalizeCheckpointScope } from "./sessio
 import type { ResumeContextResponse, SessionCheckpointRecord } from "./session-schema.js";
 import { composeResumeContext } from "./context-composer.js";
 import { cleanText } from "./context-composer-text.js";
+import { estimateTokens } from "./context-collapse-renderer.js";
+import { TraceCollector } from "./retrieval-trace.js";
 import { createComponents, loadConfig, loadDotEnv, type LocalMemoryConfig } from "./runtime-config.js";
 import { logInfo } from "./stderr-log.js";
 import { buildWorkflowObservationRecord } from "./workflow-observation-engine.js";
@@ -15,7 +17,7 @@ import type { WorkflowObservationInput } from "./workflow-observation-schema.js"
 import { WorkflowObservationStore } from "./workflow-observation-store.js";
 
 export type ProfileName = "default" | "writing" | "debug" | "fact-check";
-export type EvalMode = "retrieval" | "continuity";
+export type EvalMode = "retrieval" | "continuity" | "canary";
 
 export interface RetrievalEvalCase {
   name: string;
@@ -59,6 +61,73 @@ export interface ContinuityEvalCase {
   notes?: string;
 }
 
+export interface CanaryEvalCase {
+  name: string;
+  query: string;
+  profile?: ProfileName;
+  scope?: string;
+  limit?: number;
+  /** A 类：该召回的 memory id（按 target id 算 rank） */
+  targets?: string[];
+  /** C 类：不该进 TopK 的干扰项 memory id */
+  hardNegatives?: string[];
+  /** 目标应进前几（默认 3） */
+  expectTopK?: number;
+  /** B 类新旧序：按此顺序排列，newer 在前，前者 rank 应小于后者 */
+  expectOrder?: string[];
+  /** D 类稳定偏好：召回文本命中任一要点即算对（不钉 id） */
+  expectContentAny?: string[];
+  /** 文本层禁词（沿用 retrieval 语义） */
+  forbid?: string[];
+  notes?: string;
+}
+
+/**
+ * Per-case cost accounting for the retrieval pipeline — the X axis of the
+ * Pareto curve (see hippo-wiki [[帕累托效率曲线]]).
+ *
+ * Scope: self-comparison across runs (did a retrieval change make it slower or
+ * fatter?). NOT for cross-system comparison — `contextTokens` comes from an
+ * estimator, not a real tokenizer, and this suite is not LoCoMo, so the numbers
+ * are not commensurable with competitors' published token counts.
+ */
+export interface EvalCostMetrics {
+  /** End-to-end retrieve() wall time. */
+  totalMs: number;
+  /** Number of memories returned. */
+  resultCount: number;
+  /** Estimated tokens of returned memory text — the context a caller would spend. */
+  contextTokens: number;
+  /** Duration of the rerank stage, or null when the pipeline ran no rerank.
+   *  Left as a raw fact rather than an inferred "api calls" count: a local
+   *  lightweight rerank and a remote cross-encoder both produce a stage here. */
+  rerankMs: number | null;
+  /** Three slowest stages, for locating the bottleneck. */
+  slowestStages: Array<{ name: string; durationMs: number }>;
+}
+
+export interface CanaryCaseReport {
+  mode: "canary";
+  name: string;
+  query: string;
+  profile: ProfileName;
+  score: number;
+  passed: boolean;
+  hitCount: number;
+  targetRanks: Array<{ id: string; rank: number | null }>;
+  top1Hit: boolean;
+  top3Hit: boolean;
+  forbiddenIdMatches: string[];
+  orderOk: boolean | null;
+  matchedContentAny: string[];
+  forbiddenMatches: string[];
+  topIds: string[];
+  topSnippet: string;
+  notes?: string;
+  /** Attached by the runner after scoring; absent when scored standalone. */
+  cost?: EvalCostMetrics;
+}
+
 export interface RetrievalCaseReport {
   mode: "retrieval";
   name: string;
@@ -74,6 +143,8 @@ export interface RetrievalCaseReport {
   topScopes: string[];
   topSnippet: string;
   notes?: string;
+  /** Attached by the runner after scoring; absent when scored standalone. */
+  cost?: EvalCostMetrics;
 }
 
 export interface ContinuityCaseReport {
@@ -100,7 +171,7 @@ export interface ContinuityCaseReport {
   notes?: string;
 }
 
-type EvalReport = RetrievalCaseReport | ContinuityCaseReport;
+type EvalReport = RetrievalCaseReport | ContinuityCaseReport | CanaryCaseReport;
 
 interface EvalArgs {
   mode: EvalMode;
@@ -144,7 +215,7 @@ function parseArgs(args: string[]): EvalArgs {
   const observationScopeIdx = args.indexOf("--observation-scope");
   const observationSourceIdx = args.indexOf("--observation-source");
   const modeRaw = modeIdx >= 0 ? args[modeIdx + 1] : "retrieval";
-  const mode = modeRaw === "continuity" ? "continuity" : "retrieval";
+  const mode: EvalMode = modeRaw === "continuity" ? "continuity" : modeRaw === "canary" ? "canary" : "retrieval";
 
   return {
     mode,
@@ -158,9 +229,9 @@ function parseArgs(args: string[]): EvalArgs {
 }
 
 function defaultCasesPath(mode: EvalMode): string {
-  return mode === "continuity"
-    ? resolve(metaDir(import.meta), "../eval/continuity/cases.json")
-    : resolve(metaDir(import.meta), "../eval/cases.json");
+  if (mode === "continuity") return resolve(metaDir(import.meta), "../eval/continuity/cases.json");
+  if (mode === "canary") return resolve(metaDir(import.meta), "../eval/cases-canary.json");
+  return resolve(metaDir(import.meta), "../eval/cases.json");
 }
 
 function loadCases<T>(mode: EvalMode, pathArg?: string): T[] {
@@ -176,8 +247,16 @@ function matchedTerms(terms: string[] | undefined, haystack: string): string[] {
   return (terms || []).filter((term) => haystack.includes(term.toLowerCase()));
 }
 
+/**
+ * 命中比例 × 权重。
+ *
+ * 调用方必须先滤掉 case 没声明的期望、再对剩下的权重重新归一化——下面三个
+ * 评分器都是这么做的。空期望这里返回 0 而不是 weight：万一将来有第四个调用
+ * 点忘了滤，代价是少给分而不是白送分。三个现有调用点都已滤过，所以这个分支
+ * 当前走不到，纯属兜底。
+ */
 function scoreExpectation(expected: string[] | undefined, matched: string[], weight: number): number {
-  if (!expected || expected.length === 0) return weight;
+  if (!expected || expected.length === 0) return 0;
   return (matched.length / expected.length) * weight;
 }
 
@@ -195,10 +274,23 @@ export function scoreRetrievalCase(
   const matchedScopes = (evalCase.expectScopePrefixes || []).filter((scope) => scopes.some((item) => item.startsWith(scope)));
   const forbiddenMatches = matchedTerms(evalCase.forbid, joined);
 
-  let score = 0;
-  score += scoreExpectation(evalCase.expectAny, matchedAny, 0.4);
-  score += scoreExpectation(evalCase.expectAll, matchedAll, 0.3);
-  score += scoreExpectation(evalCase.expectScopePrefixes, matchedScopes, 0.2);
+  // 动态加权：只计 case 实际声明的维度，权重在这些维度间重新归一化——
+  // 和下面 canary、continuity 两个评分器已有的写法一致。
+  // 改之前未声明的维度按满权重计入，而 eval/cases.json 里 20 个 case 一个都没写
+  // expectAll，于是每个 case 白拿 0.3：expectAny 四个词命中一个再加 scope 前缀命中，
+  // 正好落在 0.7 及格线上。
+  const expectationScores = [
+    { expected: evalCase.expectAny, matched: matchedAny, weight: 0.4 },
+    { expected: evalCase.expectAll, matched: matchedAll, weight: 0.3 },
+    { expected: evalCase.expectScopePrefixes, matched: matchedScopes, weight: 0.2 },
+  ].filter((item) => (item.expected || []).length > 0);
+
+  const totalExpectedWeight = expectationScores.reduce((sum, item) => sum + item.weight, 0);
+  const normalizedExpectationScore = totalExpectedWeight > 0
+    ? expectationScores.reduce((sum, item) => sum + scoreExpectation(item.expected, item.matched, item.weight), 0) / totalExpectedWeight
+    : 0;
+
+  let score = normalizedExpectationScore * 0.9;
   if (results.length > 0) score += 0.1;
   if (forbiddenMatches.length > 0) score -= 0.3;
   score = Math.max(0, Math.min(1, score));
@@ -221,6 +313,109 @@ export function scoreRetrievalCase(
   };
 }
 
+export function scoreCanaryCase(
+  evalCase: CanaryEvalCase,
+  results: Array<{ entry: { id: string; text: string; scope: string; metadata?: string } }>,
+): CanaryCaseReport {
+  const profile = evalCase.profile || "default";
+  const ids = results.map((r) => r.entry.id);
+  const topK = evalCase.expectTopK ?? 3;
+  const limit = evalCase.limit ?? results.length;
+  const joined = results
+    .map((r) => `${r.entry.scope}\n${r.entry.text}\n${r.entry.metadata || ""}`)
+    .join("\n")
+    .toLowerCase();
+
+  // memory id 用前缀匹配：targets/expectOrder/hardNegatives 常写 8 位短前缀，
+  // 而召回结果的 entry.id 是完整 UUID（recallnest 惯例：≥8 hex 前缀即可定位）。
+  const idMatches = (fullId: string, target: string): boolean =>
+    fullId === target || fullId.startsWith(target);
+  const rankOf = (target: string): number | null => {
+    const idx = ids.findIndex((fullId) => idMatches(fullId, target));
+    return idx === -1 ? null : idx + 1;
+  };
+
+  // A 类：单目标命中
+  const targets = evalCase.targets || [];
+  const targetRanks = targets.map((id) => ({ id, rank: rankOf(id) }));
+  const top1Hit = targetRanks.some((t) => t.rank === 1);
+  const top3Hit = targetRanks.some((t) => t.rank !== null && t.rank <= Math.min(3, topK));
+
+  // C 类：干扰项进 TopK（limit 内）即违规（同样前缀匹配）
+  const forbiddenIdMatches = (evalCase.hardNegatives || []).filter((neg) => {
+    const rank = rankOf(neg);
+    return rank !== null && rank <= limit;
+  });
+
+  // B 类：新旧序，前者须召回且 rank 小于后者
+  let orderOk: boolean | null = null;
+  const order = evalCase.expectOrder || [];
+  if (order.length >= 2) {
+    orderOk = true;
+    for (let i = 0; i < order.length - 1; i += 1) {
+      const a = rankOf(order[i]);
+      const b = rankOf(order[i + 1]);
+      if (a === null) {
+        orderOk = false;
+        break;
+      }
+      if (b !== null && a >= b) {
+        orderOk = false;
+        break;
+      }
+    }
+  }
+
+  // D 类：内容要点命中
+  const matchedContentAny = matchedTerms(evalCase.expectContentAny, joined);
+
+  // 文本禁词
+  const forbiddenMatches = matchedTerms(evalCase.forbid, joined);
+
+  // 动态加权：只计 case 实际声明的维度
+  const parts: Array<{ weight: number; value: number }> = [];
+  if (targets.length > 0) {
+    const ranks = targetRanks.map((t) => t.rank).filter((r): r is number => r !== null);
+    const bestRank = ranks.length > 0 ? Math.min(...ranks) : null;
+    const targetScore = bestRank === null ? 0 : bestRank === 1 ? 1 : bestRank <= 3 ? 0.7 : 0.4;
+    parts.push({ weight: 0.5, value: targetScore });
+  }
+  if (orderOk !== null) {
+    parts.push({ weight: 0.2, value: orderOk ? 1 : 0 });
+  }
+  const contentAny = evalCase.expectContentAny || [];
+  if (contentAny.length > 0) {
+    parts.push({ weight: 0.3, value: matchedContentAny.length / contentAny.length });
+  }
+  const totalWeight = parts.reduce((sum, p) => sum + p.weight, 0);
+  let score = totalWeight > 0
+    ? parts.reduce((sum, p) => sum + p.weight * p.value, 0) / totalWeight
+    : (results.length > 0 ? 0.5 : 0);
+  if (forbiddenIdMatches.length > 0) score -= 0.3;
+  if (forbiddenMatches.length > 0) score -= 0.3;
+  score = Math.max(0, Math.min(1, score));
+
+  return {
+    mode: "canary",
+    name: evalCase.name,
+    query: evalCase.query,
+    profile,
+    score,
+    passed: score >= 0.7 && forbiddenIdMatches.length === 0 && forbiddenMatches.length === 0,
+    hitCount: results.length,
+    targetRanks,
+    top1Hit,
+    top3Hit,
+    forbiddenIdMatches,
+    orderOk,
+    matchedContentAny,
+    forbiddenMatches,
+    topIds: ids.slice(0, 5),
+    topSnippet: results[0] ? clip(results[0].entry.text) : "-",
+    notes: evalCase.notes,
+  };
+}
+
 function joinResumeSections(response: ResumeContextResponse): string {
   return [
     response.summary,
@@ -408,11 +603,12 @@ function markdownRetrievalReport(reports: RetrievalCaseReport[]): string {
     `- Cases: ${reports.length}`,
     `- Passed: ${passed}/${reports.length}`,
     `- Average score: ${formatPercent(average)}`,
+    ...costSummaryLines(summarizeCost(reports), reports.length),
     "",
-    "| Case | Profile | Score | Pass | Hits | Top scopes |",
-    "|------|---------|-------|------|------|------------|",
+    "| Case | Profile | Score | Pass | Hits | ms | ~tok | Top scopes |",
+    "|------|---------|-------|------|------|----|------|------------|",
     ...reports.map((item) =>
-      `| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.hitCount} | ${item.topScopes.join(", ") || "-"} |`,
+      `| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.hitCount} | ${item.cost?.totalMs ?? "-"} | ${item.cost?.contextTokens ?? "-"} | ${item.topScopes.join(", ") || "-"} |`,
     ),
     "",
     "## Case Notes",
@@ -431,6 +627,53 @@ function markdownRetrievalReport(reports: RetrievalCaseReport[]): string {
     lines.push(`- Matched all: ${item.matchedAll.join(", ") || "-"}`);
     lines.push(`- Matched scopes: ${item.matchedScopes.join(", ") || "-"}`);
     lines.push(`- Forbidden matches: ${item.forbiddenMatches.join(", ") || "-"}`);
+    if (item.cost) lines.push(`- Cost: ${formatCost(item.cost)}`);
+    if (item.notes) lines.push(`- Notes: ${item.notes}`);
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
+function markdownCanaryReport(reports: CanaryCaseReport[]): string {
+  const { passed, average } = summarizeReports(reports);
+  const top1 = reports.filter((r) => r.top1Hit).length;
+  const top3 = reports.filter((r) => r.top3Hit).length;
+
+  const lines = [
+    "# RecallNest Canary Eval",
+    "",
+    `- Generated: ${new Date().toISOString()}`,
+    `- Cases: ${reports.length}`,
+    `- Passed: ${passed}/${reports.length}`,
+    `- Average score: ${formatPercent(average)}`,
+    `- Top1 hit: ${top1}/${reports.length}`,
+    `- Top3 hit: ${top3}/${reports.length}`,
+    ...costSummaryLines(summarizeCost(reports), reports.length),
+    "",
+    "| Case | Profile | Score | Pass | Top1 | Top3 | Order | Forbid | ms | ~tok |",
+    "|------|---------|-------|------|------|------|-------|--------|----|------|",
+    ...reports.map((item) =>
+      `| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.top1Hit ? "yes" : "-"} | ${item.top3Hit ? "yes" : "-"} | ${item.orderOk === null ? "-" : item.orderOk ? "ok" : "bad"} | ${item.forbiddenIdMatches.length + item.forbiddenMatches.length || "-"} | ${item.cost?.totalMs ?? "-"} | ${item.cost?.contextTokens ?? "-"} |`,
+    ),
+    "",
+    "## Case Notes",
+    "",
+  ];
+
+  for (const item of reports) {
+    lines.push(`### ${item.name}`);
+    lines.push(`- Query: ${item.query}`);
+    lines.push(`- Score: ${(item.score * 100).toFixed(0)}% (${item.passed ? "pass" : "fail"})`);
+    lines.push(`- Target ranks: ${item.targetRanks.map((t) => `${t.id}=${t.rank ?? "miss"}`).join(", ") || "-"}`);
+    lines.push(`- Top1 / Top3: ${item.top1Hit ? "yes" : "no"} / ${item.top3Hit ? "yes" : "no"}`);
+    lines.push(`- Order ok: ${item.orderOk === null ? "n/a" : item.orderOk}`);
+    lines.push(`- Matched content: ${item.matchedContentAny.join(", ") || "-"}`);
+    lines.push(`- Forbidden id matches: ${item.forbiddenIdMatches.join(", ") || "-"}`);
+    lines.push(`- Forbidden term matches: ${item.forbiddenMatches.join(", ") || "-"}`);
+    lines.push(`- Top ids: ${item.topIds.join(", ") || "-"}`);
+    lines.push(`- Top snippet: ${item.topSnippet}`);
+    if (item.cost) lines.push(`- Cost: ${formatCost(item.cost)}`);
     if (item.notes) lines.push(`- Notes: ${item.notes}`);
     lines.push("");
   }
@@ -485,6 +728,88 @@ function markdownContinuityReport(reports: ContinuityCaseReport[]): string {
   return lines.join("\n");
 }
 
+/**
+ * Turn a finished TraceCollector plus the returned memories into cost metrics.
+ *
+ * The `mode` argument to finalize() only populates a display field on the trace
+ * object; this function consumes stages / totalMs only, so it is not inferred.
+ */
+function collectCostMetrics(
+  trace: TraceCollector,
+  query: string,
+  results: Array<{ entry: { text: string } }>,
+): EvalCostMetrics {
+  const finalized = trace.finalize(query, "hybrid");
+  const rerankStage = finalized.stages.find((stage) => stage.name === "rerank");
+  const slowestStages = [...finalized.stages]
+    .sort((a, b) => b.durationMs - a.durationMs)
+    .slice(0, 3)
+    .map((stage) => ({ name: stage.name, durationMs: stage.durationMs }));
+
+  return {
+    totalMs: finalized.totalMs,
+    resultCount: results.length,
+    contextTokens: results.reduce((sum, item) => sum + estimateTokens(item.entry.text), 0),
+    rerankMs: rerankStage ? rerankStage.durationMs : null,
+    slowestStages,
+  };
+}
+
+/** Aggregate cost across cases. Cases without cost data are skipped, and the
+ *  count of those that did carry it is returned so the report can be honest
+ *  about coverage rather than silently averaging over fewer cases. */
+function summarizeCost(reports: Array<{ cost?: EvalCostMetrics }>): {
+  measured: number;
+  totalMs: number;
+  avgMs: number;
+  totalTokens: number;
+  avgTokens: number;
+} | null {
+  const withCost = reports.map((item) => item.cost).filter((cost): cost is EvalCostMetrics => Boolean(cost));
+  if (withCost.length === 0) return null;
+
+  const totalMs = withCost.reduce((sum, cost) => sum + cost.totalMs, 0);
+  const totalTokens = withCost.reduce((sum, cost) => sum + cost.contextTokens, 0);
+  return {
+    measured: withCost.length,
+    totalMs,
+    avgMs: Math.round(totalMs / withCost.length),
+    totalTokens,
+    avgTokens: Math.round(totalTokens / withCost.length),
+  };
+}
+
+/** Aggregate cost lines for a report header. Returns [] when no case carried
+ *  cost data, so reports produced before this instrumentation still render. */
+function costSummaryLines(
+  summary: ReturnType<typeof summarizeCost>,
+  caseCount: number,
+): string[] {
+  if (!summary) return [];
+  const coverage = summary.measured === caseCount
+    ? ""
+    : ` (measured on ${summary.measured}/${caseCount} cases)`;
+  return [
+    `- Retrieval latency: ${summary.avgMs} ms avg, ${summary.totalMs} ms total${coverage}`,
+    `- Context tokens: ~${summary.avgTokens} avg, ~${summary.totalTokens} total`
+      + ` (estimated, not tokenizer-measured — compare across runs, not against other systems)`,
+  ];
+}
+
+/** One-line per-case cost breakdown. */
+function formatCost(cost: EvalCostMetrics): string {
+  const parts = [
+    `${cost.totalMs}ms`,
+    `~${cost.contextTokens} tok`,
+    `${cost.resultCount} results`,
+    cost.rerankMs === null ? "no rerank stage" : `rerank ${cost.rerankMs}ms`,
+  ];
+  const slowest = cost.slowestStages
+    .map((stage) => `${stage.name} ${stage.durationMs}ms`)
+    .join(", ");
+  return `${parts.join(" / ")}${slowest ? ` | slowest: ${slowest}` : ""}`;
+}
+
 export async function runRetrievalEval(
   cases: RetrievalEvalCase[],
   deps: {
@@ -502,13 +827,16 @@ export async function runRetrievalEval(
     const profileName = evalCase.profile || "default";
     const { retriever, accessTracker } = createEvalComponentsForCase(profileName);
     try {
+      const trace = new TraceCollector();
       const results = await retriever.retrieve({
         query: evalCase.query,
         limit: evalCase.limit || 5,
         scopeFilter: evalCase.scope ? [evalCase.scope] : undefined,
         source: "auto-recall",
+        trace,
       });
       const report = scoreRetrievalCase(evalCase, results);
+      report.cost = collectCostMetrics(trace, evalCase.query, results);
       reports.push(report);
       if (cases.length > 1) {
         logInfo(
@@ -523,6 +851,47 @@ export async function runRetrievalEval(
   return reports;
 }
 
+export async function runCanaryEval(
+  cases: CanaryEvalCase[],
+  deps: {
+    createEvalComponents?: EvalComponentFactory;
+  } = {},
+): Promise<CanaryCaseReport[]> {
+  const config = deps.createEvalComponents ? null : loadConfig();
+  const createEvalComponentsForCase = deps.createEvalComponents || createFreshEvalComponentsFactory(config!);
+  const reports: CanaryCaseReport[] = [];
+
+  for (const [index, evalCase] of cases.entries()) {
+    if (cases.length > 1) {
+      logInfo(`[INFO] canary-eval ${index + 1}/${cases.length}: ${evalCase.name}`);
+    }
+    const profileName = evalCase.profile || "default";
+    const { retriever, accessTracker } = createEvalComponentsForCase(profileName);
+    try {
+      const trace = new TraceCollector();
+      const results = await retriever.retrieve({
+        query: evalCase.query,
+        limit: evalCase.limit || 8,
+        scopeFilter: evalCase.scope ? [evalCase.scope] : undefined,
+        source: "auto-recall",
+        trace,
+      });
+      const report = scoreCanaryCase(evalCase, results);
+      report.cost = collectCostMetrics(trace, evalCase.query, results);
+      reports.push(report);
+      if (cases.length > 1) {
+        logInfo(
+          `[INFO] canary-eval ${index + 1}/${cases.length} done: ${evalCase.name} ${formatPercent(report.score)} ${report.passed ? "pass" : "fail"}`,
+        );
+      }
+    } finally {
+      accessTracker.destroy();
+    }
+  }
+
+  return reports;
+}
+
 export async function runContinuityEval(
   cases: ContinuityEvalCase[],
   options: { recordObservations?: boolean; observationScope?: string; observationSource?: string } = {},
@@ -610,6 +979,25 @@ async function main() {
     return;
   }
 
+  if (args.mode === "canary") {
+    const cases = loadCases<CanaryEvalCase>("canary", args.casesPath);
+    const reports = await runCanaryEval(cases);
+    if (args.jsonMode) {
+      const payload = JSON.stringify({
+        mode: "canary",
+        generatedAt: new Date().toISOString(),
+        reports,
+      }, null, 2);
+      writeOutput(args.outputPath, payload);
+      console.log(payload);
+      return;
+    }
+    const output = markdownCanaryReport(reports);
+    writeOutput(args.outputPath, output);
+    console.log(output);
+    return;
+  }
+
   const cases = loadCases<RetrievalEvalCase>("retrieval", args.casesPath);
   const reports = await runRetrievalEval(cases);
   if (args.jsonMode) {