recallnest 2.3.0 → 2.5.4

package/src/eval.ts

@@ -15,7 +15,7 @@ import type { WorkflowObservationInput } from "./workflow-observation-schema.js"
 import { WorkflowObservationStore } from "./workflow-observation-store.js";
 
 export type ProfileName = "default" | "writing" | "debug" | "fact-check";
-export type EvalMode = "retrieval" | "continuity";
+export type EvalMode = "retrieval" | "continuity" | "canary";
 
 export interface RetrievalEvalCase {
   name: string;
@@ -59,6 +59,47 @@ export interface ContinuityEvalCase {
   notes?: string;
 }
 
+export interface CanaryEvalCase {
+  name: string;
+  query: string;
+  profile?: ProfileName;
+  scope?: string;
+  limit?: number;
+  /** A 类：该召回的 memory id（按 target id 算 rank） */
+  targets?: string[];
+  /** C 类：不该进 TopK 的干扰项 memory id */
+  hardNegatives?: string[];
+  /** 目标应进前几（默认 3） */
+  expectTopK?: number;
+  /** B 类新旧序：按此顺序排列，newer 在前，前者 rank 应小于后者 */
+  expectOrder?: string[];
+  /** D 类稳定偏好：召回文本命中任一要点即算对（不钉 id） */
+  expectContentAny?: string[];
+  /** 文本层禁词（沿用 retrieval 语义） */
+  forbid?: string[];
+  notes?: string;
+}
+
+export interface CanaryCaseReport {
+  mode: "canary";
+  name: string;
+  query: string;
+  profile: ProfileName;
+  score: number;
+  passed: boolean;
+  hitCount: number;
+  targetRanks: Array<{ id: string; rank: number | null }>;
+  top1Hit: boolean;
+  top3Hit: boolean;
+  forbiddenIdMatches: string[];
+  orderOk: boolean | null;
+  matchedContentAny: string[];
+  forbiddenMatches: string[];
+  topIds: string[];
+  topSnippet: string;
+  notes?: string;
+}
+
 export interface RetrievalCaseReport {
   mode: "retrieval";
   name: string;
@@ -100,7 +141,7 @@ export interface ContinuityCaseReport {
   notes?: string;
 }
 
-type EvalReport = RetrievalCaseReport | ContinuityCaseReport;
+type EvalReport = RetrievalCaseReport | ContinuityCaseReport | CanaryCaseReport;
 
 interface EvalArgs {
   mode: EvalMode;
@@ -144,7 +185,7 @@ function parseArgs(args: string[]): EvalArgs {
   const observationScopeIdx = args.indexOf("--observation-scope");
   const observationSourceIdx = args.indexOf("--observation-source");
   const modeRaw = modeIdx >= 0 ? args[modeIdx + 1] : "retrieval";
-  const mode = modeRaw === "continuity" ? "continuity" : "retrieval";
+  const mode: EvalMode = modeRaw === "continuity" ? "continuity" : modeRaw === "canary" ? "canary" : "retrieval";
 
   return {
     mode,
@@ -158,9 +199,9 @@ function parseArgs(args: string[]): EvalArgs {
 }
 
 function defaultCasesPath(mode: EvalMode): string {
-  return mode === "continuity"
-    ? resolve(metaDir(import.meta), "../eval/continuity/cases.json")
-    : resolve(metaDir(import.meta), "../eval/cases.json");
+  if (mode === "continuity") return resolve(metaDir(import.meta), "../eval/continuity/cases.json");
+  if (mode === "canary") return resolve(metaDir(import.meta), "../eval/cases-canary.json");
+  return resolve(metaDir(import.meta), "../eval/cases.json");
 }
 
 function loadCases<T>(mode: EvalMode, pathArg?: string): T[] {
@@ -221,6 +262,109 @@ export function scoreRetrievalCase(
   };
 }
 
+export function scoreCanaryCase(
+  evalCase: CanaryEvalCase,
+  results: Array<{ entry: { id: string; text: string; scope: string; metadata?: string } }>,
+): CanaryCaseReport {
+  const profile = evalCase.profile || "default";
+  const ids = results.map((r) => r.entry.id);
+  const topK = evalCase.expectTopK ?? 3;
+  const limit = evalCase.limit ?? results.length;
+  const joined = results
+    .map((r) => `${r.entry.scope}\n${r.entry.text}\n${r.entry.metadata || ""}`)
+    .join("\n")
+    .toLowerCase();
+
+  // memory id 用前缀匹配：targets/expectOrder/hardNegatives 常写 8 位短前缀，
+  // 而召回结果的 entry.id 是完整 UUID（recallnest 惯例：≥8 hex 前缀即可定位）。
+  const idMatches = (fullId: string, target: string): boolean =>
+    fullId === target || fullId.startsWith(target);
+  const rankOf = (target: string): number | null => {
+    const idx = ids.findIndex((fullId) => idMatches(fullId, target));
+    return idx === -1 ? null : idx + 1;
+  };
+
+  // A 类：单目标命中
+  const targets = evalCase.targets || [];
+  const targetRanks = targets.map((id) => ({ id, rank: rankOf(id) }));
+  const top1Hit = targetRanks.some((t) => t.rank === 1);
+  const top3Hit = targetRanks.some((t) => t.rank !== null && t.rank <= Math.min(3, topK));
+
+  // C 类：干扰项进 TopK（limit 内）即违规（同样前缀匹配）
+  const forbiddenIdMatches = (evalCase.hardNegatives || []).filter((neg) => {
+    const rank = rankOf(neg);
+    return rank !== null && rank <= limit;
+  });
+
+  // B 类：新旧序，前者须召回且 rank 小于后者
+  let orderOk: boolean | null = null;
+  const order = evalCase.expectOrder || [];
+  if (order.length >= 2) {
+    orderOk = true;
+    for (let i = 0; i < order.length - 1; i += 1) {
+      const a = rankOf(order[i]);
+      const b = rankOf(order[i + 1]);
+      if (a === null) {
+        orderOk = false;
+        break;
+      }
+      if (b !== null && a >= b) {
+        orderOk = false;
+        break;
+      }
+    }
+  }
+
+  // D 类：内容要点命中
+  const matchedContentAny = matchedTerms(evalCase.expectContentAny, joined);
+
+  // 文本禁词
+  const forbiddenMatches = matchedTerms(evalCase.forbid, joined);
+
+  // 动态加权：只计 case 实际声明的维度
+  const parts: Array<{ weight: number; value: number }> = [];
+  if (targets.length > 0) {
+    const ranks = targetRanks.map((t) => t.rank).filter((r): r is number => r !== null);
+    const bestRank = ranks.length > 0 ? Math.min(...ranks) : null;
+    const targetScore = bestRank === null ? 0 : bestRank === 1 ? 1 : bestRank <= 3 ? 0.7 : 0.4;
+    parts.push({ weight: 0.5, value: targetScore });
+  }
+  if (orderOk !== null) {
+    parts.push({ weight: 0.2, value: orderOk ? 1 : 0 });
+  }
+  const contentAny = evalCase.expectContentAny || [];
+  if (contentAny.length > 0) {
+    parts.push({ weight: 0.3, value: matchedContentAny.length / contentAny.length });
+  }
+  const totalWeight = parts.reduce((sum, p) => sum + p.weight, 0);
+  let score = totalWeight > 0
+    ? parts.reduce((sum, p) => sum + p.weight * p.value, 0) / totalWeight
+    : (results.length > 0 ? 0.5 : 0);
+  if (forbiddenIdMatches.length > 0) score -= 0.3;
+  if (forbiddenMatches.length > 0) score -= 0.3;
+  score = Math.max(0, Math.min(1, score));
+
+  return {
+    mode: "canary",
+    name: evalCase.name,
+    query: evalCase.query,
+    profile,
+    score,
+    passed: score >= 0.7 && forbiddenIdMatches.length === 0 && forbiddenMatches.length === 0,
+    hitCount: results.length,
+    targetRanks,
+    top1Hit,
+    top3Hit,
+    forbiddenIdMatches,
+    orderOk,
+    matchedContentAny,
+    forbiddenMatches,
+    topIds: ids.slice(0, 5),
+    topSnippet: results[0] ? clip(results[0].entry.text) : "-",
+    notes: evalCase.notes,
+  };
+}
+
 function joinResumeSections(response: ResumeContextResponse): string {
   return [
     response.summary,
@@ -438,6 +582,50 @@ function markdownRetrievalReport(reports: RetrievalCaseReport[]): string {
   return lines.join("\n");
 }
 
+function markdownCanaryReport(reports: CanaryCaseReport[]): string {
+  const { passed, average } = summarizeReports(reports);
+  const top1 = reports.filter((r) => r.top1Hit).length;
+  const top3 = reports.filter((r) => r.top3Hit).length;
+
+  const lines = [
+    "# RecallNest Canary Eval",
+    "",
+    `- Generated: ${new Date().toISOString()}`,
+    `- Cases: ${reports.length}`,
+    `- Passed: ${passed}/${reports.length}`,
+    `- Average score: ${formatPercent(average)}`,
+    `- Top1 hit: ${top1}/${reports.length}`,
+    `- Top3 hit: ${top3}/${reports.length}`,
+    "",
+    "| Case | Profile | Score | Pass | Top1 | Top3 | Order | Forbid |",
+    "|------|---------|-------|------|------|------|-------|--------|",
+    ...reports.map((item) =>
+      `| ${item.name} | ${item.profile} | ${(item.score * 100).toFixed(0)}% | ${item.passed ? "yes" : "no"} | ${item.top1Hit ? "yes" : "-"} | ${item.top3Hit ? "yes" : "-"} | ${item.orderOk === null ? "-" : item.orderOk ? "ok" : "bad"} | ${item.forbiddenIdMatches.length + item.forbiddenMatches.length || "-"} |`,
+    ),
+    "",
+    "## Case Notes",
+    "",
+  ];
+
+  for (const item of reports) {
+    lines.push(`### ${item.name}`);
+    lines.push(`- Query: ${item.query}`);
+    lines.push(`- Score: ${(item.score * 100).toFixed(0)}% (${item.passed ? "pass" : "fail"})`);
+    lines.push(`- Target ranks: ${item.targetRanks.map((t) => `${t.id}=${t.rank ?? "miss"}`).join(", ") || "-"}`);
+    lines.push(`- Top1 / Top3: ${item.top1Hit ? "yes" : "no"} / ${item.top3Hit ? "yes" : "no"}`);
+    lines.push(`- Order ok: ${item.orderOk === null ? "n/a" : item.orderOk}`);
+    lines.push(`- Matched content: ${item.matchedContentAny.join(", ") || "-"}`);
+    lines.push(`- Forbidden id matches: ${item.forbiddenIdMatches.join(", ") || "-"}`);
+    lines.push(`- Forbidden term matches: ${item.forbiddenMatches.join(", ") || "-"}`);
+    lines.push(`- Top ids: ${item.topIds.join(", ") || "-"}`);
+    lines.push(`- Top snippet: ${item.topSnippet}`);
+    if (item.notes) lines.push(`- Notes: ${item.notes}`);
+    lines.push("");
+  }
+
+  return lines.join("\n");
+}
+
 function markdownContinuityReport(reports: ContinuityCaseReport[]): string {
   const { passed, average } = summarizeReports(reports);
 
@@ -523,6 +711,44 @@ export async function runRetrievalEval(
   return reports;
 }
 
+export async function runCanaryEval(
+  cases: CanaryEvalCase[],
+  deps: {
+    createEvalComponents?: EvalComponentFactory;
+  } = {},
+): Promise<CanaryCaseReport[]> {
+  const config = deps.createEvalComponents ? null : loadConfig();
+  const createEvalComponentsForCase = deps.createEvalComponents || createFreshEvalComponentsFactory(config!);
+  const reports: CanaryCaseReport[] = [];
+
+  for (const [index, evalCase] of cases.entries()) {
+    if (cases.length > 1) {
+      logInfo(`[INFO] canary-eval ${index + 1}/${cases.length}: ${evalCase.name}`);
+    }
+    const profileName = evalCase.profile || "default";
+    const { retriever, accessTracker } = createEvalComponentsForCase(profileName);
+    try {
+      const results = await retriever.retrieve({
+        query: evalCase.query,
+        limit: evalCase.limit || 8,
+        scopeFilter: evalCase.scope ? [evalCase.scope] : undefined,
+        source: "auto-recall",
+      });
+      const report = scoreCanaryCase(evalCase, results);
+      reports.push(report);
+      if (cases.length > 1) {
+        logInfo(
+          `[INFO] canary-eval ${index + 1}/${cases.length} done: ${evalCase.name} ${formatPercent(report.score)} ${report.passed ? "pass" : "fail"}`,
+        );
+      }
+    } finally {
+      accessTracker.destroy();
+    }
+  }
+
+  return reports;
+}
+
 export async function runContinuityEval(
   cases: ContinuityEvalCase[],
   options: { recordObservations?: boolean; observationScope?: string; observationSource?: string } = {},
@@ -610,6 +836,25 @@ async function main() {
     return;
   }
 
+  if (args.mode === "canary") {
+    const cases = loadCases<CanaryEvalCase>("canary", args.casesPath);
+    const reports = await runCanaryEval(cases);
+    if (args.jsonMode) {
+      const payload = JSON.stringify({
+        mode: "canary",
+        generatedAt: new Date().toISOString(),
+        reports,
+      }, null, 2);
+      writeOutput(args.outputPath, payload);
+      console.log(payload);
+      return;
+    }
+    const output = markdownCanaryReport(reports);
+    writeOutput(args.outputPath, output);
+    console.log(output);
+    return;
+  }
+
   const cases = loadCases<RetrievalEvalCase>("retrieval", args.casesPath);
   const reports = await runRetrievalEval(cases);
   if (args.jsonMode) {

package/src/idempotency.ts

@@ -0,0 +1,8 @@
+import { createHash } from "node:crypto";
+
+export function buildIdempotentRecordId(prefix: string, key?: string): string | undefined {
+  const normalized = key?.replace(/\s+/g, " ").trim();
+  if (!normalized) return undefined;
+  const digest = createHash("sha256").update(normalized).digest("hex").slice(0, 32);
+  return `${prefix}-${digest}`;
+}

package/src/ingest.ts

@@ -12,6 +12,7 @@ import { readFileSync, readdirSync, existsSync, statSync, writeFileSync } from "
 import { join, basename, resolve, dirname } from "node:path";
 import { homedir } from "node:os";
 import { detectLang, tokenizeFts } from "./language-hook.js";
+import { redactSecrets } from "./pii-detector.js";
 import type { MemoryStore, MemoryEntry } from "./store.js";
 import type { Embedder } from "./embedder.js";
 import { chunkDocument, type ChunkerConfig } from "./chunker.js";
@@ -20,6 +21,7 @@ import { isConnectorOutputV1 } from "./connector-types.js";
 import { isObsidianVault, scanVault } from "./obsidian-connector.js";
 import { segmentEvents, type EventSegmenterConfig, DEFAULT_EVENT_SEGMENTER_CONFIG } from "./event-segmenter.js";
 import { isProcessed, markProcessed } from "./tracker.js";
+import * as envConfig from "./env-config.js";
 import type { LLMClient, SmartExtraction } from "./llm-client.js";
 import { resolveIngestBoundary } from "./memory-boundaries.js";
 import { isNoise } from "./noise-filter.js";
@@ -507,7 +509,7 @@ async function smartExtractBatch(
  * Tier 3.1: Check if core summary generation is enabled.
  */
 function isCoreSummaryEnabled(): boolean {
-  return process.env.RECALLNEST_CORE_SUMMARY === "true";
+  return envConfig.coreSummary();
 }
 
 /**
@@ -625,7 +627,8 @@ export async function drainPendingQueue(
   let processed = 0, errors = 0;
   for (let i = 0; i < pending.length; i += 20) {
     const batch = pending.slice(i, i + 20);
-    const texts = batch.map(c => c.text);
+    // F-3b: scrub before the LLM extraction call and downstream embedding
+    const texts = batch.map(c => redactSecrets(c.text).text);
     const extractions = await smartExtractBatch(texts, llm);
     const embeddingTexts = extractions.map(e => e.l1 || e.l0);
     const vectors = await embedder.embedBatchPassage(embeddingTexts);
@@ -860,7 +863,7 @@ function filterNoiseTurns(turns: ConversationTurn[]): ConversationTurn[] {
   return turns.filter((turn) => !isNoise(turn.text));
 }
 
-function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
+export function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
   text: string;
   timestamp: string;
   sessionId: string;
@@ -933,7 +936,18 @@ function groupTurnsIntoChunks(turns: ConversationTurn[]): Array<{
     }
   }
 
-  return chunks;
+  // P1 defence-in-depth: re-filter AFTER splitting. Long turns pass the
+  // turn-level filterNoiseTurns gate, but chunking can peel their tail into
+  // severed context-JSON fragments that the turn-level check never saw.
+  // F-3b: scrub secrets at the chunk boundary — this is the shared text
+  // finalization point for all session-ingest paths (CC/Codex/Gemini), before
+  // any chunk reaches embedding, LLM extraction, or storage.
+  return chunks
+    .filter((c) => !isNoise(c.text))
+    .map((c) => {
+      const r = redactSecrets(c.text);
+      return r.redacted > 0 ? { ...c, text: r.text } : c;
+    });
 }
 
 // ============================================================================
@@ -1027,9 +1041,14 @@ export async function ingestCodexSessions(
     errors: [],
   };
 
-  const baseDir = expandHome("~/.codex/sessions");
-  if (!existsSync(baseDir)) {
-    result.errors.push(`Codex sessions directory not found: ${baseDir}`);
+  // 同时扫 sessions（活跃）+ archived_sessions（App 内归档会把文件移到这里，否则会漏）
+  const baseDirs = [
+    expandHome("~/.codex/sessions"),
+    expandHome("~/.codex/archived_sessions"),
+  ];
+  const existingDirs = baseDirs.filter((d) => existsSync(d));
+  if (existingDirs.length === 0) {
+    result.errors.push(`Codex sessions directory not found: ${baseDirs.join(", ")}`);
     return result;
   }
 
@@ -1048,7 +1067,7 @@ export async function ingestCodexSessions(
       }
     }
   }
-  walk(baseDir);
+  for (const dir of existingDirs) walk(dir);
   allFiles.sort();
 
   // --recent: only keep files modified within N hours
@@ -1650,7 +1669,8 @@ export async function ingestMarkdownFiles(
         continue;
       }
 
-      const texts = sections.map((s) => s.text);
+      // F-3b: scrub before embedding and storage
+      const texts = sections.map((s) => redactSecrets(s.text).text);
       const batchSize = 32;
       let fileChunks = 0;
 
@@ -1788,7 +1808,8 @@ export async function ingestGenericText(
 
       const chunkResult = chunkDocument(textToChunk, CONVERSATION_CHUNK_CONFIG);
 
-      const texts = chunkResult.chunks;
+      // F-3b: scrub before embedding and storage
+      const texts = chunkResult.chunks.map((t) => redactSecrets(t).text);
       const batchSize = 32;
 
       for (let i = 0; i < texts.length; i += batchSize) {

package/src/kg-extractor.ts

@@ -8,6 +8,7 @@ import type { LLMClient } from "./llm-client.js";
 import type { KGStore } from "./kg-store.js";
 import { logInfo, logWarn } from "./stderr-log.js";
 import { detectLang, getKgPromptHook } from "./language-hook.js";
+import * as envConfig from "./env-config.js";
 
 // ============================================================================
 // Types
@@ -39,7 +40,7 @@ interface LlmExtractionResponse {
 
 /** Check if KG mode is enabled (env var) */
 export function isKGModeEnabled(): boolean {
-  return process.env.RECALLNEST_KG_MODE === "true";
+  return envConfig.kgMode();
 }
 
 // ============================================================================
@@ -163,8 +164,11 @@ export class KGExtractor {
         prompt,
       );
     } catch (err) {
+      // Transport/API failure is NOT "zero triples" — propagate so callers can
+      // distinguish (backfill must not journal it as processed). The capture
+      // path's fire-and-forget .catch() keeps memory writes unaffected.
       logWarn(`[KG] LLM extraction failed: ${String(err)}`);
-      return [];
+      throw err instanceof Error ? err : new Error(String(err));
     }
 
     if (!response?.triples || !Array.isArray(response.triples)) {