recallnest 2.3.0 → 2.6.0

package/src/retroactive-boost.ts

@@ -21,6 +21,7 @@
  */
 
 import type { MemoryStore, MemoryEntry } from "./store.js";
+import { isActiveMemory } from "./memory-evolution.js";
 
 // ---------------------------------------------------------------------------
 // Config
@@ -85,12 +86,15 @@ export async function retroactiveBoost(
   }
 
   // Find related entries in the same scope
-  const candidates = await store.vectorSearch(
+  // Superseded belief-history rows come back from vectorSearch too. Boosting one is wasted
+  // — they are excluded from retrieval anyway — and it burns maxBoostPerTrigger budget that
+  // live entries should be getting.
+  const candidates = (await store.vectorSearch(
     newEntry.vector,
     config.maxBoostPerTrigger * 3, // fetch more, filter down
     config.similarityThreshold,
     [newEntry.scope],
-  );
+  )).filter((candidate) => isActiveMemory(candidate.entry.metadata));
 
   const now = Date.now();
   const minAgeMs = config.minAgeDays * 86_400_000;

package/src/runtime-config.ts

@@ -8,9 +8,11 @@ import { createEmbedder, getVectorDimensions, type EmbeddingConfig } from "./emb
 import { createRetriever, type RetrievalConfig, DEFAULT_RETRIEVAL_CONFIG } from "./retriever.js";
 import { applyRetrievalProfile } from "./retrieval-profiles.js";
 import { AccessTracker } from "./access-tracker.js";
+import { createAuditLogger } from "./audit-log.js";
 import { FrequencyTracker } from "./frequency-tracker.js";
 import { createLLMClient, type LLMClient, type LLMConfig } from "./llm-client.js";
 import { logInfo } from "./stderr-log.js";
+import * as envConfig from "./env-config.js";
 
 export type RecallMode = "full" | "light" | "summary" | "off";
 
@@ -33,6 +35,12 @@ export interface LocalMemoryConfig {
   };
   sources: Record<string, { path: string; glob: string; description: string }>;
   retrieval?: Partial<RetrievalConfig>;
+  /**
+   * Opt-in cross-scope sidecar map for search_memory.
+   * Main scoped search remains isolated; callers must pass includeRelatedScopes
+   * before configured related scopes are queried and shown separately.
+   */
+  scopeRelations?: Record<string, string[]>;
   /**
    * Default depth for auto-recall injection.
    * - "full": inject complete text (default, backward compatible)
@@ -60,7 +68,20 @@ export function loadDotEnv(): void {
 }
 
 export function findConfigPath(): string {
-  if (process.env.LOCAL_MEMORY_CONFIG) return resolve(process.env.LOCAL_MEMORY_CONFIG);
+  if (process.env.LOCAL_MEMORY_CONFIG) {
+    const explicit = resolve(process.env.LOCAL_MEMORY_CONFIG);
+    // Check here rather than letting readFileSync throw a bare ENOENT: scripts
+    // resolve the database at module load, so a stale env var would otherwise
+    // surface as an unexplained crash before any of their own output appears.
+    if (!existsSync(explicit)) {
+      throw new Error(
+        `LOCAL_MEMORY_CONFIG points to a file that does not exist:\n` +
+        `  ${explicit}\n` +
+        `  Fix: correct the env var, or unset it to use the repo's config.json.`
+      );
+    }
+    return explicit;
+  }
 
   const localConfig = resolve(metaDir(import.meta), "../config.json");
   if (existsSync(localConfig)) return localConfig;
@@ -96,8 +117,35 @@ export function expandHome(p: string): string {
   return p;
 }
 
+/**
+ * Single source of truth for "which LanceDB directory do we operate on".
+ *
+ * Every entry point — MCP server, CLI, and maintenance scripts — must resolve
+ * the database through this function. Scripts that hardcode "./data/lancedb"
+ * or read a non-existent config key silently operate on a *different* database
+ * than the server does, so a health check, cleanup, or backfill can report on
+ * an empty directory while the real store sits elsewhere.
+ *
+ * Relative dbPath values resolve against the repo root (not process.cwd()), so
+ * a script's behaviour does not depend on where it was invoked from.
+ *
+ * Pass an already-loaded config where you have one. Calling this with no
+ * argument reads config.json from disk — convenient at a script's top level,
+ * but it means a missing or misconfigured file surfaces here, before the
+ * script's own output. findConfigPath() raises an actionable error for that.
+ */
+export function resolveDbPath(config?: LocalMemoryConfig): string {
+  const cfg = config ?? loadConfig();
+  return resolve(metaDir(import.meta), "..", expandHome(cfg.dbPath));
+}
+
+/** Sidecar directory (frequency-stats.json etc.) that lives beside the database. */
+export function resolveDataDir(config?: LocalMemoryConfig): string {
+  return resolve(resolveDbPath(config), "..");
+}
+
 export function createComponents(config: LocalMemoryConfig, profileName?: string) {
-  const dbPath = resolve(metaDir(import.meta), "..", expandHome(config.dbPath));
+  const dbPath = resolveDbPath(config);
   validateStoragePath(dbPath);
 
   const embeddingConfig: EmbeddingConfig = {
@@ -112,24 +160,48 @@ export function createComponents(config: LocalMemoryConfig, profileName?: string
 
   const embedder = createEmbedder(embeddingConfig);
   const store = new MemoryStore({ dbPath, vectorDim: embedder.dimensions });
-  const baseRetrievalConfig = {
+  const mergedRetrievalConfig = {
     ...DEFAULT_RETRIEVAL_CONFIG,
     ...(config.retrieval || {}),
   };
+  // rerankApiKey 与 embedding.apiKey 同款支持 ${ENV_VAR} 引用。此前 retrieval 段是整体
+  // 展开、不过 resolveEnv，导致这个字段只能写明文——等于逼配置文件存凭证，是 rerank
+  // 一直没被启用的障碍之一。
+  const baseRetrievalConfig = mergedRetrievalConfig.rerankApiKey
+    ? { ...mergedRetrievalConfig, rerankApiKey: resolveEnv(mergedRetrievalConfig.rerankApiKey) }
+    : mergedRetrievalConfig;
   const { profile, config: retrieverConfig } = applyRetrievalProfile(baseRetrievalConfig, profileName);
   const retriever = createRetriever(store, embedder, retrieverConfig);
 
   // Attach access tracker for reinforcement-based decay
   const accessTracker = new AccessTracker(store);
+  accessTracker.registerExitFlush();
   retriever.setAccessTracker(accessTracker);
 
   // P0.2: Attach frequency tracker for hit-count based boosting
-  const dataDir = resolve(metaDir(import.meta), "..", expandHome(config.dbPath), "..");
+  const dataDir = resolveDataDir(config);
   const frequencyTracker = new FrequencyTracker({
     filePath: join(dataDir, "frequency-stats.json"),
   });
   retriever.setFrequencyTracker(frequencyTracker);
 
+  // F-1: 补接 audit logger，让 retrieve 操作真的被记录。
+  //
+  // 2026-08-12 修复：`setAuditLogger`（retriever.ts:749）自定义以来**全仓零调用者**，
+  // 而 retriever.ts:897 的 `this.auditLogger?.log({ operation: "retrieve", ... })`
+  // 早就写好了 —— 可选链让它永远静默 no-op。结果是 audit-log.ts 文件头注释声称
+  // "record every store/update/delete/retrieve operation"、AuditOperation 类型里
+  // 也列着 "retrieve"，而生产 audit.jsonl 里 2841 条记录**零条 retrieve**。
+  //
+  // 同族的 setAccessTracker / setFrequencyTracker 就在上面两行，都接了，唯独这个漏了。
+  // 与同日修的 dream 末尾 resetWriteCount 漏传 scope 完全同型：接线漏一处、运行时
+  // 不报错、静默失效 —— 差别只在那个是漏参数、这个是漏调用。
+  //
+  // simplified: audit.jsonl 无轮转机制（修复时 644K / 2841 行，约 230 字节一行）。
+  // 接上 retrieve 后增速会明显上升，因为检索远比写入频繁。
+  // 升级触发条件：文件超过约 50MB 时加按月轮转，别等到它拖慢 append。
+  retriever.setAuditLogger(createAuditLogger(join(dataDir, "audit.jsonl")));
+
   // Create LLM client if configured (optional, graceful)
   let llm: LLMClient | null = null;
   if (config.llm) {
@@ -143,7 +215,7 @@ export function createComponents(config: LocalMemoryConfig, profileName?: string
 }
 
 export function createStoreOnly(config: LocalMemoryConfig): MemoryStore {
-  const dbPath = resolve(metaDir(import.meta), "..", expandHome(config.dbPath));
+  const dbPath = resolveDbPath(config);
   validateStoragePath(dbPath);
   return new MemoryStore({
     dbPath,
@@ -179,7 +251,7 @@ export function resolveRecallMode(config: LocalMemoryConfig, perCallOverride?: s
   if (perCallOverride && VALID_RECALL_MODES.includes(perCallOverride as RecallMode)) {
     return perCallOverride as RecallMode;
   }
-  const envMode = process.env.RECALLNEST_RECALL_MODE;
+  const envMode = envConfig.recallModeRaw();
   if (envMode && VALID_RECALL_MODES.includes(envMode as RecallMode)) {
     return envMode as RecallMode;
   }

package/src/scope-suggester.ts

@@ -0,0 +1,113 @@
+/**
+ * P1-A: scope 拼写漂移提示(FAILURES.md 5-13 retrieval gap)。
+ *
+ * matchesScopeFilter 走精确/前缀匹配,拼错或惯例不一致的 scope(`recallnst`、
+ * `cc_55bcbfb3`)会静默返回 0 hits,使用者无从知道是"真没有"还是"scope 写错"。
+ * 本模块在 0-hit 且显式传了 scope 时给出相近 scope 提示。
+ *
+ * 只提示、不自动 fallback 重查——自动改写会静默改变查询语义,agent/用户
+ * 看到提示后自行决定。
+ */
+
+// ---------------------------------------------------------------------------
+// Pure matching
+// ---------------------------------------------------------------------------
+
+/** 归一化:小写 + 吃掉 -/_/空格(惯例差异:`cc-foo` vs `cc_foo` vs `ccfoo`)。 */
+export function normalizeScope(s: string): string {
+  return s.toLowerCase().replace(/[-_\s]+/g, "");
+}
+
+/** Two-row Levenshtein with early exit — 距离超过 max 直接返回 max+1。 */
+export function boundedLevenshtein(a: string, b: string, max: number): number {
+  if (Math.abs(a.length - b.length) > max) return max + 1;
+  if (a === b) return 0;
+
+  let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+  let curr = new Array<number>(b.length + 1);
+
+  for (let i = 1; i <= a.length; i++) {
+    curr[0] = i;
+    let rowMin = curr[0];
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
+      if (curr[j] < rowMin) rowMin = curr[j];
+    }
+    if (rowMin > max) return max + 1;
+    [prev, curr] = [curr, prev];
+  }
+  return prev[b.length];
+}
+
+/**
+ * 对输入 scope 给出相近的已知 scope。
+ * 两级:① normalize 后精确相等 ② 前缀包含(双向)或编辑距离 ≤ 2。
+ * 候选应同时包含完整 scope 与其 prefix 家族(`cc:55bc...` 的 `cc`)——
+ * 拼错的通常是短 prefix,对长 session scope 做编辑距离没有意义。
+ */
+export function suggestScopes(input: string, knownScopes: string[], maxSuggestions = 3): string[] {
+  const norm = normalizeScope(input);
+  if (norm.length === 0) return [];
+
+  const exact: string[] = [];
+  const close: Array<{ scope: string; rank: number }> = [];
+
+  for (const known of knownScopes) {
+    const kn = normalizeScope(known);
+    if (kn === norm) {
+      exact.push(known);
+      continue;
+    }
+    if (kn.startsWith(norm) || norm.startsWith(kn)) {
+      close.push({ scope: known, rank: 0.5 });
+      continue;
+    }
+    const d = boundedLevenshtein(norm, kn, 2);
+    if (d <= 2) close.push({ scope: known, rank: d });
+  }
+
+  if (exact.length > 0) return exact.slice(0, maxSuggestions);
+  return close
+    .sort((a, b) => a.rank - b.rank || a.scope.length - b.scope.length)
+    .map(c => c.scope)
+    .slice(0, maxSuggestions);
+}
+
+// ---------------------------------------------------------------------------
+// Cached provider
+// ---------------------------------------------------------------------------
+
+/**
+ * 带 60s 缓存的建议器工厂——known scopes 来自 store.stats().scopeCounts,
+ * 全表聚合不便宜,0-hit 提示是低频路径,短缓存足够新鲜。
+ * 候选 = prefix 家族(冒号前段,去重) + 完整 scope 串。
+ */
+export function createScopeSuggester(
+  getScopeCounts: () => Promise<Record<string, number>>,
+  ttlMs = 60_000,
+): (input: string) => Promise<string[]> {
+  let cache: { at: number; candidates: string[] } | null = null;
+
+  return async function suggest(input: string): Promise<string[]> {
+    const now = Date.now();
+    if (!cache || now - cache.at > ttlMs) {
+      const counts = await getScopeCounts();
+      const keys = Object.keys(counts);
+      const prefixes = new Set<string>();
+      for (const k of keys) {
+        const i = k.indexOf(":");
+        if (i > 0) prefixes.add(k.slice(0, i));
+      }
+      // prefix 家族在前:作为查询 scope 它们最常被手敲、也最常被拼错
+      cache = { at: now, candidates: [...prefixes, ...keys] };
+    }
+    return suggestScopes(input, cache.candidates);
+  };
+}
+
+/** 0-hit 提示行(空建议返回空串,caller 原样拼接即可)。 */
+export function formatScopeSuggestion(inputScope: string, suggestions: string[]): string {
+  if (suggestions.length === 0) return "";
+  return `⚠️ scope '${inputScope}' 命中 0 条,相近 scope: [${suggestions.join(", ")}]`;
+}

package/src/session-distiller.ts

@@ -7,6 +7,8 @@ import type { AuditLogger } from "./audit-log.js";
 import type { StoredMemoryRecord, DurableMemoryCategory } from "./memory-schema.js";
 import { persistMemory } from "./capture-engine.js";
 import { detectLang, getSessionPromptHook } from "./language-hook.js";
+import { redactSecrets } from "./pii-detector.js";
+import { withWriteLock } from "./distill-lock.js";
 
 // ============================================================================
 // Types
@@ -71,6 +73,7 @@ const DEFAULT_DIMENSION_LABELS: Record<string, string> = {
   errors_and_fixes: "Errors and fix records",
   problem_solving: "Problem solving process",
   user_quotes: "User original quotes preserved",
+  user_profile: "Stable facts about the user: identity, role, background, long-term context",
   unfinished_tasks: "Unfinished tasks",
   current_state: "Current work state",
   next_steps: "Suggested next steps",
@@ -93,6 +96,15 @@ const DIMENSION_TO_MEMORY: Record<string, DimensionMapping> = {
   errors_and_fixes: { category: "cases", importance: 0.7 },
   problem_solving: { category: "patterns", importance: 0.8 },
   user_quotes: { category: "preferences", importance: 0.7 },
+  // user_profile → profile：让 distill 能沉淀"用户是谁"的稳定身份事实（不只是偏好）。
+  // distill 走 source=agent，不被 resolveIngestBoundary 降级，直接进 durable profile。
+  user_profile: { category: "profile", importance: 0.8 },
+  // 2026-05-13 P1.2: 加 unfinished_tasks + next_steps 持久化映射
+  // LLM 早在 DEFAULT_DIMENSION_LABELS 行 74/76 已产出这两条，但缺映射被丢弃
+  // 都映射到 cases/patterns 走 tag 兜底（extractAndPersist 自动加 tags: ["session_distill", dimKey]）
+  // 未来如要 unresolved-first 召回，按 tags 含 "unfinished_tasks" 过滤即可
+  unfinished_tasks: { category: "cases", importance: 0.75 },
+  next_steps: { category: "patterns", importance: 0.65 },
 };
 
 // ============================================================================
@@ -158,8 +170,10 @@ export async function summarizeSession(
   messages: ConversationMessage[],
   llm: LLMClient,
 ): Promise<SummaryResult> {
+  // F-3b: Redact per-message BEFORE the 500-char cut — truncating first could
+  // slice a secret in half and leak its prefix past the redaction regexes.
   const conversationText = messages
-    .map((m) => `[${m.role}${m.tool_name ? `:${m.tool_name}` : ""}] ${m.content.slice(0, 500)}`)
+    .map((m) => `[${m.role}${m.tool_name ? `:${m.tool_name}` : ""}] ${redactSecrets(m.content).text.slice(0, 500)}`)
     .join("\n")
     .slice(0, 8000);
 
@@ -295,7 +309,14 @@ export async function distillSession(
   };
 
   if (shouldPersist && Object.keys(summaryResult.dimensions).length > 0) {
-    persistResult = await extractAndPersist(summaryResult, deps, scope);
+    // P0-1: serialize the persist phase per scope across processes. Wait (not skip) —
+    // skipping would drop this session's distilled knowledge when two sessions distill
+    // the same scope at once. Writes are fast and store-write already makes each write
+    // idempotent, so the brief wait is lossless; microcompact/summary above run unlocked.
+    persistResult = await withWriteLock(
+      `distill-${scope}`,
+      () => extractAndPersist(summaryResult, deps, scope),
+    );
   }
 
   return {

package/src/session-engine.ts

@@ -6,6 +6,7 @@ import {
   type SessionCheckpointRecord,
   SessionCheckpointRecordSchema,
 } from "./session-schema.js";
+import { buildIdempotentRecordId } from "./idempotency.js";
 
 const REPO_STATE_PATTERN = /\bgit status\b|未提交|staged|uncommitted|已修改文件|modified files?|untracked|新增文件|dirty repo|dirty worktree/iu;
 const SENTENCE_SPLIT_PATTERN = /(?:\r?\n)+|(?<=[。！？!?;；])\s+/u;
@@ -141,7 +142,7 @@ export function buildSessionCheckpointResult(rawInput: unknown): SessionCheckpoi
 
   const record = SessionCheckpointRecordSchema.parse({
     ...sanitizedInput,
-    checkpointId: randomUUID(),
+    checkpointId: buildIdempotentRecordId("chk", sanitizedInput.idempotencyKey) ?? randomUUID(),
     resolvedScope: resolveCheckpointScope(sanitizedInput),
   });
 

package/src/session-schema.ts

@@ -14,6 +14,7 @@ export const SessionCheckpointInputSchema = z.object({
   nextActions: normalizedStringListSchema("nextActions", 6, 200),
   entities: normalizedStringListSchema("entities", 8, 120),
   files: normalizedStringListSchema("files", 12, 220),
+  idempotencyKey: optionalBoundedStringSchema(160),
   updatedAt: z.string().datetime().default(() => new Date().toISOString()),
 });
 

package/src/session-store.ts

@@ -78,6 +78,11 @@ export class SessionCheckpointStore {
     const timestampToken = parsed.updatedAt.replace(/[:.]/g, "-");
     const path = join(this.dataDir, `${timestampToken}-${parsed.checkpointId}.json`);
     try {
+      const suffix = `-${parsed.checkpointId}.json`;
+      for (const name of readdirSync(this.dataDir).filter((item) => item.endsWith(suffix))) {
+        const existingPath = join(this.dataDir, name);
+        if (existingPath !== path) unlinkSync(existingPath);
+      }
       writeFileSync(path, JSON.stringify(parsed, null, 2) + "\n");
     } catch (err) {
       console.error("[recallnest] Failed to save session checkpoint:", err instanceof Error ? err.message : String(err));

package/src/skill-engine.ts

@@ -177,6 +177,84 @@ export async function persistSkill(
   };
 }
 
+type OutcomeDeps = Pick<MemoryStore, "getById" | "update">;
+
+export type SkillOutcomeUpdate =
+  | { updated: true; successCount: number; failureCount: number; lastRefinedAt: string }
+  | { updated: false; reason: "skill_not_found" | "not_a_skill" | "metadata_parse_error" | "skill_metadata_missing" | "store_update_failed" };
+
+/**
+ * Record a skill usage outcome by incrementing successCount/failureCount on the skill's metadata.
+ *
+ * Mapping: `success` → successCount +1; everything else (`failure`/`corrected`/`missed`) → failureCount +1.
+ * Returns `{ updated: false, reason }` for missing/corrupt skills without throwing — callers can log and continue.
+ */
+export async function recordSkillOutcome(
+  store: OutcomeDeps,
+  skillId: string,
+  outcome: "success" | "failure" | "corrected" | "missed",
+): Promise<SkillOutcomeUpdate> {
+  const entry = await store.getById(skillId);
+  if (!entry) {
+    return { updated: false, reason: "skill_not_found" };
+  }
+
+  if (entry.category !== SKILL_CATEGORY) {
+    return { updated: false, reason: "not_a_skill" };
+  }
+
+  let meta: Record<string, unknown>;
+  try {
+    meta = JSON.parse(entry.metadata || "{}");
+  } catch {
+    return { updated: false, reason: "metadata_parse_error" };
+  }
+
+  const skill = meta.skill;
+  if (!skill || typeof skill !== "object") {
+    return { updated: false, reason: "skill_metadata_missing" };
+  }
+
+  const s = skill as Record<string, unknown>;
+  const currentSuccess = typeof s.successCount === "number" ? s.successCount : 0;
+  const currentFailure = typeof s.failureCount === "number" ? s.failureCount : 0;
+
+  const nextSuccessCount = outcome === "success" ? currentSuccess + 1 : currentSuccess;
+  const nextFailureCount = outcome === "success" ? currentFailure : currentFailure + 1;
+  const nextLastRefinedAt = new Date().toISOString();
+
+  const nextSkill = {
+    ...s,
+    successCount: nextSuccessCount,
+    failureCount: nextFailureCount,
+    lastRefinedAt: nextLastRefinedAt,
+  };
+
+  const updatedMetadata = JSON.stringify({
+    ...meta,
+    skill: nextSkill,
+  });
+
+  // Use entry.id (full UUID resolved by getById's prefix lookup) for update,
+  // not skillId (caller-provided, may be 8+ hex prefix). This avoids ambiguous-prefix
+  // mutation risk: if a prefix matched here, update would also need to do prefix
+  // resolution, doubling the chance of writing to the wrong entry.
+  const updated = await store.update(entry.id, {
+    metadata: updatedMetadata,
+  });
+
+  if (!updated) {
+    return { updated: false, reason: "store_update_failed" };
+  }
+
+  return {
+    updated: true,
+    successCount: nextSuccessCount,
+    failureCount: nextFailureCount,
+    lastRefinedAt: nextLastRefinedAt,
+  };
+}
+
 export async function retrieveSkills(
   store: SearchDeps,
   embedder: EmbedDeps,