recallnest 2.3.0 → 2.6.0

package/src/decay-engine.ts

@@ -122,26 +122,63 @@ export function computeArousalBoost(
 // HP-7: Decay Exemptions
 // ============================================================================
 
+/**
+ * Categories whose entries are procedural memory (knowing *how*), not declarative.
+ *
+ * Only `patterns` qualifies: RecallNest's skill schema stores skills under
+ * `category = "patterns"` (see memory-schema.ts), so this covers both workflow
+ * patterns and skills.
+ *
+ * `cases` is deliberately NOT here. A case is a problem/solution pair with a real
+ * episodic component ("that CI failure on 07-24"), and its solution can go stale
+ * when the underlying tool changes — it *should* decay. Procedural rules do not:
+ * see the "什么该衰减" discussion in hippo [[条件失效优于时间衰减]].
+ */
+export const PROCEDURAL_CATEGORIES = new Set<string>(["patterns"]);
+
 /**
  * Check if a memory should be exempt from time decay.
  * Exempt entries keep their original score without time-based penalties.
  *
  * Exemption rules:
+ * 0. Procedural category → knowing-how doesn't fade with wall-clock time
  * 1. Core tier + high importance (≥ 0.95) → identity/correction, must not fade
  * 2. Recently accessed (within 7 days) → actively used, not stale
  * 3. Pinned → user explicitly marked as persistent
+ *
+ * @param category Optional entry category. Omitting it preserves the pre-Rule-0
+ *                 behaviour exactly, so existing callers stay valid.
  */
 export function isDecayExempt(
   metadata: string | undefined,
   importance: number,
+  category?: string,
 ): boolean {
+  // Rule 0: Procedural memory is exempt from *time* decay.
+  //
+  // Rules 1-3 all key off access statistics or explicit user action, which
+  // systematically punishes procedural memory: its defining trait is low access
+  // frequency + long validity ("restart launchd with bootout+sleep+bootstrap" may
+  // be needed once a quarter but is correct every time). Combined with `patterns`
+  // carrying the highest retrieval threshold in DEFAULT_CATEGORY_MIN_SCORES (0.45),
+  // that produced a feedback trap: ranked lower → read less → ranked lower still.
+  //
+  // Cognitive-science backing: procedural memory is non-declarative (Squire) and is
+  // the most decay-resistant human memory type — a different curve, not a slower one.
+  //
+  // NOTE: this exempts from *time* decay only. Procedural memory still expires, but
+  // by condition (its premise changed), not by age. Deliberately checked before the
+  // `!metadata` guard so entries without metadata are still covered.
+  if (category !== undefined && PROCEDURAL_CATEGORIES.has(category)) return true;
+
   if (!metadata) return false;
 
   try {
     const meta = JSON.parse(metadata) as Record<string, unknown>;
 
     // Rule 1: Core + very high importance
-    const tier = meta.tier ?? resolveTierFromMeta(meta);
+    // importance 是本函数的入参（来自 entry 列），一并传下去——否则 heuristic 读不到它。
+    const tier = meta.tier ?? resolveTierFromMeta(meta, importance);
     if (tier === "core" && importance >= 0.95) return true;
 
     // Rule 2: Accessed within last 7 days
@@ -158,12 +195,21 @@ export function isDecayExempt(
   }
 }
 
-/** Internal helper: resolve tier from parsed metadata (avoids re-parsing). */
-function resolveTierFromMeta(meta: Record<string, unknown>): MemoryTier {
+/**
+ * Internal helper: resolve tier from parsed metadata (avoids re-parsing).
+ *
+ * `entryImportance` 是 MemoryEntry 的列字段。必须由调用方传进来：importance 不存在
+ * 于 metadata（accessCount / lastAccessedAt / tier 才是 access-tracker 写在顶层的），
+ * 所以只读 meta.importance 会恒等于 0，让所有还没有显式 tier 的条目一律落到
+ * peripheral——2026-07 实测全库 3214 条走这个分支，顶层有 importance 的是 0 条。
+ */
+function resolveTierFromMeta(meta: Record<string, unknown>, entryImportance?: number): MemoryTier {
   if (meta.tier === "core" || meta.tier === "working" || meta.tier === "peripheral") {
     return meta.tier;
   }
-  const imp = typeof meta.importance === "number" ? meta.importance : 0;
+  const imp = typeof meta.importance === "number"
+    ? meta.importance
+    : (typeof entryImportance === "number" ? entryImportance : 0);
   const ac = typeof meta.accessCount === "number" ? meta.accessCount : 0;
   if (imp >= 0.95 || ac >= 10) return "core";
   if (imp >= 0.8 || ac >= 3) return "working";
@@ -178,13 +224,13 @@ function resolveTierFromMeta(meta: Record<string, unknown>): MemoryTier {
  * Determine a memory's current tier from its metadata.
  * Falls back to heuristic based on importance if no tier is stored.
  */
-export function resolveTier(metadata?: string): MemoryTier {
+export function resolveTier(metadata?: string, entryImportance?: number): MemoryTier {
   if (!metadata) return "peripheral";
 
   try {
     const meta = JSON.parse(metadata) as Record<string, unknown>;
 
-    // Explicit tier stored in metadata
+    // Explicit tier stored in metadata（access-tracker 算过一次就会写在这里）
     if (meta.tier === "core" || meta.tier === "working" || meta.tier === "peripheral") {
       return meta.tier;
     }
@@ -193,7 +239,13 @@ export function resolveTier(metadata?: string): MemoryTier {
     // - Pinned assets (importance ≥ 0.95) → core
     // - High importance (≥ 0.8) → working
     // - Everything else → peripheral
-    const importance = typeof meta.importance === "number" ? meta.importance : 0;
+    //
+    // importance 优先取调用方传入的 entry 列值。metadata 里没有这个字段——顶层住着的是
+    // accessCount / lastAccessedAt / tier（access-tracker 写的），importance 从来不在。
+    // 不传就只能退回 0，那会把所有还没被访问过的条目一律压成 peripheral。
+    const importance = typeof meta.importance === "number"
+      ? meta.importance
+      : (typeof entryImportance === "number" ? entryImportance : 0);
     const accessCount = typeof meta.accessCount === "number" ? meta.accessCount : 0;
 
     if (importance >= 0.95 || accessCount >= 10) return "core";

package/src/distill-lock.ts

@@ -18,7 +18,9 @@ import {
   utimesSync,
   existsSync,
 } from "node:fs";
+import { createHash, randomUUID } from "node:crypto";
 import { join, dirname } from "node:path";
+import * as envConfig from "./env-config.js";
 
 // ---------------------------------------------------------------------------
 // Config
@@ -34,7 +36,7 @@ export interface DistillLockConfig {
 }
 
 export const DEFAULT_DISTILL_LOCK_CONFIG: DistillLockConfig = {
-  lockPath: join(process.env.RECALLNEST_DATA_DIR || "data", "distill.lock"),
+  lockPath: join(envConfig.dataDir(), "distill.lock"),
   expireMs: 3_600_000,
   minCheckpoints: 3,
 };
@@ -56,6 +58,17 @@ function isPidAlive(pid: number): boolean {
   }
 }
 
+// P2: in-process record of the ownership token we wrote into each lock file we hold.
+// releaseLock only removes a lock whose on-disk token still matches — so if our lock
+// expired mid-run and another process legitimately reclaimed it, our finally-release
+// won't delete the new owner's lock (which would break mutual exclusion).
+const heldTokens = new Map<string, string>();
+
+/** Parse the PID prefix from a lock file body (`<pid>` legacy or `<pid>:<token>`). */
+function parsePid(content: string): number {
+  return parseInt((content.split(":")[0] ?? "").trim(), 10);
+}
+
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -66,29 +79,75 @@ function isPidAlive(pid: number): boolean {
  */
 export function acquireLock(cfg?: Partial<DistillLockConfig>): boolean {
   const { lockPath, expireMs } = resolveConfig(cfg);
+  mkdirSync(dirname(lockPath), { recursive: true });
 
-  if (existsSync(lockPath)) {
-    const existingPid = parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
-    const mtime = statSync(lockPath).mtimeMs;
-    const expired = Date.now() - mtime > expireMs;
+  // Fast path: atomic exclusive create. On POSIX the O_EXCL ("wx") open wins the
+  // cross-process race — only one of N processes creating the same lock succeeds.
+  // This closes the check-then-write TOCTOU the previous existsSync+writeFileSync
+  // form had (two processes could both see "no lock" then both write); the P0-1
+  // store-write lock relies on acquisition being truly exclusive.
+  if (tryExclusiveCreate(lockPath)) return true;
 
-    if (!isNaN(existingPid) && isPidAlive(existingPid) && !expired) {
-      return false;
-    }
-    // PID dead or lock expired — overwrite
+  // Lock file exists — decide whether it is reclaimable (dead holder or expired).
+  let existingPid = NaN;
+  let mtime = 0;
+  try {
+    existingPid = parsePid(readFileSync(lockPath, "utf-8"));
+    mtime = statSync(lockPath).mtimeMs;
+  } catch {
+    // Holder released between our failed create and this read — retry the create.
+    return tryExclusiveCreate(lockPath);
+  }
+  const expired = Date.now() - mtime > expireMs;
+  if (!isNaN(existingPid) && isPidAlive(existingPid) && !expired) {
+    return false; // held by a live process, not expired
   }
 
-  mkdirSync(dirname(lockPath), { recursive: true });
-  writeFileSync(lockPath, String(process.pid), "utf-8");
-  return true;
+  // Reclaim a stale lock: remove it, then re-create atomically. If a competing
+  // process reclaims first, our O_EXCL create fails and we concede the race.
+  try {
+    unlinkSync(lockPath);
+  } catch {
+    /* already gone — fall through to the atomic create */
+  }
+  return tryExclusiveCreate(lockPath);
+}
+
+/**
+ * Atomic O_EXCL create of the lock file. Writes a unique ownership token (`<pid>:<uuid>`)
+ * and records it so releaseLock can verify we still own the lock. false on EEXIST.
+ */
+function tryExclusiveCreate(lockPath: string): boolean {
+  const token = `${process.pid}:${randomUUID()}`;
+  try {
+    writeFileSync(lockPath, token, { encoding: "utf-8", flag: "wx" });
+    heldTokens.set(lockPath, token);
+    return true;
+  } catch (err) {
+    if ((err as { code?: unknown }).code === "EEXIST") return false;
+    throw err;
+  }
 }
 
-/** Release the distill lock by removing the lock file. */
+/**
+ * Release the distill lock. If we hold a known ownership token for this path, only
+ * remove the file when its on-disk token still matches ours — otherwise our lock was
+ * expired and reclaimed by another process, and deleting would evict the new owner.
+ * When we hold no token (e.g. a direct caller/test-written lock) fall back to removing.
+ */
 export function releaseLock(cfg?: Partial<DistillLockConfig>): void {
   const { lockPath } = resolveConfig(cfg);
-  if (existsSync(lockPath)) {
-    unlinkSync(lockPath);
+  const ourToken = heldTokens.get(lockPath);
+  heldTokens.delete(lockPath);
+  if (!existsSync(lockPath)) return;
+  if (ourToken !== undefined) {
+    try {
+      if (readFileSync(lockPath, "utf-8").trim() !== ourToken) return; // reclaimed by another owner
+    } catch {
+      return;
+    }
   }
+  unlinkSync(lockPath);
 }
 
 /**
@@ -120,3 +179,123 @@ export function getLastDistillTime(cfg?: Partial<DistillLockConfig>): number {
   if (!existsSync(lockPath)) return 0;
   return statSync(lockPath).mtimeMs;
 }
+
+// ---------------------------------------------------------------------------
+// P0-1: Named cross-process locks (store-write + maintenance families)
+//
+// distill-lock was dead code (0 src imports). P0-1 wires it up: one global
+// `store-write` lock serializes every `memories`-table write across the 11
+// mcp-server processes, and scope-keyed maintenance locks (dream/consolidate/
+// distill) + a global gc lock stop redundant concurrent maintenance runs.
+// ---------------------------------------------------------------------------
+
+/** Base directory for named locks. Defaults to <dataDir>/locks; overridable in tests. */
+export function locksDir(lockDir?: string): string {
+  return lockDir ?? join(envConfig.dataDir(), "locks");
+}
+
+/**
+ * Resolve a lock key (e.g. "store-write", "dream-<scope>", "gc-global") to a safe
+ * lock-file path. The key is sanitized for the filesystem and suffixed with a short
+ * hash of the raw key so distinct keys never collide after sanitization/truncation.
+ */
+export function lockPathForKey(key: string, lockDir?: string): string {
+  const safe = key.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
+  const hash = createHash("sha256").update(key).digest("hex").slice(0, 8);
+  return join(locksDir(lockDir), `${safe}-${hash}.lock`);
+}
+
+export interface WithLockOptions {
+  /** Stale-lock reclaim window in ms (default 600_000 = 10min). */
+  expireMs?: number;
+  /** Override the base lock directory (tests). */
+  lockDir?: string;
+  /** Behavior when another live process holds the lock (default "skip"). */
+  onBusy?: "skip" | "wait";
+  /** For onBusy="wait": total time budget before throwing (default 10_000). */
+  waitTimeoutMs?: number;
+  /** For onBusy="wait": poll interval in ms (default 50). */
+  pollMs?: number;
+}
+
+export type WithLockOutcome<T> =
+  | { ran: true; result: T }
+  | { ran: false; reason: "locked_by_another_process" };
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Run `fn` while holding a named cross-process lock, releasing it afterwards
+ * (even on throw). Maintenance-type writes use onBusy:"skip" — for a personal
+ * store, skipping a redundant dream/consolidate/gc run beats queueing it.
+ */
+export async function withLock<T>(
+  key: string,
+  fn: () => Promise<T>,
+  opts: WithLockOptions = {},
+): Promise<WithLockOutcome<T>> {
+  const lockPath = lockPathForKey(key, opts.lockDir);
+  const lockCfg = { lockPath, expireMs: opts.expireMs ?? 600_000 };
+
+  if (!acquireLock(lockCfg)) {
+    if ((opts.onBusy ?? "skip") === "skip") {
+      return { ran: false, reason: "locked_by_another_process" };
+    }
+    // wait: poll until acquired or the budget is exhausted.
+    const timeoutMs = opts.waitTimeoutMs ?? 10_000;
+    const deadline = Date.now() + timeoutMs;
+    const pollMs = opts.pollMs ?? 50;
+    let acquired = false;
+    while (Date.now() < deadline) {
+      await sleep(pollMs);
+      if (acquireLock(lockCfg)) {
+        acquired = true;
+        break;
+      }
+    }
+    if (!acquired) {
+      throw new Error(`lock '${key}' timed out after ${timeoutMs}ms (held by another process)`);
+    }
+  }
+
+  try {
+    return { ran: true, result: await fn() };
+  } finally {
+    releaseLock(lockCfg);
+  }
+}
+
+/**
+ * Run `fn` under a named write lock, waiting for the lock rather than skipping —
+ * losing a memory write would be data loss. Throws if the lock can't be acquired
+ * within the timeout. Returns fn's result directly. Used by store.upsert/storeBatch
+ * so all `memories`-table writes serialize across the 11 mcp-server processes.
+ */
+export async function withWriteLock<T>(
+  key: string,
+  fn: () => Promise<T>,
+  opts: Omit<WithLockOptions, "onBusy"> = {},
+): Promise<T> {
+  const outcome = await withLock(key, fn, { ...opts, onBusy: "wait" });
+  // onBusy:"wait" resolves to {ran:true} or throws; ran:false is unreachable here.
+  if (!outcome.ran) {
+    throw new Error(`write lock '${key}' unavailable`);
+  }
+  return outcome.result;
+}
+
+/**
+ * Stamp a lock file's mtime to now without releasing it, marking "last completed".
+ * gc (auto-gc.ts) uses this to persist a cross-process throttle timestamp in the
+ * lock file's mtime (getLastDistillTime pattern) so the 24h throttle holds across
+ * processes rather than each process tracking its own module-level timestamp.
+ */
+export function stampLock(cfg?: Partial<DistillLockConfig>): void {
+  const { lockPath } = resolveConfig(cfg);
+  mkdirSync(dirname(lockPath), { recursive: true });
+  // Create-or-overwrite so the mtime becomes now. Unlike releaseLock this leaves the
+  // file in place — it is a persistent "last run" marker (gc's cross-process throttle),
+  // read back via getLastDistillTime.
+  writeFileSync(lockPath, String(process.pid), "utf-8");
+}

package/src/doctor.ts

@@ -14,7 +14,7 @@ import { existsSync, accessSync, constants, readFileSync } from "node:fs";
 import { resolve, join } from "node:path";
 import { homedir } from "node:os";
 import { metaDir } from "./compat.js";
-import { loadConfig, expandHome, resolveEnv, findConfigPath, loadDotEnv } from "./runtime-config.js";
+import { loadConfig, expandHome, resolveEnv, findConfigPath, loadDotEnv, resolveDbPath } from "./runtime-config.js";
 import { createEmbedder, type EmbeddingConfig } from "./embedder.js";
 import { extractCanonicalKey, normalizeCanonicalKey } from "./memory-boundaries.js";
 import {
@@ -62,6 +62,13 @@ function warn(name: string, message: string, fix?: string): CheckResult {
   return { name, status: "warn", message, fix };
 }
 
+export function describeSecretPresence(
+  value: string | undefined,
+  placeholder: string,
+): string | null {
+  return value && value !== placeholder ? "set" : null;
+}
+
 function summarizeScopeInventoryLayers(report: ScopeInventoryReport): string {
   return report.layers
     .filter((layer) => layer.anomalyCount > 0 || layer.invalidCount > 0)
@@ -323,8 +330,8 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
   // 3. .env + JINA_API_KEY
   loadDotEnv();
   const jinaKey = process.env.JINA_API_KEY;
-  if (jinaKey && jinaKey !== "your_jina_api_key_here") {
-    results.push(pass("JINA_API_KEY", `set (${jinaKey.slice(0, 8)}...)`));
+  if (describeSecretPresence(jinaKey, "your_jina_api_key_here")) {
+    results.push(pass("JINA_API_KEY", "set"));
   } else if (options.ci) {
     results.push(warn(
       "JINA_API_KEY",
@@ -349,7 +356,9 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
   }
 
   // 5. LanceDB data directory
-  const dbPath = resolve(configPath ? join(configPath, "..") : process.cwd(), expandHome(config.dbPath));
+  // Must resolve exactly like the server does — a doctor that reports on a
+  // different directory than the one actually in use is worse than no doctor.
+  const dbPath = resolveDbPath(config);
   try {
     validateStoragePath(dbPath);
     results.push(pass("Data directory", dbPath));