recallnest 2.3.0 → 2.6.0

package/src/auto-gc.ts

@@ -9,16 +9,20 @@
  * instead of raw age/importance heuristics. Archive-first, delete-never.
  */
 
-import type { MemoryStore } from "./store.js";
+import type { MemoryStorePort } from "./memory-store-port.js";
 import {
   parseEvolution,
   computeDecayScore,
+  computeUsageAdjustedDecayScore,
   buildArchivedMetadata,
   isActiveMemory,
 } from "./memory-evolution.js";
 import { loadRetentionPolicy, shouldArchiveByPolicy } from "./retention-policy.js";
 import type { RetentionPolicy } from "./retention-policy.js";
 import type { AuditLogger } from "./audit-log.js";
+import * as envConfig from "./env-config.js";
+import { isUsageSignalActive } from "./usage-tracker.js";
+import { acquireLock, releaseLock, getLastDistillTime, stampLock, lockPathForKey } from "./distill-lock.js";
 
 // ---------------------------------------------------------------------------
 // Config
@@ -73,48 +77,89 @@ export interface GcResult {
  * - Not pinned (importance >= 0.95 is considered pinned)
  */
 export async function maybeRunGc(
-  store: MemoryStore,
+  store: MemoryStorePort,
   config: AutoGcConfig = DEFAULT_AUTO_GC_CONFIG,
   retentionConfigDir?: string,
   auditLogger?: AuditLogger,
 ): Promise<GcResult> {
   const stats = await store.stats();
-  const totalMemories = stats.total ?? 0;
+  const totalMemories = stats.totalCount ?? 0;
 
   // Condition 1: enough memories to warrant GC
   if (totalMemories < config.minMemoryCount) {
     return { triggered: false, reason: "below_memory_threshold", archivedCount: 0, totalChecked: 0 };
   }
 
-  // Condition 2: enough time since last GC
-  const hoursSinceLastGc = (Date.now() - lastGcTimestamp) / 3_600_000;
+  // Condition 2: enough time since last GC. P0-1: the throttle is cross-process now —
+  // take the max of this process's in-memory timestamp and the shared gc-last stamp
+  // file's mtime. The old module-level var alone let each of the 11 mcp-server
+  // processes run gc on its own independent 24h clock.
+  const gcStampPath = lockPathForKey("gc-last");
+  const lastGc = Math.max(lastGcTimestamp, getLastDistillTime({ lockPath: gcStampPath }));
+  const hoursSinceLastGc = (Date.now() - lastGc) / 3_600_000;
   if (hoursSinceLastGc < config.minHoursSinceLastGc) {
     return { triggered: false, reason: "too_soon", archivedCount: 0, totalChecked: 0 };
   }
 
+  // Concurrency guard: only one process scans the full corpus at a time. A process that
+  // passed the throttle in the same instant loses the O_EXCL race here and skips.
+  const gcRunLock = { lockPath: lockPathForKey("gc-run"), expireMs: 30 * 60_000 };
+  if (!acquireLock(gcRunLock)) {
+    return { triggered: false, reason: "locked_by_another_process", archivedCount: 0, totalChecked: 0 };
+  }
+
   // All conditions met — run GC
-  lastGcTimestamp = Date.now();
+  lastGcTimestamp = Date.now(); // in-process immediate throttle
 
+  try {
+    const { archivedCount, totalChecked } = await runGcScan(store, config, retentionConfigDir, auditLogger);
+    // Stamp completion into the shared throttle marker (create-or-touch → mtime = now)
+    // so the other processes observe "last run" via getLastDistillTime.
+    stampLock({ lockPath: gcStampPath });
+    return { triggered: true, archivedCount, totalChecked };
+  } finally {
+    releaseLock(gcRunLock);
+  }
+}
+
+/**
+ * The full-corpus archive scan — the body of a triggered GC run. Extracted from
+ * maybeRunGc so the P0-1 lock/throttle wiring stays readable; the scan logic itself
+ * is unchanged.
+ */
+async function runGcScan(
+  store: MemoryStorePort,
+  config: AutoGcConfig,
+  retentionConfigDir: string | undefined,
+  auditLogger: AuditLogger | undefined,
+): Promise<{ archivedCount: number; totalChecked: number }> {
   const now = Date.now();
   let archivedCount = 0;
   let totalChecked = 0;
 
-  // Scan memories by listing them
-  const entries = await store.list({ limit: 5000 });
-  totalChecked = entries.length;
+  // Full-corpus scan, page by page. The previous implementation listed only
+  // the newest 5000 rows (list() sorts by timestamp desc before slicing) —
+  // old low-value memories, the very target of GC, never entered the scan
+  // window on a 99K+ corpus. Two passes keep memory at one page instead of
+  // the whole corpus; pass 2 mutates rows (archive), which may shift fragment
+  // scan order — a row missed this run is caught by the next one.
+  const GC_PAGE_SIZE = 2000;
 
-  // Pre-compute per-scope active memory counts and cache retention policies.
-  // This avoids repeated file reads inside the loop.
+  // Pass 1: per-scope active memory counts for retention policies.
   const activeCounts = new Map<string, number>();
   const policyCache = new Map<string, RetentionPolicy>();
   const scopesSeen = new Set<string>();
 
-  for (const entry of entries) {
-    const scope = entry.scope ?? "";
-    scopesSeen.add(scope);
-    if (isActiveMemory(entry.metadata)) {
-      activeCounts.set(scope, (activeCounts.get(scope) ?? 0) + 1);
+  for (let offset = 0; ; offset += GC_PAGE_SIZE) {
+    const page = await store.listPage({ limit: GC_PAGE_SIZE, offset });
+    for (const entry of page) {
+      const scope = entry.scope ?? "";
+      scopesSeen.add(scope);
+      if (isActiveMemory(entry.metadata)) {
+        activeCounts.set(scope, (activeCounts.get(scope) ?? 0) + 1);
+      }
     }
+    if (page.length < GC_PAGE_SIZE) break;
   }
 
   // Batch-load retention policies for all scopes (one file read per scope)
@@ -122,8 +167,15 @@ export async function maybeRunGc(
     policyCache.set(scope, loadRetentionPolicy(scope, retentionConfigDir));
   }
 
-  for (const entry of entries) {
-    if (archivedCount >= config.maxArchivePerRun) break;
+  // Pass 2: archive sweep.
+  const useUsageAdjustedDecay = envConfig.usageDecay() && isUsageSignalActive();
+  outer:
+  for (let offset = 0; ; offset += GC_PAGE_SIZE) {
+    const page = await store.listPage({ limit: GC_PAGE_SIZE, offset });
+    totalChecked += page.length;
+
+  for (const entry of page) {
+    if (archivedCount >= config.maxArchivePerRun) break outer;
 
     // Only archive active memories
     if (!isActiveMemory(entry.metadata)) continue;
@@ -139,7 +191,9 @@ export async function maybeRunGc(
     let shouldArchive = false;
     if (ageDays >= config.minAgeDays) {
       const evo = parseEvolution(entry.metadata, entry.timestamp);
-      let decayScore = computeDecayScore(evo, importance, now, entry.metadata);
+      let decayScore = useUsageAdjustedDecay
+        ? computeUsageAdjustedDecayScore(evo, importance, now, entry.metadata)
+        : computeDecayScore(evo, importance, now, entry.metadata);
       // F3: Expired memories (validUntil < now) get 2x decay acceleration
       if (evo.validUntil != null && evo.validUntil < now) {
         decayScore *= 0.5; // Halve the score → archives faster
@@ -182,12 +236,22 @@ export async function maybeRunGc(
     }
   }
 
-  return { triggered: true, archivedCount, totalChecked };
+    if (page.length < GC_PAGE_SIZE) break;
+  }
+
+  return { archivedCount, totalChecked };
 }
 
 /**
- * Reset last GC timestamp (for testing).
+ * Reset last GC timestamp (for testing). Clears both the in-memory timestamp and the
+ * persisted cross-process throttle marker / run lock so tests start from a clean slate.
  */
 export function resetGcTimestamp(): void {
   lastGcTimestamp = 0;
+  try {
+    releaseLock({ lockPath: lockPathForKey("gc-last") });
+    releaseLock({ lockPath: lockPathForKey("gc-run") });
+  } catch {
+    /* ignore — nothing to clear */
+  }
 }

package/src/auto-recall.ts

@@ -1,6 +1,12 @@
 import { composeResumeContext, type ResumeContextDeps } from "./context-composer.js";
 import { resolveTier } from "./decay-engine.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
+import {
+  classifyRecallGate,
+  logRecallGateShadow,
+  resolveRecallGateMode,
+  type RecallGateResult,
+} from "./recall-gate.js";
 import { governResults, truncateQuery, type GovernorConfig, type GovernorSession } from "./recall-governor.js";
 import type { RetrievalResult } from "./retriever.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
@@ -33,6 +39,20 @@ export interface AutoRecallResponse {
   searchSkippedReason?: string;
 }
 
+/** Minimal valid ResumeContextResponse for enforce-mode skip-all short-circuits. */
+function emptyResumeContext(): ResumeContextResponse {
+  return {
+    summary: "",
+    stableContext: [],
+    relevantPatterns: [],
+    recentCases: [],
+    injectionHint: "user_attachment",
+    ephemeral: true,
+    responseMode: "default",
+    generatedAt: new Date().toISOString(),
+  };
+}
+
 export async function runAutoRecall(
   deps: ResumeContextDeps,
   request: AutoRecallRequest,
@@ -46,6 +66,34 @@ export async function runAutoRecall(
     throw new Error("message is required");
   }
 
+  // Recall gate (mlp adaptive-retrieval borrow, three-way per Codex review).
+  // Sits BEFORE composeResumeContext — compose itself hits the retriever, so a
+  // gate placed after it would save nothing. Default mode is observe: verdicts
+  // are computed and shadow-logged but behavior is unchanged until real-sample
+  // false-positive rates have been reviewed (shared-behaviors §5).
+  const gateMode = resolveRecallGateMode(request.env);
+  let gateVerdict: RecallGateResult | undefined;
+  if (gateMode !== "off") {
+    gateVerdict = classifyRecallGate(message);
+    logRecallGateShadow({
+      ts: new Date().toISOString(),
+      decision: gateVerdict.decision,
+      ruleId: gateVerdict.ruleId,
+      msgLen: message.length,
+      mode: gateMode,
+      source: request.operation,
+    });
+    if (gateMode === "enforce" && gateVerdict.decision === "skip-all") {
+      return {
+        mode: "resume-only",
+        resolvedScope: undefined,
+        resume: emptyResumeContext(),
+        results: [],
+        searchSkippedReason: `recall-gate: skip-all (${gateVerdict.ruleId})`,
+      };
+    }
+  }
+
   const scopeSelection = resolveScopeSelection({
     scope: request.scope,
     sessionId: request.sessionId,
@@ -67,6 +115,18 @@ export async function runAutoRecall(
     profile: request.profile,
   });
 
+  // Continuity nudges ("继续", "下一步") want the resume context they just got,
+  // not an embedding search over their own two characters.
+  if (gateMode === "enforce" && gateVerdict?.decision === "resume-only") {
+    return {
+      mode: "resume-only",
+      resolvedScope: resume.resolvedScope || resume.latestCheckpoint?.resolvedScope || scopeSelection.resolvedScope,
+      resume,
+      results: [],
+      searchSkippedReason: `recall-gate: resume-only (${gateVerdict.ruleId})`,
+    };
+  }
+
   const resolvedScope = request.allScopes
     ? undefined
     : (resume.resolvedScope || resume.latestCheckpoint?.resolvedScope || scopeSelection.resolvedScope);
@@ -101,7 +161,7 @@ export async function runAutoRecall(
   const promoted: RetrievalResult[] = [];
   const demoted: RetrievalResult[] = [];
   for (const r of rawResults) {
-    const tier = resolveTier(r.entry.metadata);
+    const tier = resolveTier(r.entry.metadata, r.entry.importance);
     if (tier === "core" || tier === "working") {
       promoted.push(r);
     } else {

package/src/belief-history.ts

@@ -0,0 +1,164 @@
+/**
+ * Belief history — copy-on-write archival for belief-changing writes.
+ *
+ * Problem this solves: the latest-wins and conflict-resolution paths used to call
+ * store.update() on the canonical row, replacing text + vector in place. The previous
+ * belief was physically gone, so no retrieval strategy could ever answer
+ * "what did I believe last month?".
+ *
+ * Approach: before overwriting canonical row C, copy C's current content into a new
+ * `superseded` history row H, then update C in place as before. C keeps its id, so the
+ * `deterministicId(scope, canonicalKey)` invariant every canonical lookup depends on is
+ * untouched — history lives beside it rather than replacing it.
+ *
+ * Chain shape (C is the stable canonical id, H1 older than H2):
+ *   H1 --supersededBy--> C     C  --supersedes--> H2
+ *   H2 --supersededBy--> C     H2 --supersedes--> H1
+ * so traceEvolution() walks backward C → H2 → H1 to rebuild the timeline, and forward
+ * from any history row to the belief that is live now.
+ */
+
+import { deterministicId, type MemoryEntry, type MemoryStore } from "./store.js";
+import { parseEvolution, patchEvolution, type EvolutionMetadata } from "./memory-evolution.js";
+import { detectLang, tokenizeFts } from "./language-hook.js";
+import type { Embedder } from "./embedder.js";
+
+/** Namespace for history-row ids — keeps them deterministic (retry-safe) yet distinct from C. */
+const BELIEF_HISTORY_ID_PREFIX = "belief-history:";
+
+export interface BeliefHistoryDeps {
+  store: Pick<MemoryStore, "store"> & Partial<Pick<MemoryStore, "upsert" | "getById">>;
+  embedder: Pick<Embedder, "embedPassage">;
+}
+
+export interface BeliefArchiveResult {
+  /** Id of the newly written history row — becomes the canonical row's `supersedes`. */
+  historyId: string;
+  /** When the archived version stopped being the live belief. */
+  validUntil: number;
+}
+
+/**
+ * Resolve a usable vector for the row being archived.
+ *
+ * store.list() deliberately returns `vector: []` for performance, and that is where the
+ * canonical-match candidates come from — so the entry handed to us usually has no vector.
+ * Re-read the full row first; only re-embed if the store can't give it back.
+ */
+async function resolveVector(deps: BeliefHistoryDeps, current: MemoryEntry): Promise<number[]> {
+  if (current.vector && current.vector.length > 0) return current.vector;
+
+  if (deps.store.getById) {
+    const full = await deps.store.getById(current.id);
+    if (full?.vector && full.vector.length > 0) return full.vector;
+  }
+
+  return await deps.embedder.embedPassage(current.text);
+}
+
+/**
+ * Archive the current content of a canonical memory as a `superseded` history row.
+ *
+ * Throws rather than degrading to a silent overwrite: a belief-changing write that cannot
+ * preserve the old belief is exactly the failure this module exists to prevent, so callers
+ * must not proceed with the overwrite if this rejects.
+ *
+ * Known limitation — archive and overwrite are two writes with no transaction around them.
+ * If the caller's update() fails after this returns, the history row is already committed
+ * while the canonical row still holds that same text, leaving a superseded duplicate of a
+ * belief that is in fact still live. Damage is bounded: nothing is destroyed, and the
+ * deterministic id means a retry reuses the same row rather than piling up copies.
+ */
+export async function archiveBeliefVersion(
+  deps: BeliefHistoryDeps,
+  current: MemoryEntry,
+  options: { now?: number } = {},
+): Promise<BeliefArchiveResult> {
+  const now = options.now ?? Date.now();
+  const currentEvo = parseEvolution(current.metadata, current.timestamp);
+
+  // Text is part of the id, not just (id, timestamp): two belief changes landing in the
+  // same millisecond would otherwise collide and the second archive would overwrite the
+  // first. With the text folded in, a collision means the content is identical anyway.
+  const historyId = deterministicId(
+    current.scope,
+    `${BELIEF_HISTORY_ID_PREFIX}${current.id}@${current.timestamp}\n${current.text}`,
+  );
+  if (historyId === current.id) {
+    throw new Error(`Belief history id collided with canonical id ${current.id}`);
+  }
+
+  const vector = await resolveVector(deps, current);
+  if (vector.length === 0) {
+    throw new Error(`Cannot archive belief ${current.id}: no vector available`);
+  }
+
+  // The history row inherits the old metadata wholesale (canonicalKey, boundary, emotion…)
+  // and only has its evolution block rewritten to close the interval.
+  const historyMetadata = patchEvolution(current.metadata, {
+    status: "superseded",
+    validFrom: currentEvo.validFrom,
+    validUntil: now,
+    supersededBy: current.id,
+    // Point at the version this one replaced, so the backward walk stays a chain.
+    supersedes: currentEvo.supersedes,
+  });
+
+  const historyRow: MemoryEntry = {
+    id: historyId,
+    text: current.text,
+    vector,
+    category: current.category,
+    scope: current.scope,
+    importance: current.importance,
+    // Keep the original timestamp: canonical matching scans the most recent
+    // CANONICAL_SCAN_LIMIT rows, and stamping history rows "now" would let them crowd
+    // live entries out of that window.
+    timestamp: current.timestamp,
+    metadata: historyMetadata,
+    // Derive rather than copy: neither list() nor getById() selects these columns, so
+    // `current.language` / `current.fts_text` are all but guaranteed to be undefined here.
+    // Falling back to the "en" / raw-text defaults would mislabel Chinese memories and
+    // store them unsegmented for FTS — so recompute them the same way live entries do.
+    language: current.language || detectLang(current.text),
+    fts_text: current.fts_text || tokenizeFts(current.text, current.language || detectLang(current.text)),
+  };
+
+  if (deps.store.upsert) {
+    await deps.store.upsert(historyRow);
+  } else {
+    // Fallback for narrower store deps (mocks): store() stamps its own timestamp, which
+    // only costs us the scan-window optimisation above — the history row is still written.
+    await deps.store.store(historyRow);
+  }
+
+  return { historyId, validUntil: now };
+}
+
+/**
+ * Build the evolution block for the canonical row that is about to be overwritten.
+ *
+ * Callers previously passed the incoming metadata straight to store.update(), which wiped
+ * the evolution block along with the text. This merges the incoming metadata with a fresh
+ * evolution block that links back to the archived version.
+ */
+export function buildSupersedingBeliefMetadata(
+  incomingMetadata: string | undefined,
+  previousEvolution: EvolutionMetadata,
+  historyId: string,
+  now: number,
+  evolutionNote?: string | null,
+): string {
+  return patchEvolution(incomingMetadata, {
+    status: "active",
+    version: previousEvolution.version + 1,
+    supersedes: historyId,
+    supersededBy: null,
+    validFrom: now,
+    validUntil: null,
+    // Access stats belong to the canonical key, not to a single version's text.
+    accessCount: previousEvolution.accessCount,
+    lastAccessedAt: previousEvolution.lastAccessedAt,
+    ...(evolutionNote ? { evolutionNote } : {}),
+  });
+}