recallnest 1.4.1 → 2.2.1

package/src/admission-control.ts

@@ -0,0 +1,163 @@
+/**
+ * LME-8: Admission Control — 写入准入控制
+ *
+ * 在 capture-engine.ts 的 persistMemory 入口处执行写前门控，
+ * 拦截低质量记忆，避免垃圾进库再过滤。
+ *
+ * 4 层准入检查：
+ *   1. 最短文本长度 — 过短文本拒绝
+ *   2. Noise filter — 噪声模式前置过滤
+ *   3. Importance 下限 — 低 importance 拒绝
+ *   4. Scope 写入频率限制 — 防 flood
+ */
+
+import { isNoise } from "./noise-filter.js";
+import { logInfo } from "./stderr-log.js";
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface AdmissionConfig {
+  /** Minimum text length to accept (default: 10) */
+  minTextLength: number;
+  /** Enable noise-filter pre-check (default: true) */
+  noiseFilterEnabled: boolean;
+  /** Minimum importance to accept (default: 0.2) */
+  minImportance: number;
+  /** Max writes per scope within the rate window (default: 50) */
+  maxWritesPerScope: number;
+  /** Rate limit window in milliseconds (default: 60_000 = 1 minute) */
+  rateLimitWindowMs: number;
+}
+
+const DEFAULT_CONFIG: AdmissionConfig = {
+  minTextLength: 10,
+  noiseFilterEnabled: true,
+  minImportance: 0.2,
+  maxWritesPerScope: 50,
+  rateLimitWindowMs: 60_000,
+};
+
+export function resolveAdmissionConfig(
+  overrides?: Partial<AdmissionConfig>,
+): AdmissionConfig {
+  return { ...DEFAULT_CONFIG, ...overrides };
+}
+
+// ============================================================================
+// Admission Result
+// ============================================================================
+
+export type AdmissionVerdict = "accepted" | "rejected";
+export type RejectionReason =
+  | "text_too_short"
+  | "noise_detected"
+  | "importance_too_low"
+  | "rate_limited";
+
+export interface AdmissionResult {
+  verdict: AdmissionVerdict;
+  reason?: RejectionReason;
+}
+
+const ACCEPTED: AdmissionResult = { verdict: "accepted" };
+
+// ============================================================================
+// Rate Limiter — per-scope write frequency tracking
+// ============================================================================
+
+export class ScopeRateLimiter {
+  private readonly windows = new Map<string, number[]>();
+
+  /**
+   * Record a write for a scope and check if rate limit is exceeded.
+   * Returns true if the write should be allowed.
+   */
+  check(scope: string, config: AdmissionConfig): boolean {
+    const now = Date.now();
+    const cutoff = now - config.rateLimitWindowMs;
+
+    let timestamps = this.windows.get(scope);
+    if (!timestamps) {
+      timestamps = [];
+      this.windows.set(scope, timestamps);
+    }
+
+    // Prune expired timestamps
+    const firstValid = timestamps.findIndex((t) => t >= cutoff);
+    if (firstValid > 0) {
+      timestamps.splice(0, firstValid);
+    } else if (firstValid === -1) {
+      timestamps.length = 0;
+    }
+
+    if (timestamps.length >= config.maxWritesPerScope) {
+      return false;
+    }
+
+    timestamps.push(now);
+    return true;
+  }
+
+  /** Number of scopes currently tracked. */
+  get size(): number {
+    return this.windows.size;
+  }
+
+  /** Clear all tracking state. */
+  clear(): void {
+    this.windows.clear();
+  }
+}
+
+// ============================================================================
+// Public API — admission gate
+// ============================================================================
+
+/**
+ * Check whether a memory write should be admitted.
+ * Call this at the top of persistMemory, before embedding.
+ *
+ * @param text       - Memory text to evaluate
+ * @param importance - Importance score (0–1)
+ * @param scope      - Target scope
+ * @param rateLimiter - Optional shared rate limiter instance
+ * @param config     - Optional config overrides
+ */
+export function checkAdmission(
+  text: string,
+  importance: number,
+  scope: string,
+  rateLimiter?: ScopeRateLimiter,
+  config?: Partial<AdmissionConfig>,
+): AdmissionResult {
+  const cfg = resolveAdmissionConfig(config);
+
+  // Layer 1: minimum text length
+  const trimmed = text.trim();
+  if (trimmed.length < cfg.minTextLength) {
+    logInfo(`[admission-control] rejected: text too short (${trimmed.length} < ${cfg.minTextLength})`);
+    return { verdict: "rejected", reason: "text_too_short" };
+  }
+
+  // Layer 2: noise filter pre-check
+  if (cfg.noiseFilterEnabled && isNoise(trimmed)) {
+    logInfo(`[admission-control] rejected: noise detected`);
+    return { verdict: "rejected", reason: "noise_detected" };
+  }
+
+  // Layer 3: importance floor
+  if (importance < cfg.minImportance) {
+    logInfo(`[admission-control] rejected: importance too low (${importance} < ${cfg.minImportance})`);
+    return { verdict: "rejected", reason: "importance_too_low" };
+  }
+
+  // Layer 4: scope rate limiting
+  if (rateLimiter && !rateLimiter.check(scope, cfg)) {
+    logInfo(`[admission-control] rejected: rate limited for scope "${scope}"`);
+    return { verdict: "rejected", reason: "rate_limited" };
+  }
+
+  return ACCEPTED;
+}

package/src/anchor-generator.ts

@@ -0,0 +1,80 @@
+/**
+ * P0.1 — Anchor Generator
+ *
+ * Generates a short (≤80 chars) retrieval anchor for each memory.
+ * Short anchors bridge the embedding distance gap between brief queries
+ * (e.g. "轮巡") and long stored documents.
+ *
+ * Heuristic-only — no LLM calls, no latency added to writes.
+ */
+
+const MAX_ANCHOR_LENGTH = 80;
+
+/** CJK character detector */
+const CJK_RE = /[\u4e00-\u9fff\u3400-\u4dbf]/;
+
+/** Extract first meaningful sentence (period, or newline). */
+function firstSentence(text: string): string {
+  // Split on newline first (preserves paragraph structure)
+  const firstLine = text.split(/\n/)[0].trim();
+
+  // Then split on sentence boundaries within that line
+  const match = firstLine.match(/^(.+?)[。！？.!?]/);
+  return match ? match[1].trim() : firstLine;
+}
+
+/** Extract structured title from metadata (patterns, cases). */
+function extractTitle(metadata?: Record<string, unknown>): string | undefined {
+  if (!metadata) return undefined;
+  const wp = metadata.workflowPattern as Record<string, unknown> | undefined;
+  if (wp?.title && typeof wp.title === "string") return wp.title;
+  const cm = metadata.caseMemory as Record<string, unknown> | undefined;
+  if (cm?.title && typeof cm.title === "string") return cm.title;
+  return undefined;
+}
+
+/**
+ * Generate a retrieval anchor (≤80 chars) for a memory entry.
+ *
+ * Priority:
+ * 1. Structured title from metadata (patterns/cases always have one)
+ * 2. First sentence of text (heuristic)
+ *
+ * If the text is already short (≤ MAX_ANCHOR_LENGTH), returns undefined
+ * (no anchor needed — the text itself is a good match target).
+ */
+export function generateAnchor(
+  text: string,
+  metadata?: Record<string, unknown>,
+): string | undefined {
+  // Short text already matches short queries well — skip anchor
+  if (text.length <= MAX_ANCHOR_LENGTH) return undefined;
+
+  // Try structured title first (most precise)
+  const title = extractTitle(metadata);
+  if (title && title.length <= MAX_ANCHOR_LENGTH) {
+    return title;
+  }
+
+  // Fall back to first sentence
+  let anchor = firstSentence(text);
+
+  // Truncate to max length, preserving word/char boundary
+  if (anchor.length > MAX_ANCHOR_LENGTH) {
+    if (CJK_RE.test(anchor)) {
+      // CJK: cut at char boundary
+      anchor = anchor.slice(0, MAX_ANCHOR_LENGTH);
+    } else {
+      // Latin: cut at word boundary
+      anchor = anchor.slice(0, MAX_ANCHOR_LENGTH).replace(/\s+\S*$/, "");
+    }
+  }
+
+  // If anchor is too similar to full text start, it adds no value
+  if (anchor === text.slice(0, anchor.length)) {
+    // Still useful — the shorter embedding will be closer to short queries
+    return anchor;
+  }
+
+  return anchor || undefined;
+}

package/src/api-server.ts

@@ -24,6 +24,7 @@ import { WorkflowObservationStore } from "./workflow-observation-store.js";
 import { buildManagedCheckpointObservation, buildManagedResumeObservation } from "./workflow-observation-managed.js";
 import { runAutoRecall } from "./auto-recall.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
+import { runMemoryLint } from "./memory-lint.js";
 
 const config = (loadDotEnv(), loadConfig());
 const getComponents = createComponentResolver(config);
@@ -589,6 +590,14 @@ async function handleStats(): Promise<Response> {
   });
 }
 
+/** GET /v1/lint — memory quality lint */
+async function handleLint(searchParams: URLSearchParams): Promise<Response> {
+  const { store } = getComponents();
+  const scope = searchParams.get("scope") || undefined;
+  const report = await runMemoryLint({ store, scope });
+  return jsonResponse(report);
+}
+
 /** GET /v1/health — health check */
 async function handleHealth(): Promise<Response> {
   try {
@@ -652,6 +661,7 @@ const server = Bun.serve({
         if (pathname === "/v1/workflow-health") return await handleWorkflowHealth(request);
         if (pathname === "/v1/workflow-evidence") return await handleWorkflowEvidence(request);
         if (pathname === "/v1/stats") return await handleStats();
+        if (pathname === "/v1/lint") return await handleLint(url.searchParams);
         if (pathname === "/v1/health") return await handleHealth();
       }
 

package/src/audit-log.ts

@@ -0,0 +1,122 @@
+/**
+ * F-1: Audit log — record every store/update/delete/retrieve operation.
+ * Logs are written to a separate file (not mixed into memory data).
+ *
+ * Each log entry: { timestamp, operation, scope, memoryId, actor, details }
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+export type AuditOperation =
+  | "store"
+  | "update"
+  | "delete"
+  | "retrieve"
+  | "archive"
+  | "supersede"
+  | "consolidate"
+  | "forget"
+  | "cascade_forget";
+
+export interface AuditEntry {
+  timestamp: string; // ISO 8601
+  operation: AuditOperation;
+  scope?: string;
+  memoryId?: string;
+  actor: string; // "manual" | "agent" | "api" | "system"
+  details?: string; // brief context (<=200 chars)
+}
+
+export interface AuditLogger {
+  log(entry: Omit<AuditEntry, "timestamp">): void;
+  /** Read recent entries (newest first) */
+  getRecent(limit?: number): AuditEntry[];
+  /** Export all entries as JSON array */
+  exportAll(): AuditEntry[];
+  /** Count total entries */
+  count(): number;
+}
+
+function parseLines(raw: string): AuditEntry[] {
+  const entries: AuditEntry[] = [];
+  for (const line of raw.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try {
+      entries.push(JSON.parse(trimmed) as AuditEntry);
+    } catch {
+      // skip malformed lines
+    }
+  }
+  return entries;
+}
+
+/**
+ * File-based audit logger. Appends JSON lines to a log file.
+ * Non-blocking: write failures are silently ignored (audit must never block operations).
+ */
+export function createAuditLogger(logPath?: string): AuditLogger {
+  const resolvedPath =
+    logPath ??
+    join(process.env.RECALLNEST_DATA_DIR || "data", "audit.jsonl");
+
+  // Ensure directory exists (silent on failure)
+  try {
+    const dir = dirname(resolvedPath);
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+  } catch {
+    // silent — audit must never block
+  }
+
+  return {
+    log(entry: Omit<AuditEntry, "timestamp">): void {
+      try {
+        const full: AuditEntry = {
+          timestamp: new Date().toISOString(),
+          ...entry,
+        };
+        // Truncate details to 200 chars
+        if (full.details && full.details.length > 200) {
+          full.details = full.details.slice(0, 200);
+        }
+        appendFileSync(resolvedPath, JSON.stringify(full) + "\n", "utf-8");
+      } catch {
+        // silent — audit must never block operations
+      }
+    },
+
+    getRecent(limit = 20): AuditEntry[] {
+      try {
+        if (!existsSync(resolvedPath)) return [];
+        const raw = readFileSync(resolvedPath, "utf-8");
+        const all = parseLines(raw);
+        return all.slice(-limit).reverse();
+      } catch {
+        return [];
+      }
+    },
+
+    exportAll(): AuditEntry[] {
+      try {
+        if (!existsSync(resolvedPath)) return [];
+        const raw = readFileSync(resolvedPath, "utf-8");
+        return parseLines(raw);
+      } catch {
+        return [];
+      }
+    },
+
+    count(): number {
+      try {
+        if (!existsSync(resolvedPath)) return 0;
+        const raw = readFileSync(resolvedPath, "utf-8");
+        return parseLines(raw).length;
+      } catch {
+        return 0;
+      }
+    },
+  };
+}

package/src/auto-gc.ts

@@ -5,10 +5,20 @@
  * AND enough time has passed since last GC. Inspired by NREM triple-coupling:
  * multiple conditions must be satisfied before consolidation fires.
  *
- * Wraps existing cleanup scripts into a programmatic API.
+ * B-2 upgrade: Uses evolution system's computeDecayScore + buildArchivedMetadata
+ * instead of raw age/importance heuristics. Archive-first, delete-never.
  */
 
 import type { MemoryStore } from "./store.js";
+import {
+  parseEvolution,
+  computeDecayScore,
+  buildArchivedMetadata,
+  isActiveMemory,
+} from "./memory-evolution.js";
+import { loadRetentionPolicy, shouldArchiveByPolicy } from "./retention-policy.js";
+import type { RetentionPolicy } from "./retention-policy.js";
+import type { AuditLogger } from "./audit-log.js";
 
 // ---------------------------------------------------------------------------
 // Config
@@ -19,17 +29,20 @@ export interface AutoGcConfig {
   minMemoryCount: number;
   /** Minimum hours since last GC (default: 24) */
   minHoursSinceLastGc: number;
-  /** Score below which memories are candidates for archival (default: 0.2) */
-  staleScoreThreshold: number;
+  /** Decay score below which memories are archive candidates (default: 0.15) */
+  decayScoreThreshold: number;
   /** Max entries to archive per run (default: 100) */
   maxArchivePerRun: number;
+  /** Minimum age in days before a memory can be archived (default: 30) */
+  minAgeDays: number;
 }
 
 export const DEFAULT_AUTO_GC_CONFIG: AutoGcConfig = {
   minMemoryCount: 1000,
   minHoursSinceLastGc: 24,
-  staleScoreThreshold: 0.2,
+  decayScoreThreshold: 0.15,
   maxArchivePerRun: 100,
+  minAgeDays: 30,
 };
 
 // ---------------------------------------------------------------------------
@@ -52,10 +65,18 @@ export interface GcResult {
 /**
  * Check conditions and run GC if all thresholds are met.
  * Returns immediately if conditions not met (no-op).
+ *
+ * Archive criteria (B-2 evolution-aware):
+ * - Memory must be "active" (not already superseded/archived/consolidated)
+ * - Composite decay_score < threshold (default 0.15)
+ * - Age > minAgeDays (default 30 days)
+ * - Not pinned (importance >= 0.95 is considered pinned)
  */
 export async function maybeRunGc(
   store: MemoryStore,
   config: AutoGcConfig = DEFAULT_AUTO_GC_CONFIG,
+  retentionConfigDir?: string,
+  auditLogger?: AuditLogger,
 ): Promise<GcResult> {
   const stats = await store.stats();
   const totalMemories = stats.total ?? 0;
@@ -74,8 +95,7 @@ export async function maybeRunGc(
   // All conditions met — run GC
   lastGcTimestamp = Date.now();
 
-  // Find stale peripheral memories with low importance and no recent access
-  const cutoffMs = Date.now() - 60 * 86_400_000; // 60 days ago
+  const now = Date.now();
   let archivedCount = 0;
   let totalChecked = 0;
 
@@ -83,33 +103,82 @@ export async function maybeRunGc(
   const entries = await store.list({ limit: 5000 });
   totalChecked = entries.length;
 
+  // Pre-compute per-scope active memory counts and cache retention policies.
+  // This avoids repeated file reads inside the loop.
+  const activeCounts = new Map<string, number>();
+  const policyCache = new Map<string, RetentionPolicy>();
+  const scopesSeen = new Set<string>();
+
+  for (const entry of entries) {
+    const scope = entry.scope ?? "";
+    scopesSeen.add(scope);
+    if (isActiveMemory(entry.metadata)) {
+      activeCounts.set(scope, (activeCounts.get(scope) ?? 0) + 1);
+    }
+  }
+
+  // Batch-load retention policies for all scopes (one file read per scope)
+  for (const scope of scopesSeen) {
+    policyCache.set(scope, loadRetentionPolicy(scope, retentionConfigDir));
+  }
+
   for (const entry of entries) {
     if (archivedCount >= config.maxArchivePerRun) break;
 
-    let meta: Record<string, any> = {};
-    try { meta = JSON.parse(entry.metadata || "{}"); } catch { /* skip */ }
-
-    const isArchived = meta.archived === true;
-    if (isArchived) continue;
-
-    const tier = meta.tier || "peripheral";
-    const accessCount = meta.accessCount || 0;
-    const importance = entry.importance ?? 0;
-    const age = Date.now() - entry.timestamp;
-    const ageDays = age / 86_400_000;
-
-    // Archive criteria: peripheral + old + low importance + rarely accessed
-    if (
-      tier === "peripheral" &&
-      ageDays > 60 &&
-      importance < config.staleScoreThreshold &&
-      accessCount < 2
-    ) {
-      meta.archived = true;
-      meta.archivedAt = new Date().toISOString();
-      meta.archivedReason = "auto-gc:stale-peripheral";
-      await store.update(entry.id, { metadata: JSON.stringify(meta) });
+    // Only archive active memories
+    if (!isActiveMemory(entry.metadata)) continue;
+
+    // Never archive pinned memories (importance >= 0.95)
+    const importance = entry.importance ?? 0.5;
+    if (importance >= 0.95) continue;
+
+    // Check minimum age
+    const ageDays = (now - entry.timestamp) / 86_400_000;
+
+    // Decay-based archival (existing logic)
+    let shouldArchive = false;
+    if (ageDays >= config.minAgeDays) {
+      const evo = parseEvolution(entry.metadata, entry.timestamp);
+      let decayScore = computeDecayScore(evo, importance, now, entry.metadata);
+      // F3: Expired memories (validUntil < now) get 2x decay acceleration
+      if (evo.validUntil != null && evo.validUntil < now) {
+        decayScore *= 0.5; // Halve the score → archives faster
+      }
+      if (decayScore < config.decayScoreThreshold) {
+        shouldArchive = true;
+      }
+    }
+
+    // Retention-policy-based archival (F-2): OR with decay-based
+    if (!shouldArchive) {
+      const scope = entry.scope ?? "";
+      const policy = policyCache.get(scope) ?? loadRetentionPolicy(scope, retentionConfigDir);
+      const activeCount = activeCounts.get(scope) ?? 0;
+      const policyCheck = shouldArchiveByPolicy(policy, ageDays, activeCount);
+      if (policyCheck.archive) {
+        shouldArchive = true;
+      }
+    }
+
+    if (shouldArchive) {
+      const archivedMeta = buildArchivedMetadata(entry.metadata);
+      await store.update(entry.id, { metadata: archivedMeta });
       archivedCount++;
+      // F-1: Audit log — record archive operation (silent on failure)
+      try {
+        auditLogger?.log({
+          operation: "archive",
+          memoryId: entry.id,
+          actor: "system",
+          details: `auto-gc: age=${Math.floor(ageDays)}d`,
+        });
+      } catch { /* Audit must never block GC */ }
+      // Decrement active count for the scope (memory is no longer active)
+      const scope = entry.scope ?? "";
+      const prev = activeCounts.get(scope) ?? 0;
+      if (prev > 0) {
+        activeCounts.set(scope, prev - 1);
+      }
     }
   }
 

package/src/auto-recall.ts

@@ -1,5 +1,7 @@
 import { composeResumeContext, type ResumeContextDeps } from "./context-composer.js";
+import { resolveTier } from "./decay-engine.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
+import { governResults, truncateQuery, type GovernorConfig, type GovernorSession } from "./recall-governor.js";
 import type { RetrievalResult } from "./retriever.js";
 import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
 import type { RetrievalProfileName, ResumeContextResponse } from "./session-schema.js";
@@ -17,6 +19,10 @@ export interface AutoRecallRequest {
   profile?: RetrievalProfileName;
   env?: NodeJS.ProcessEnv;
   operation?: string;
+  /** Governor config overrides (charBudget, maxItems, maxQueryChars) */
+  governor?: Partial<GovernorConfig>;
+  /** Governor session for cross-call dedup within a single conversation turn */
+  governorSession?: GovernorSession;
 }
 
 export interface AutoRecallResponse {
@@ -31,8 +37,10 @@ export async function runAutoRecall(
   deps: ResumeContextDeps,
   request: AutoRecallRequest,
 ): Promise<AutoRecallResponse> {
-  const message = request.message.replace(/\s+/g, " ").trim();
-  const task = (request.task || message).replace(/\s+/g, " ").trim();
+  const govCfg = request.governor;
+  const maxQ = govCfg?.maxQueryChars ?? 1000;
+  const message = truncateQuery(request.message.replace(/\s+/g, " ").trim(), maxQ);
+  const task = truncateQuery((request.task || message).replace(/\s+/g, " ").trim(), maxQ);
 
   if (!message) {
     throw new Error("message is required");
@@ -72,9 +80,11 @@ export async function runAutoRecall(
     };
   }
 
-  const results = await deps.retriever.retrieve(buildRetrievalContext({
+  const effectiveLimit = request.limit || 5;
+  // Fetch extra candidates so tier filtering still yields enough results
+  const rawResults = await deps.retriever.retrieve(buildRetrievalContext({
     query: message,
-    limit: request.limit || 5,
+    limit: effectiveLimit * 3,
     category: request.category,
     scope: resolvedScope,
     sessionId: request.sessionId,
@@ -86,6 +96,25 @@ export async function runAutoRecall(
     allowUnscoped: request.allScopes === true,
   }));
 
+  // Tier-aware filtering: prefer core/working memories, backfill with peripheral.
+  // Peripheral memories get a 15% score discount to reduce noise injection.
+  const promoted: RetrievalResult[] = [];
+  const demoted: RetrievalResult[] = [];
+  for (const r of rawResults) {
+    const tier = resolveTier(r.entry.metadata);
+    if (tier === "core" || tier === "working") {
+      promoted.push(r);
+    } else {
+      demoted.push({ ...r, score: r.score * 0.85 });
+    }
+  }
+  const tierFiltered = [...promoted, ...demoted]
+    .sort((a, b) => b.score - a.score)
+    .slice(0, effectiveLimit);
+
+  // LME-7: Recall Governor — evolution filter, budget control, session dedup
+  const results = governResults(tierFiltered, request.governorSession, govCfg);
+
   return {
     mode: "resume+search",
     resolvedScope,