recallnest 1.4.1 → 2.2.1

package/src/forget-engine.ts

@@ -0,0 +1,269 @@
+/**
+ * Forget Engine — Ethics-aware memory deletion with full cascade.
+ *
+ * Philosophy: "The right to forget" — memories can be explicitly removed,
+ * but the process must be auditable, reversible (via evidence export),
+ * and propagated to all derived artifacts (KG triples, pins, evolution chains).
+ *
+ * Sequence:
+ * 1. Fetch target memory
+ * 2. Privacy tier check (durable requires explicit confirm)
+ * 3. Evidence export (snapshot before deletion for audit trail)
+ * 4. KG triple cleanup (via KGStore.deleteBySource)
+ * 5. Pin archive (mark related pins as forgotten)
+ * 6. Cascade demote (related memories via cascade-forget.ts)
+ * 7. Primary delete (remove from LanceDB)
+ * 8. Audit log (record the forget operation)
+ */
+
+import type { MemoryStore, MemoryEntry } from "./store.js";
+import type { KGStore } from "./kg-store.js";
+import type { AuditLogger } from "./audit-log.js";
+import { parsePrivacyTier, type PrivacyTier } from "./memory-schema.js";
+import { parseEvolution, patchEvolution } from "./memory-evolution.js";
+import { cascadeForget, type CascadeForgetConfig, DEFAULT_CASCADE_FORGET_CONFIG } from "./cascade-forget.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ForgetRequest {
+  /** Memory ID to forget (full UUID or 8+ hex prefix) */
+  memoryId: string;
+  /** Explicit confirmation — required for "durable" tier memories */
+  confirm: boolean;
+  /** Reason for forgetting (audit trail) */
+  reason?: string;
+  /** Scope filter for permission check */
+  scopeFilter?: string[];
+}
+
+export interface ForgetEvidence {
+  /** Snapshot of the memory before deletion */
+  entry: MemoryEntry;
+  /** Privacy tier at time of deletion */
+  privacyTier: PrivacyTier;
+  /** Evolution chain snapshot */
+  evolution: ReturnType<typeof parseEvolution>;
+  /** Timestamp of forget operation */
+  forgottenAt: string;
+  /** Reason provided */
+  reason?: string;
+}
+
+export interface ForgetResult {
+  /** Whether the memory was successfully forgotten */
+  success: boolean;
+  /** ID of the forgotten memory */
+  memoryId: string;
+  /** Evidence snapshot (for audit/undo) */
+  evidence?: ForgetEvidence;
+  /** Number of KG triples removed */
+  kgTriplesRemoved: boolean;
+  /** Cascade demote results */
+  cascadeResult: { demotedCount: number; demotedIds: string[] };
+  /** Error message if failed */
+  error?: string;
+}
+
+export interface ForgetByIdDeps {
+  store: MemoryStore;
+  kgStore?: KGStore | null;
+  auditLogger?: AuditLogger | null;
+  cascadeConfig?: CascadeForgetConfig;
+}
+
+// ---------------------------------------------------------------------------
+// Core: Forget a single memory
+// ---------------------------------------------------------------------------
+
+export async function forgetMemory(
+  deps: ForgetByIdDeps,
+  request: ForgetRequest,
+): Promise<ForgetResult> {
+  const { store, kgStore, auditLogger, cascadeConfig } = deps;
+  const { memoryId, confirm, reason, scopeFilter } = request;
+
+  // 1. Fetch target
+  const entry = await store.get(memoryId, scopeFilter);
+  if (!entry) {
+    return {
+      success: false,
+      memoryId,
+      kgTriplesRemoved: false,
+      cascadeResult: { demotedCount: 0, demotedIds: [] },
+      error: `Memory ${memoryId} not found`,
+    };
+  }
+
+  // 2. Privacy tier check
+  const privacyTier = parsePrivacyTier(entry.metadata);
+  if (privacyTier === "durable" && !confirm) {
+    return {
+      success: false,
+      memoryId: entry.id,
+      kgTriplesRemoved: false,
+      cascadeResult: { demotedCount: 0, demotedIds: [] },
+      error: `Memory ${entry.id} has privacy tier "durable" — set confirm=true to proceed`,
+    };
+  }
+
+  // 3. Evidence export (snapshot before deletion)
+  const evolution = parseEvolution(entry.metadata, entry.timestamp);
+  const evidence: ForgetEvidence = {
+    entry: { ...entry },
+    privacyTier,
+    evolution,
+    forgottenAt: new Date().toISOString(),
+    reason,
+  };
+
+  // 4. KG triple cleanup
+  let kgTriplesRemoved = false;
+  if (kgStore) {
+    try {
+      await kgStore.deleteBySource(entry.id);
+      kgTriplesRemoved = true;
+    } catch (err) {
+      console.error("[recallnest] KG cleanup failed during forget:", err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  // 5. Cascade demote (related memories get importance reduction)
+  let cascadeResult = { demotedCount: 0, demotedIds: [] as string[] };
+  try {
+    cascadeResult = await cascadeForget(
+      store,
+      { id: entry.id, vector: entry.vector, scope: entry.scope },
+      cascadeConfig ?? DEFAULT_CASCADE_FORGET_CONFIG,
+    );
+  } catch (err) {
+    console.error("[recallnest] Cascade demote failed during forget:", err instanceof Error ? err.message : String(err));
+  }
+
+  // 6. Mark evolution status as "forgotten" before delete (audit breadcrumb)
+  try {
+    const patchedMetadata = patchEvolution(entry.metadata, {
+      status: "archived" as any,
+      evolutionNote: `forgotten: ${reason || "user request"}`,
+    });
+    await store.update(entry.id, { metadata: patchedMetadata }, scopeFilter);
+  } catch (err) {
+    console.error("[recallnest] Evolution patch failed during forget:", err instanceof Error ? err.message : String(err));
+  }
+
+  // 7. Primary delete
+  try {
+    await store.delete(entry.id, scopeFilter);
+  } catch (err) {
+    return {
+      success: false,
+      memoryId: entry.id,
+      evidence,
+      kgTriplesRemoved,
+      cascadeResult,
+      error: `Delete failed: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+
+  // 8. Audit log
+  try {
+    auditLogger?.log({
+      operation: "forget",
+      scope: entry.scope,
+      memoryId: entry.id,
+      actor: "system",
+      details: `tier=${privacyTier} reason=${reason || "none"} cascade=${cascadeResult.demotedCount}`,
+    });
+    if (cascadeResult.demotedCount > 0) {
+      auditLogger?.log({
+        operation: "cascade_forget",
+        scope: entry.scope,
+        memoryId: entry.id,
+        actor: "system",
+        details: `demoted ${cascadeResult.demotedCount} related memories: ${cascadeResult.demotedIds.map(id => id.slice(0, 8)).join(",")}`,
+      });
+    }
+  } catch (err) {
+    console.error("[recallnest] Audit log failed during forget:", err instanceof Error ? err.message : String(err));
+  }
+
+  return {
+    success: true,
+    memoryId: entry.id,
+    evidence,
+    kgTriplesRemoved,
+    cascadeResult,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Bulk: Forget all memories in a scope
+// ---------------------------------------------------------------------------
+
+export interface ForgetByScopeResult {
+  forgottenCount: number;
+  failedCount: number;
+  kgScopeCleared: boolean;
+  totalCascadeDemoted: number;
+}
+
+export async function forgetByScope(
+  deps: ForgetByIdDeps,
+  scope: string,
+  confirm: boolean,
+  reason?: string,
+): Promise<ForgetByScopeResult> {
+  if (!confirm) {
+    return { forgottenCount: 0, failedCount: 0, kgScopeCleared: false, totalCascadeDemoted: 0 };
+  }
+
+  const { store, kgStore, auditLogger } = deps;
+  let forgottenCount = 0;
+  let failedCount = 0;
+  let totalCascadeDemoted = 0;
+
+  // Fetch all entries in scope
+  const entries = await store.list([scope], undefined, 5000);
+
+  for (const entry of entries) {
+    const result = await forgetMemory(deps, {
+      memoryId: entry.id,
+      confirm: true,
+      reason: reason || `scope-level forget: ${scope}`,
+      scopeFilter: [scope],
+    });
+
+    if (result.success) {
+      forgottenCount++;
+      totalCascadeDemoted += result.cascadeResult.demotedCount;
+    } else {
+      failedCount++;
+    }
+  }
+
+  // Bulk KG scope cleanup
+  let kgScopeCleared = false;
+  if (kgStore) {
+    try {
+      await kgStore.deleteByScope(scope);
+      kgScopeCleared = true;
+    } catch (err) {
+      console.error("[recallnest] KG scope cleanup failed:", err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  // Audit the bulk operation
+  try {
+    auditLogger?.log({
+      operation: "forget",
+      scope,
+      actor: "system",
+      details: `scope-forget: ${forgottenCount} deleted, ${failedCount} failed, ${totalCascadeDemoted} cascade-demoted`,
+    });
+  } catch (err) {
+    console.error("[recallnest] Audit log failed during scope forget:", err instanceof Error ? err.message : String(err));
+  }
+
+  return { forgottenCount, failedCount, kgScopeCleared, totalCascadeDemoted };
+}

package/src/frequency-tracker.ts

@@ -0,0 +1,175 @@
+/**
+ * P0.2 — Frequency Tracker
+ *
+ * Tracks how often each memory_id gets retrieved (hit_count).
+ * Used to boost frequently accessed memories in scoring.
+ *
+ * Key design decisions:
+ * - Counts per memory_id, not per query text (avoids synonym fragmentation)
+ * - Persists to JSON file, loaded on startup
+ * - Time decay: 30 days without a hit → effective count halves
+ * - Superseded entries do NOT inherit hit_count (fresh starts)
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
+import { dirname, join } from "path";
+import { logWarn } from "./stderr-log.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface FrequencyEntry {
+  hitCount: number;
+  lastHitAt: number; // ms since epoch
+}
+
+export interface FrequencyStats {
+  [memoryId: string]: FrequencyEntry;
+}
+
+export interface FrequencyTrackerConfig {
+  /** Path to persistence file. Default: data/frequency-stats.json */
+  filePath: string;
+  /** Boost factor per log2(hitCount). Default: 0.15 */
+  boostFactor: number;
+  /** Days without a hit before effective count halves. Default: 30 */
+  decayHalfLifeDays: number;
+  /** Minimum hits before boost kicks in. Default: 2 */
+  minHitsForBoost: number;
+  /** Hit count threshold for auto-promoting to core tier. Default: 3 */
+  corePromotionThreshold: number;
+  /** Debounce write interval (ms). Default: 10000 */
+  flushIntervalMs: number;
+}
+
+export const DEFAULT_FREQUENCY_CONFIG: FrequencyTrackerConfig = {
+  filePath: join(process.cwd(), "data", "frequency-stats.json"),
+  boostFactor: 0.15,
+  decayHalfLifeDays: 30,
+  minHitsForBoost: 2,
+  corePromotionThreshold: 3,
+  flushIntervalMs: 10000,
+};
+
+// ============================================================================
+// Frequency Tracker
+// ============================================================================
+
+export class FrequencyTracker {
+  private stats: FrequencyStats = {};
+  private dirty = false;
+  private flushTimer: ReturnType<typeof setTimeout> | null = null;
+  readonly config: FrequencyTrackerConfig;
+
+  constructor(config?: Partial<FrequencyTrackerConfig>) {
+    this.config = { ...DEFAULT_FREQUENCY_CONFIG, ...config };
+    this.load();
+  }
+
+  /** Record hits for a set of memory IDs (called after retrieval). */
+  recordHits(memoryIds: string[]): void {
+    const now = Date.now();
+    for (const id of memoryIds) {
+      const existing = this.stats[id];
+      if (existing) {
+        existing.hitCount += 1;
+        existing.lastHitAt = now;
+      } else {
+        this.stats[id] = { hitCount: 1, lastHitAt: now };
+      }
+    }
+    this.dirty = true;
+    this.scheduleSave();
+  }
+
+  /**
+   * Compute the frequency boost multiplier for a memory.
+   * Returns 1.0 (no boost) if below threshold.
+   * Formula: 1 + log2(effectiveHits) * boostFactor
+   */
+  getBoostMultiplier(memoryId: string): number {
+    const entry = this.stats[memoryId];
+    if (!entry || entry.hitCount < this.config.minHitsForBoost) return 1.0;
+
+    const effectiveHits = this.effectiveHitCount(entry);
+    if (effectiveHits < this.config.minHitsForBoost) return 1.0;
+
+    return 1 + Math.log2(effectiveHits) * this.config.boostFactor;
+  }
+
+  /** Check if a memory should be auto-promoted to core tier. */
+  shouldPromoteToCore(memoryId: string): boolean {
+    const entry = this.stats[memoryId];
+    if (!entry) return false;
+    return this.effectiveHitCount(entry) >= this.config.corePromotionThreshold;
+  }
+
+  /** Get raw stats for a memory (for debugging/testing). */
+  getStats(memoryId: string): FrequencyEntry | undefined {
+    return this.stats[memoryId];
+  }
+
+  /** Total tracked entries. */
+  get size(): number {
+    return Object.keys(this.stats).length;
+  }
+
+  /** Force flush to disk (for testing or shutdown). */
+  flush(): void {
+    if (!this.dirty) return;
+    this.save();
+  }
+
+  /** Clean up timer on shutdown. */
+  dispose(): void {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+    this.flush();
+  }
+
+  // --------------------------------------------------------------------------
+  // Internal
+  // --------------------------------------------------------------------------
+
+  private effectiveHitCount(entry: FrequencyEntry): number {
+    const daysSinceHit = (Date.now() - entry.lastHitAt) / 86_400_000;
+    const decayFactor = Math.exp(
+      (-daysSinceHit * Math.LN2) / this.config.decayHalfLifeDays,
+    );
+    return entry.hitCount * decayFactor;
+  }
+
+  private load(): void {
+    try {
+      if (existsSync(this.config.filePath)) {
+        const raw = readFileSync(this.config.filePath, "utf-8");
+        this.stats = JSON.parse(raw) as FrequencyStats;
+      }
+    } catch {
+      logWarn("frequency-tracker: failed to load stats, starting fresh");
+      this.stats = {};
+    }
+  }
+
+  private save(): void {
+    try {
+      const dir = dirname(this.config.filePath);
+      if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+      writeFileSync(this.config.filePath, JSON.stringify(this.stats, null, 2));
+      this.dirty = false;
+    } catch (err) {
+      logWarn(`frequency-tracker: failed to save stats: ${err}`);
+    }
+  }
+
+  private scheduleSave(): void {
+    if (this.flushTimer) return;
+    this.flushTimer = setTimeout(() => {
+      this.flushTimer = null;
+      this.save();
+    }, this.config.flushIntervalMs);
+  }
+}