recallnest 1.4.1 → 2.2.1

package/src/decay-engine.ts

@@ -10,6 +10,8 @@
  * No LLM required — pure math + access statistics.
  */
 
+import type { EmotionMetadata } from "./memory-schema.js";
+
 // ============================================================================
 // Tier Definitions
 // ============================================================================
@@ -86,6 +88,88 @@ export function weibullDecay(
   return floor + (1 - floor) * decay;
 }
 
+// ============================================================================
+// Emotion-Adjusted Decay
+// ============================================================================
+
+/**
+ * Adjust half-life based on emotional intensity.
+ * Strong emotion extends half-life by up to 30%.
+ */
+export function adjustHalfLifeForEmotion(
+  baseHalfLife: number,
+  emotion: EmotionMetadata | null | undefined,
+): number {
+  if (!emotion) return baseHalfLife;
+  const raw = Math.abs(emotion.valence);
+  // Dead-zone: ignore negligible emotional signal below threshold
+  const intensity = raw < 0.1 ? 0 : raw;
+  return baseHalfLife * (1 + 0.3 * intensity);
+}
+
+/**
+ * Compute initial strength boost from arousal (flashbulb memory effect).
+ * Returns multiplier in [1.0, 1.1].
+ */
+export function computeArousalBoost(
+  emotion: EmotionMetadata | null | undefined,
+): number {
+  if (!emotion) return 1.0;
+  return 1 + 0.1 * emotion.arousal;
+}
+
+// ============================================================================
+// HP-7: Decay Exemptions
+// ============================================================================
+
+/**
+ * Check if a memory should be exempt from time decay.
+ * Exempt entries keep their original score without time-based penalties.
+ *
+ * Exemption rules:
+ * 1. Core tier + high importance (≥ 0.95) → identity/correction, must not fade
+ * 2. Recently accessed (within 7 days) → actively used, not stale
+ * 3. Pinned → user explicitly marked as persistent
+ */
+export function isDecayExempt(
+  metadata: string | undefined,
+  importance: number,
+): boolean {
+  if (!metadata) return false;
+
+  try {
+    const meta = JSON.parse(metadata) as Record<string, unknown>;
+
+    // Rule 1: Core + very high importance
+    const tier = meta.tier ?? resolveTierFromMeta(meta);
+    if (tier === "core" && importance >= 0.95) return true;
+
+    // Rule 2: Accessed within last 7 days
+    const lastAccess = typeof meta.lastAccessedAt === "number" ? meta.lastAccessedAt : 0;
+    if (lastAccess > 0 && (Date.now() - lastAccess) < 7 * 86_400_000) return true;
+
+    // Rule 3: Pinned
+    const tags = Array.isArray(meta.tags) ? meta.tags : [];
+    if (tags.includes("pinned")) return true;
+
+    return false;
+  } catch {
+    return false;
+  }
+}
+
+/** Internal helper: resolve tier from parsed metadata (avoids re-parsing). */
+function resolveTierFromMeta(meta: Record<string, unknown>): MemoryTier {
+  if (meta.tier === "core" || meta.tier === "working" || meta.tier === "peripheral") {
+    return meta.tier;
+  }
+  const imp = typeof meta.importance === "number" ? meta.importance : 0;
+  const ac = typeof meta.accessCount === "number" ? meta.accessCount : 0;
+  if (imp >= 0.95 || ac >= 10) return "core";
+  if (imp >= 0.8 || ac >= 3) return "working";
+  return "peripheral";
+}
+
 // ============================================================================
 // Tier Resolution
 // ============================================================================

package/src/distill-lock.ts

@@ -0,0 +1,122 @@
+/**
+ * CC-6: Distill gate — PID-based lock file + session gate.
+ * Lock file mtime serves as lastDistillAt timestamp (no separate field).
+ *
+ * - acquireLock(): write current PID, check existing lock PID liveness + mtime expiry
+ * - releaseLock(): delete lock file (mtime naturally updates to now on success)
+ * - rollbackLock(previousMtime): on failure, rewind lock mtime so next run retries
+ * - shouldDistill(checkpointCount): session gate — checkpoint count < 3 → skip
+ * - getLastDistillTime(): read lock file mtime, return 0 if absent
+ */
+
+import {
+  mkdirSync,
+  writeFileSync,
+  readFileSync,
+  unlinkSync,
+  statSync,
+  utimesSync,
+  existsSync,
+} from "node:fs";
+import { join, dirname } from "node:path";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface DistillLockConfig {
+  /** Path to the lock file (default: data/distill.lock) */
+  lockPath: string;
+  /** Lock expiry in ms (default: 3600000 = 1h) */
+  expireMs: number;
+  /** Minimum checkpoint count before distill is allowed (default: 3) */
+  minCheckpoints: number;
+}
+
+export const DEFAULT_DISTILL_LOCK_CONFIG: DistillLockConfig = {
+  lockPath: join(process.env.RECALLNEST_DATA_DIR || "data", "distill.lock"),
+  expireMs: 3_600_000,
+  minCheckpoints: 3,
+};
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function resolveConfig(cfg?: Partial<DistillLockConfig>): DistillLockConfig {
+  return { ...DEFAULT_DISTILL_LOCK_CONFIG, ...cfg };
+}
+
+function isPidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Try to acquire the distill lock.
+ * Returns true if lock was acquired, false if another live process holds it.
+ */
+export function acquireLock(cfg?: Partial<DistillLockConfig>): boolean {
+  const { lockPath, expireMs } = resolveConfig(cfg);
+
+  if (existsSync(lockPath)) {
+    const existingPid = parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
+    const mtime = statSync(lockPath).mtimeMs;
+    const expired = Date.now() - mtime > expireMs;
+
+    if (!isNaN(existingPid) && isPidAlive(existingPid) && !expired) {
+      return false;
+    }
+    // PID dead or lock expired — overwrite
+  }
+
+  mkdirSync(dirname(lockPath), { recursive: true });
+  writeFileSync(lockPath, String(process.pid), "utf-8");
+  return true;
+}
+
+/** Release the distill lock by removing the lock file. */
+export function releaseLock(cfg?: Partial<DistillLockConfig>): void {
+  const { lockPath } = resolveConfig(cfg);
+  if (existsSync(lockPath)) {
+    unlinkSync(lockPath);
+  }
+}
+
+/**
+ * Roll back lock mtime to a previous value so the next run retries sooner.
+ * Used when distill fails and we want to preserve the "last success" timestamp.
+ */
+export function rollbackLock(
+  previousMtime: Date,
+  cfg?: Partial<DistillLockConfig>,
+): void {
+  const { lockPath } = resolveConfig(cfg);
+  if (existsSync(lockPath)) {
+    utimesSync(lockPath, previousMtime, previousMtime);
+  }
+}
+
+/** Session gate: returns true when enough checkpoints have accumulated. */
+export function shouldDistill(
+  checkpointCountSinceLastDistill: number,
+  cfg?: Partial<DistillLockConfig>,
+): boolean {
+  const { minCheckpoints } = resolveConfig(cfg);
+  return checkpointCountSinceLastDistill >= minCheckpoints;
+}
+
+/** Read lock file mtime as the last-distill timestamp. Returns 0 if absent. */
+export function getLastDistillTime(cfg?: Partial<DistillLockConfig>): number {
+  const { lockPath } = resolveConfig(cfg);
+  if (!existsSync(lockPath)) return 0;
+  return statSync(lockPath).mtimeMs;
+}

package/src/doctor.ts

@@ -13,6 +13,7 @@
 import { existsSync, accessSync, constants, readFileSync } from "node:fs";
 import { resolve, join } from "node:path";
 import { homedir } from "node:os";
+import { metaDir } from "./compat.js";
 import { loadConfig, expandHome, resolveEnv, findConfigPath, loadDotEnv } from "./runtime-config.js";
 import { createEmbedder, type EmbeddingConfig } from "./embedder.js";
 import { extractCanonicalKey, normalizeCanonicalKey } from "./memory-boundaries.js";
@@ -96,15 +97,15 @@ export function assessScopeInventoryReport(report: ScopeInventoryReport): CheckR
 }
 
 function workflowPatternSeedsPath(): string {
-  return resolve(import.meta.dir, "../eval/continuity/pattern-seeds.json");
+  return resolve(metaDir(import.meta), "../eval/continuity/pattern-seeds.json");
 }
 
 function caseMemorySeedsPath(): string {
-  return resolve(import.meta.dir, "../eval/continuity/case-seeds.json");
+  return resolve(metaDir(import.meta), "../eval/continuity/case-seeds.json");
 }
 
 function memorySeedsPath(): string {
-  return resolve(import.meta.dir, "../eval/continuity/memory-seeds.json");
+  return resolve(metaDir(import.meta), "../eval/continuity/memory-seeds.json");
 }
 
 function parseMetadata(raw?: string): Record<string, unknown> {
@@ -295,16 +296,13 @@ export function assessContinuityBaseline(
 export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckResult[]> {
   const results: CheckResult[] = [];
 
-  // 1. Bun runtime
+  // 1. Runtime check
   const bunVersion = typeof Bun !== "undefined" ? Bun.version : null;
   if (bunVersion) {
-    results.push(pass("Bun runtime", `v${bunVersion}`));
+    results.push(pass("Runtime", `Bun v${bunVersion}`));
   } else {
-    results.push(fail(
-      "Bun runtime",
-      "Bun not detected (running under Node?)",
-      "Install Bun: curl -fsSL https://bun.sh/install | bash"
-    ));
+    const nodeVersion = process.version;
+    results.push(pass("Runtime", `Node.js ${nodeVersion} (Bun recommended for best performance)`));
   }
 
   // 2. Config file

package/src/dream-pipeline.ts

@@ -0,0 +1,227 @@
+/**
+ * AD-1: Unified Dream Pipeline
+ *
+ * Inspired by Claude Code's Auto Dream — a four-phase consolidation pipeline
+ * that orchestrates existing RecallNest components into a coherent "dream" cycle.
+ *
+ * Phases:
+ * 1. Orient  — scan memory state, get latest checkpoint, assess activity
+ * 2. Gather  — collect recent signals (new writes since last dream)
+ * 3. Consolidate — cluster, merge, extract patterns, generate insights
+ * 4. Prune   — archive low-value memories, enforce storage hygiene
+ *
+ * Unlike Auto Dream's grep-only approach, RecallNest uses vector search +
+ * LLM-driven consolidation for semantic-level maintenance.
+ */
+
+import type { MemoryStore } from "./store.js";
+import type { LLMClient } from "./llm-client.js";
+import type { Embedder } from "./embedder.js";
+import { ConsolidationEngine, clusterAndConsolidate, DEFAULT_CONSOLIDATION_CONFIG } from "./consolidation-engine.js";
+import { maybeRunGc, type GcResult, type AutoGcConfig, DEFAULT_AUTO_GC_CONFIG } from "./auto-gc.js";
+import { getWriteCount, resetWriteCount } from "./activity-counter.js";
+import { isActiveMemory } from "./memory-evolution.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface DreamConfig {
+  /** Minimum writes since last dream to justify running (default: 10) */
+  minWritesForDream: number;
+  /** Consolidation cluster threshold (default: 0.82) */
+  clusterThreshold: number;
+  /** Minimum cluster size for insight/pattern generation (default: 3) */
+  minClusterSize: number;
+  /** Enable cross-memory pattern extraction in consolidation (default: true) */
+  extractPatterns: boolean;
+  /** Max entries to scan per consolidation run (default: 500) */
+  maxEntriesPerRun: number;
+  /** GC config for prune phase */
+  gc: AutoGcConfig;
+}
+
+export const DEFAULT_DREAM_CONFIG: DreamConfig = {
+  minWritesForDream: 10,
+  clusterThreshold: 0.82,
+  minClusterSize: 3,
+  extractPatterns: true,
+  maxEntriesPerRun: 500,
+  gc: DEFAULT_AUTO_GC_CONFIG,
+};
+
+export interface DreamPhaseResult {
+  phase: "orient" | "gather" | "consolidate" | "prune";
+  detail: string;
+}
+
+export interface DreamResult {
+  ran: boolean;
+  reason?: string;
+  phases: DreamPhaseResult[];
+  stats: {
+    totalMemories: number;
+    activeMemories: number;
+    writesSinceLastDream: number;
+    clustersFound: number;
+    insightsGenerated: number;
+    patternsExtracted: number;
+    mergedCount: number;
+    archivedCount: number;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline
+// ---------------------------------------------------------------------------
+
+export async function runDream(params: {
+  store: MemoryStore;
+  llm: LLMClient | null;
+  embedder: Pick<Embedder, "embedPassage">;
+  scope: string;
+  config?: Partial<DreamConfig>;
+  /** Skip the minimum-writes gate (force run) */
+  force?: boolean;
+}): Promise<DreamResult> {
+  const { store, llm, embedder, scope, force = false } = params;
+  const config = { ...DEFAULT_DREAM_CONFIG, ...params.config };
+  const phases: DreamPhaseResult[] = [];
+
+  const stats = {
+    totalMemories: 0,
+    activeMemories: 0,
+    writesSinceLastDream: 0,
+    clustersFound: 0,
+    insightsGenerated: 0,
+    patternsExtracted: 0,
+    mergedCount: 0,
+    archivedCount: 0,
+  };
+
+  // =========================================================================
+  // Phase 1: Orient — assess current memory state
+  // =========================================================================
+  const writeCount = getWriteCount();
+  stats.writesSinceLastDream = writeCount;
+
+  const storeStats = await store.stats([scope]);
+  stats.totalMemories = storeStats.totalCount ?? 0;
+
+  if (!force && writeCount < config.minWritesForDream) {
+    return {
+      ran: false,
+      reason: `insufficient_writes (${writeCount}/${config.minWritesForDream})`,
+      phases: [{
+        phase: "orient",
+        detail: `${stats.totalMemories} memories, ${writeCount} writes since last dream — below threshold`,
+      }],
+      stats,
+    };
+  }
+
+  phases.push({
+    phase: "orient",
+    detail: `${stats.totalMemories} memories, ${writeCount} writes since last dream`,
+  });
+
+  // =========================================================================
+  // Phase 2: Gather — collect active entries for consolidation
+  // =========================================================================
+  const entries = await store.list([scope], undefined, config.maxEntriesPerRun, 0);
+  const active = entries.filter(e => isActiveMemory(e.metadata));
+  stats.activeMemories = active.length;
+
+  phases.push({
+    phase: "gather",
+    detail: `${active.length} active entries gathered from ${entries.length} total`,
+  });
+
+  if (active.length < config.minClusterSize) {
+    phases.push({ phase: "consolidate", detail: "skipped — too few active entries" });
+    phases.push({ phase: "prune", detail: "skipped — too few entries for GC" });
+    resetWriteCount();
+    return { ran: true, reason: "completed_early", phases, stats };
+  }
+
+  // =========================================================================
+  // Phase 3: Consolidate — cluster, merge, generate insights + patterns
+  // =========================================================================
+
+  // 3a: Deterministic consolidation (merge near-duplicates, link clusters)
+  const engine = new ConsolidationEngine(store, {
+    ...DEFAULT_CONSOLIDATION_CONFIG,
+    clusterThreshold: config.clusterThreshold,
+    maxEntriesPerRun: config.maxEntriesPerRun,
+  });
+  const consolidation = await engine.run(scope);
+  stats.clustersFound += consolidation.clustersFound;
+  stats.mergedCount += consolidation.mergedCount;
+
+  // 3b: LLM-driven cluster consolidation (insights + patterns) — requires LLM
+  if (llm) {
+    const clusterResult = await clusterAndConsolidate({
+      entries: active,
+      embedder,
+      llm,
+      store,
+      scope,
+      minClusterSize: config.minClusterSize,
+      clusterThreshold: config.clusterThreshold - 0.07, // slightly lower for semantic clustering
+      extractPatterns: config.extractPatterns,
+    });
+    stats.clustersFound += clusterResult.clustersFound;
+    stats.insightsGenerated = clusterResult.insightsGenerated;
+    stats.patternsExtracted = clusterResult.patternsExtracted;
+  }
+
+  phases.push({
+    phase: "consolidate",
+    detail: `${stats.clustersFound} clusters, ${stats.mergedCount} merged, ${stats.insightsGenerated} insights, ${stats.patternsExtracted} patterns`,
+  });
+
+  // =========================================================================
+  // Phase 4: Prune — archive low-value memories
+  // =========================================================================
+  const gcResult = await maybeRunGc(store, config.gc);
+  stats.archivedCount = gcResult.archivedCount;
+
+  phases.push({
+    phase: "prune",
+    detail: gcResult.triggered
+      ? `${gcResult.archivedCount} entries archived`
+      : `skipped — ${gcResult.reason}`,
+  });
+
+  // Reset activity counter after successful dream
+  resetWriteCount();
+
+  return { ran: true, phases, stats };
+}
+
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+
+export function formatDreamResult(result: DreamResult): string {
+  if (!result.ran) {
+    return `Dream skipped: ${result.reason}\n${result.phases[0]?.detail ?? ""}`;
+  }
+
+  const lines = [
+    "Dream completed",
+    "",
+    ...result.phases.map(p => `[${p.phase}] ${p.detail}`),
+    "",
+    "Stats:",
+    `  Total memories: ${result.stats.totalMemories}`,
+    `  Active: ${result.stats.activeMemories}`,
+    `  Clusters: ${result.stats.clustersFound}`,
+    `  Insights: ${result.stats.insightsGenerated}`,
+    `  Patterns: ${result.stats.patternsExtracted}`,
+    `  Merged: ${result.stats.mergedCount}`,
+    `  Archived: ${result.stats.archivedCount}`,
+  ];
+
+  return lines.join("\n");
+}

package/src/emotion-detector.ts

@@ -0,0 +1,90 @@
+import type { EmotionMetadata } from "./memory-schema.js";
+import { isEmotionScoringEnabled } from "./memory-schema.js";
+
+const NEGATIVE_SIGNALS: string[] = [
+  // English
+  "fail", "failed", "failure", "broken", "bug", "error", "wrong", "crash",
+  "frustrat", "hate", "terrible", "awful", "annoying", "pain", "stuck",
+  "problem", "issue", "mess", "ugly", "worst",
+  // Chinese
+  "失败", "痛苦", "困扰", "崩溃", "报错", "出错", "难受", "烦",
+  "卡住", "折腾", "头疼", "坑", "讨厌", "不喜欢", "糟糕", "恶心",
+];
+
+const POSITIVE_SIGNALS: string[] = [
+  // English
+  "solved", "fixed", "works", "perfect", "great", "love", "excellent",
+  "success", "breakthrough", "finally", "awesome", "beautiful", "clean",
+  "elegant", "smooth", "done", "shipped",
+  // Chinese
+  "搞定", "成功", "突破", "完美", "太好了", "顺利", "解决", "漂亮",
+  "优雅", "通过", "上线", "喜欢", "开心", "厉害",
+];
+
+const HIGH_AROUSAL_SIGNALS: string[] = [
+  "!", "!!", "urgent", "critical", "immediately", "ASAP", "emergency",
+  "紧急", "立刻", "马上", "赶紧", "救命",
+];
+
+/** Chinese negation prefixes that flip positive → negative */
+const CN_NEGATION_PREFIXES = ["不", "没", "非", "无", "别", "未"];
+
+function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
+/**
+ * Count positive signal matches, handling Chinese negation.
+ * "不喜欢" → skip positive, add to negatedCount (caller adds to negative).
+ * Returns { positive, negated } counts.
+ */
+function countPositiveSignals(lower: string): { positive: number; negated: number } {
+  let positive = 0;
+  let negated = 0;
+  for (const signal of POSITIVE_SIGNALS) {
+    const s = signal.toLowerCase();
+    const idx = lower.indexOf(s);
+    if (idx === -1) continue;
+    const isNegated = idx > 0 && CN_NEGATION_PREFIXES.some(neg => lower.substring(idx - neg.length, idx) === neg);
+    if (isNegated) {
+      negated++;
+    } else {
+      positive++;
+    }
+  }
+  return { positive, negated };
+}
+
+/**
+ * Detect emotional valence and arousal from text using keyword heuristics.
+ * Zero LLM cost. Returns neutral emotion for empty text.
+ */
+export function detectEmotion(text: string): EmotionMetadata {
+  if (!text || text.length === 0) {
+    return { valence: 0, arousal: 0, label: "neutral", salience: 0, source: "keyword" };
+  }
+
+  const lower = text.toLowerCase();
+
+  const negCount = NEGATIVE_SIGNALS.filter(s => lower.includes(s.toLowerCase())).length;
+  const { positive: posCount, negated: negatedPosCount } = countPositiveSignals(lower);
+  const arousalCount = HIGH_AROUSAL_SIGNALS.filter(s => text.includes(s)).length;
+
+  // Negated positive signals (e.g. "不开心") count as additional negatives
+  const valence = clamp((posCount - negCount - negatedPosCount) * 0.3, -1, 1);
+  const arousal = clamp(arousalCount * 0.25, 0, 1);
+  const label = valence > 0.25 ? "positive" : valence < -0.25 ? "negative" : "neutral";
+
+  // Salience: composite mnemonic significance — average of emotional intensity and arousal
+  const salience = clamp((Math.abs(valence) + arousal) / 2, 0, 1);
+
+  return { valence, arousal, label, salience, source: "keyword" };
+}
+
+/**
+ * Conditionally detect emotion. Returns null when feature flag is off.
+ */
+export function detectEmotionIfEnabled(text: string): EmotionMetadata | null {
+  if (!isEmotionScoringEnabled()) return null;
+  return detectEmotion(text);
+}

package/src/eval.ts

@@ -3,6 +3,7 @@
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { dirname, resolve } from "node:path";
 
+import { metaDir } from "./compat.js";
 import { buildSessionCheckpointRecord, normalizeCheckpointScope } from "./session-engine.js";
 import type { ResumeContextResponse, SessionCheckpointRecord } from "./session-schema.js";
 import { composeResumeContext } from "./context-composer.js";
@@ -158,8 +159,8 @@ function parseArgs(args: string[]): EvalArgs {
 
 function defaultCasesPath(mode: EvalMode): string {
   return mode === "continuity"
-    ? resolve(import.meta.dir, "../eval/continuity/cases.json")
-    : resolve(import.meta.dir, "../eval/cases.json");
+    ? resolve(metaDir(import.meta), "../eval/continuity/cases.json")
+    : resolve(metaDir(import.meta), "../eval/cases.json");
 }
 
 function loadCases<T>(mode: EvalMode, pathArg?: string): T[] {