recallnest 1.4.1

package/src/retroactive-boost.ts

@@ -0,0 +1,154 @@
+/**
+ * Retroactive Association Boost (STC — Synaptic Tagging & Capture)
+ *
+ * Agent-First reasoning: when a new high-importance memory is stored
+ * (e.g. "I'm moving to Singapore"), older weak fragments that are
+ * semantically related (e.g. "Singapore weather notes") become more
+ * valuable. Without this, those old fragments stay buried at low
+ * importance and never surface in relevant retrievals.
+ *
+ * Brain-science label: STC hypothesis — a strong event "captures"
+ * weakly tagged memories and promotes them. We keep the name for
+ * branding; the real mechanism is a vector similarity search +
+ * importance bump on related old entries.
+ *
+ * Rules:
+ * - Only triggers on high-importance writes (≥ threshold)
+ * - Only boosts entries that are currently low-importance (below cap)
+ * - Boost is additive with a ceiling — never exceeds maxBoostedImportance
+ * - Writes metadata breadcrumb for auditability
+ * - No LLM calls — pure vector search + arithmetic
+ */
+
+import type { MemoryStore, MemoryEntry } from "./store.js";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface RetroactiveBoostConfig {
+  /** Minimum importance of the NEW memory to trigger a boost sweep (default: 0.8) */
+  triggerImportanceMin: number;
+  /** Only boost entries whose current importance is below this (default: 0.6) */
+  boostCandidateMaxImportance: number;
+  /** Additive importance bump (default: 0.15) */
+  boostAmount: number;
+  /** Ceiling: boosted importance never exceeds this (default: 0.75) */
+  maxBoostedImportance: number;
+  /** Minimum vector similarity to consider "related" (default: 0.72) */
+  similarityThreshold: number;
+  /** Maximum entries to boost per trigger (default: 5) */
+  maxBoostPerTrigger: number;
+  /** Minimum age in days for a candidate (avoid boosting very recent entries) (default: 1) */
+  minAgeDays: number;
+}
+
+export const DEFAULT_RETROACTIVE_BOOST_CONFIG: RetroactiveBoostConfig = {
+  triggerImportanceMin: 0.8,
+  boostCandidateMaxImportance: 0.6,
+  boostAmount: 0.15,
+  maxBoostedImportance: 0.75,
+  similarityThreshold: 0.72,
+  maxBoostPerTrigger: 5,
+  minAgeDays: 1,
+};
+
+// ---------------------------------------------------------------------------
+// Core
+// ---------------------------------------------------------------------------
+
+export interface BoostResult {
+  triggered: boolean;
+  reason?: string;
+  boostedCount: number;
+  boostedIds: string[];
+}
+
+/**
+ * After a high-importance memory is stored, search for related older
+ * low-importance entries and boost them.
+ *
+ * Call this right after store.add() for the new entry.
+ */
+export async function retroactiveBoost(
+  store: MemoryStore,
+  newEntry: Pick<MemoryEntry, "id" | "text" | "vector" | "importance" | "scope" | "timestamp">,
+  config: RetroactiveBoostConfig = DEFAULT_RETROACTIVE_BOOST_CONFIG,
+): Promise<BoostResult> {
+  // Gate: only trigger for high-importance new memories
+  if (newEntry.importance < config.triggerImportanceMin) {
+    return { triggered: false, reason: "below_importance_threshold", boostedCount: 0, boostedIds: [] };
+  }
+
+  if (!newEntry.vector?.length) {
+    return { triggered: false, reason: "no_vector", boostedCount: 0, boostedIds: [] };
+  }
+
+  // Find related entries in the same scope
+  const candidates = await store.vectorSearch(
+    newEntry.vector,
+    config.maxBoostPerTrigger * 3, // fetch more, filter down
+    config.similarityThreshold,
+    [newEntry.scope],
+  );
+
+  const now = Date.now();
+  const minAgeMs = config.minAgeDays * 86_400_000;
+  const boostedIds: string[] = [];
+
+  for (const candidate of candidates) {
+    if (boostedIds.length >= config.maxBoostPerTrigger) break;
+
+    const entry = candidate.entry;
+
+    // Skip self
+    if (entry.id === newEntry.id) continue;
+
+    // Skip entries that are already high importance
+    if (entry.importance >= config.boostCandidateMaxImportance) continue;
+
+    // Skip very recent entries (they'll get their own natural scoring)
+    const ageDays = (now - entry.timestamp) / 86_400_000;
+    if (ageDays < config.minAgeDays) continue;
+
+    // Compute boost: additive, capped
+    const newImportance = Math.min(
+      entry.importance + config.boostAmount,
+      config.maxBoostedImportance,
+    );
+
+    // Skip if no actual change
+    if (newImportance <= entry.importance) continue;
+
+    // Update metadata with audit trail
+    let meta: Record<string, any> = {};
+    try { meta = JSON.parse(entry.metadata || "{}"); } catch { /* skip */ }
+
+    if (!Array.isArray(meta.stc_boosts)) meta.stc_boosts = [];
+    meta.stc_boosts.push({
+      triggeredBy: newEntry.id.slice(0, 8),
+      from: entry.importance,
+      to: newImportance,
+      similarity: candidate.score,
+      date: new Date().toISOString().slice(0, 10),
+    });
+
+    // Promote tier if appropriate
+    if (meta.tier === "peripheral" && newImportance >= 0.5) {
+      meta.tier = "working";
+    }
+
+    await store.update(entry.id, {
+      importance: newImportance,
+      metadata: JSON.stringify(meta),
+    }, [newEntry.scope]);
+
+    boostedIds.push(entry.id);
+  }
+
+  return {
+    triggered: true,
+    boostedCount: boostedIds.length,
+    boostedIds,
+  };
+}

package/src/rif.ts

@@ -0,0 +1,73 @@
+/**
+ * Retrieval Interference Filter (RIF)
+ *
+ * Brain-inspired: retrieval-induced forgetting — retrieving A actively
+ * suppresses similar-but-not-selected B, improving future signal-to-noise.
+ *
+ * In practice: after scoring, demote near-duplicate results from the same
+ * topic that are significantly weaker than a higher-ranked result.
+ * They're moved to the end (not removed) as fallback.
+ */
+
+import type { RetrievalResult } from "./retriever.js";
+
+// ---------------------------------------------------------------------------
+// Core
+// ---------------------------------------------------------------------------
+
+/**
+ * Filter interference: demote near-duplicate results that are significantly
+ * weaker than a higher-ranked similar result.
+ *
+ * @param results - Scored results sorted by score descending
+ * @param similarityThreshold - Cosine similarity above which two results are "near-duplicate" (default: 0.85)
+ * @param scoreRatio - A result scoring below this ratio of its near-duplicate is demoted (default: 0.80)
+ */
+export function filterInterference(
+  results: RetrievalResult[],
+  similarityThreshold = 0.85,
+  scoreRatio = 0.80,
+): RetrievalResult[] {
+  if (results.length <= 2) return results;
+
+  const kept: RetrievalResult[] = [];
+  const demoted: RetrievalResult[] = [];
+
+  for (const candidate of results) {
+    // Check if this candidate is a weak near-duplicate of any kept result
+    let isDemoted = false;
+
+    for (const selected of kept) {
+      const sim = cosineSimilarity(candidate.entry.vector, selected.entry.vector);
+      if (sim > similarityThreshold && candidate.score < selected.score * scoreRatio) {
+        isDemoted = true;
+        break;
+      }
+    }
+
+    if (isDemoted) {
+      demoted.push(candidate);
+    } else {
+      kept.push(candidate);
+    }
+  }
+
+  // Demoted results go to the end as fallback
+  return [...kept, ...demoted];
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function cosineSimilarity(a: number[] | Float32Array, b: number[] | Float32Array): number {
+  if (!a || !b || a.length !== b.length || a.length === 0) return 0;
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom > 0 ? dot / denom : 0;
+}

package/src/runtime-config.ts

@@ -0,0 +1,172 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolve, join } from "node:path";
+import { homedir } from "node:os";
+
+import { MemoryStore, validateStoragePath } from "./store.js";
+import { createEmbedder, getVectorDimensions, type EmbeddingConfig } from "./embedder.js";
+import { createRetriever, type RetrievalConfig, DEFAULT_RETRIEVAL_CONFIG } from "./retriever.js";
+import { applyRetrievalProfile } from "./retrieval-profiles.js";
+import { AccessTracker } from "./access-tracker.js";
+import { createLLMClient, type LLMClient, type LLMConfig } from "./llm-client.js";
+import { logInfo } from "./stderr-log.js";
+
+export type RecallMode = "full" | "summary" | "off";
+
+export interface LocalMemoryConfig {
+  dbPath: string;
+  recallMode?: RecallMode;
+  embedding: {
+    provider: string;
+    apiKey: string;
+    model: string;
+    baseURL?: string;
+    dimensions?: number;
+    taskQuery?: string;
+    taskPassage?: string;
+  };
+  llm?: {
+    apiKey: string;
+    model: string;
+    baseURL: string;
+  };
+  sources: Record<string, { path: string; glob: string; description: string }>;
+  retrieval?: Partial<RetrievalConfig>;
+  /**
+   * Default depth for auto-recall injection.
+   * - "full": inject complete text (default, backward compatible)
+   * - "l1": inject L1 overview from metadata (~500 tokens)
+   * - "l0": inject L0 abstract from metadata (~100 tokens)
+   * Agent can use memory_drill_down tool to get deeper content on demand.
+   */
+  recallDepthDefault?: "l0" | "l1" | "full";
+}
+
+export function loadDotEnv(): void {
+  const envPath = resolve(import.meta.dir, "../.env");
+  if (!existsSync(envPath)) return;
+  const content = readFileSync(envPath, "utf-8");
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eqIdx = trimmed.indexOf("=");
+    if (eqIdx > 0) {
+      const key = trimmed.slice(0, eqIdx).trim();
+      const val = trimmed.slice(eqIdx + 1).trim();
+      if (!process.env[key]) process.env[key] = val;
+    }
+  }
+}
+
+export function findConfigPath(): string {
+  if (process.env.LOCAL_MEMORY_CONFIG) return resolve(process.env.LOCAL_MEMORY_CONFIG);
+
+  const localConfig = resolve(import.meta.dir, "../config.json");
+  if (existsSync(localConfig)) return localConfig;
+
+  const branded = join(homedir(), ".config", "recallnest", "config.json");
+  if (existsSync(branded)) return branded;
+
+  const exampleExists = existsSync(resolve(import.meta.dir, "../config.json.example"));
+  throw new Error(
+    "Config not found.\n" +
+    (exampleExists
+      ? "  Quick fix: cp config.json.example config.json\n"
+      : "") +
+    "  Or set LOCAL_MEMORY_CONFIG env var, or place config.json in ~/.config/recallnest/"
+  );
+}
+
+export function loadConfig(): LocalMemoryConfig {
+  const raw = readFileSync(findConfigPath(), "utf-8");
+  return JSON.parse(raw) as LocalMemoryConfig;
+}
+
+export function resolveEnv(value: string): string {
+  return value.replace(/\$\{([^}]+)\}/g, (_, name) => {
+    const envVal = process.env[name];
+    if (!envVal) throw new Error(`Environment variable ${name} not set`);
+    return envVal;
+  });
+}
+
+export function expandHome(p: string): string {
+  if (p.startsWith("~/")) return join(homedir(), p.slice(2));
+  return p;
+}
+
+export function createComponents(config: LocalMemoryConfig, profileName?: string) {
+  const dbPath = resolve(import.meta.dir, "..", expandHome(config.dbPath));
+  validateStoragePath(dbPath);
+
+  const embeddingConfig: EmbeddingConfig = {
+    provider: "openai-compatible",
+    apiKey: resolveEnv(config.embedding.apiKey),
+    model: config.embedding.model,
+    baseURL: config.embedding.baseURL,
+    dimensions: config.embedding.dimensions,
+    taskQuery: config.embedding.taskQuery,
+    taskPassage: config.embedding.taskPassage,
+  };
+
+  const embedder = createEmbedder(embeddingConfig);
+  const store = new MemoryStore({ dbPath, vectorDim: embedder.dimensions });
+  const baseRetrievalConfig = {
+    ...DEFAULT_RETRIEVAL_CONFIG,
+    ...(config.retrieval || {}),
+  };
+  const { profile, config: retrieverConfig } = applyRetrievalProfile(baseRetrievalConfig, profileName);
+  const retriever = createRetriever(store, embedder, retrieverConfig);
+
+  // Attach access tracker for reinforcement-based decay
+  const accessTracker = new AccessTracker(store);
+  retriever.setAccessTracker(accessTracker);
+
+  // Create LLM client if configured (optional, graceful)
+  let llm: LLMClient | null = null;
+  if (config.llm) {
+    llm = createLLMClient(config.llm);
+    if (llm) {
+      logInfo(`[INFO] LLM client initialized: ${config.llm.model} @ ${config.llm.baseURL}`);
+    }
+  }
+
+  return { store, embedder, retriever, profile, accessTracker, llm };
+}
+
+export function createStoreOnly(config: LocalMemoryConfig): MemoryStore {
+  const dbPath = resolve(import.meta.dir, "..", expandHome(config.dbPath));
+  validateStoragePath(dbPath);
+  return new MemoryStore({
+    dbPath,
+    vectorDim: getVectorDimensions(config.embedding.model, config.embedding.dimensions),
+  });
+}
+
+export function createComponentResolver(config: LocalMemoryConfig) {
+  const cache = new Map<string, ReturnType<typeof createComponents>>();
+
+  return function getComponents(profileName?: string) {
+    const key = profileName || "default";
+    const cached = cache.get(key);
+    if (cached) return cached;
+    const created = createComponents(config, profileName);
+    cache.set(key, created);
+    return created;
+  };
+}
+
+const VALID_RECALL_MODES: RecallMode[] = ["full", "summary", "off"];
+
+/**
+ * Resolve effective recall mode: per-call override > env var > config > default ("summary").
+ */
+export function resolveRecallMode(config: LocalMemoryConfig, perCallOverride?: string): RecallMode {
+  if (perCallOverride && VALID_RECALL_MODES.includes(perCallOverride as RecallMode)) {
+    return perCallOverride as RecallMode;
+  }
+  const envMode = process.env.RECALLNEST_RECALL_MODE;
+  if (envMode && VALID_RECALL_MODES.includes(envMode as RecallMode)) {
+    return envMode as RecallMode;
+  }
+  return config.recallMode ?? "summary";
+}

package/src/schema-utils.ts

@@ -0,0 +1,56 @@
+import { z } from "zod";
+
+function collapseWhitespace(value: string): string {
+  return value.replace(/\s+/g, " ").trim();
+}
+
+export function boundedStringSchema(field: string, maxLength: number) {
+  return z.string({ required_error: `${field} is required` })
+    .transform(collapseWhitespace)
+    .pipe(
+      z.string()
+        .min(1, `${field} is required`)
+        .max(maxLength, `${field} must be at most ${maxLength} characters`),
+    );
+}
+
+export function optionalBoundedStringSchema(maxLength: number) {
+  return z.string()
+    .transform(collapseWhitespace)
+    .pipe(z.string().min(1).max(maxLength))
+    .optional();
+}
+
+export function identifierSchema(field: string, maxLength = 160) {
+  return z.string({ required_error: `${field} is required` })
+    .transform(collapseWhitespace)
+    .pipe(
+      z.string()
+        .min(1, `${field} is required`)
+        .max(maxLength, `${field} must be at most ${maxLength} characters`),
+    );
+}
+
+export function normalizedStringListSchema(
+  field: string,
+  maxItems: number,
+  maxItemLength: number,
+) {
+  return z.array(boundedStringSchema(field, maxItemLength))
+    .default([])
+    .transform((items) => {
+      const deduped: string[] = [];
+      const seen = new Set<string>();
+      for (const item of items) {
+        const key = item.toLowerCase();
+        if (seen.has(key)) continue;
+        seen.add(key);
+        deduped.push(item);
+      }
+      return deduped;
+    })
+    .pipe(
+      z.array(z.string())
+        .max(maxItems, `${field} must contain at most ${maxItems} items`),
+    );
+}