recallnest 1.4.1

package/src/asset-sync.ts

@@ -0,0 +1,79 @@
+import type { Embedder } from "./embedder.js";
+import type { MemoryStore } from "./store.js";
+import type { BriefAsset, MemoryAsset, PinAsset } from "./memory-assets.js";
+
+function buildPinnedAssetText(asset: PinAsset): string {
+  const tags = asset.tags.join(", ");
+  return [
+    `[Pinned Asset] ${asset.title}`,
+    `Summary: ${asset.summary}`,
+    `Snippet: ${asset.snippet}`,
+    `Original Scope: ${asset.source.scope}`,
+    tags ? `Tags: ${tags}` : "",
+  ].filter(Boolean).join("\n");
+}
+
+function buildBriefAssetText(asset: BriefAsset): string {
+  const tags = asset.tags.join(", ");
+  return [
+    `[Memory Brief] ${asset.title}`,
+    `Query: ${asset.query}`,
+    `Profile: ${asset.profile}`,
+    `Summary: ${asset.summary}`,
+    asset.takeaways.length > 0 ? `Takeaways: ${asset.takeaways.join(" | ")}` : "",
+    asset.reusableCandidates.length > 0 ? `Reusable: ${asset.reusableCandidates.join(" | ")}` : "",
+    asset.sources.length > 0 ? `Sources: ${asset.sources.map((item) => `${item.source}(${item.hits})`).join(", ")}` : "",
+    tags ? `Tags: ${tags}` : "",
+  ].filter(Boolean).join("\n");
+}
+
+function buildAssetText(asset: MemoryAsset): string {
+  return asset.type === "memory-brief"
+    ? buildBriefAssetText(asset)
+    : buildPinnedAssetText(asset);
+}
+
+function assetImportance(asset: MemoryAsset): number {
+  return asset.type === "memory-brief" ? 0.72 : 0.96;
+}
+
+export async function indexAsset(
+  store: MemoryStore,
+  embedder: Embedder,
+  asset: MemoryAsset,
+): Promise<void> {
+  const text = buildAssetText(asset);
+  const vector = await embedder.embedPassage(text);
+  const metadata = asset.type === "memory-brief"
+    ? {
+        source: "asset",
+        assetId: asset.id,
+        assetType: asset.type,
+        title: asset.title,
+        query: asset.query,
+        profile: asset.profile,
+        tags: asset.tags,
+        sources: asset.sources.map((item) => item.source),
+      }
+    : {
+        source: "asset",
+        assetId: asset.id,
+        assetType: asset.type,
+        title: asset.title,
+        originalMemoryId: asset.source.memoryId,
+        originalScope: asset.source.scope,
+        tags: asset.tags,
+      };
+  await store.store({
+    text,
+    vector,
+    category: "decision",
+    scope: asset.type === "memory-brief"
+      ? `asset:brief:${asset.id.slice(0, 8)}`
+      : `asset:${asset.id.slice(0, 8)}`,
+    importance: assetImportance(asset),
+    metadata: JSON.stringify(metadata),
+  });
+}
+
+export const indexPinnedAsset = indexAsset;

package/src/auto-consolidation.ts

@@ -0,0 +1,133 @@
+/**
+ * Auto Consolidation — Condition-driven consolidation trigger.
+ *
+ * Agent-First reasoning: manual consolidation is unsustainable. Trigger
+ * automatically when enough new memories have accumulated AND enough time
+ * has passed since the last run. Same dual-gate pattern as auto-gc.ts.
+ *
+ * Brain-science label: "NREM triple-coupling" — multiple conditions must
+ * align before consolidation fires. The label is for branding; the real
+ * reason is operational hygiene.
+ */
+
+import type { MemoryStore } from "./store.js";
+import { ConsolidationEngine, type ConsolidationConfig, type ConsolidationResult, DEFAULT_CONSOLIDATION_CONFIG } from "./consolidation-engine.js";
+import type { LLMClient } from "./llm-client.js";
+import { isLLMConsolidationEnabled, evaluateCluster, executeMergeDecisions } from "./llm-consolidation.js";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface AutoConsolidationConfig {
+  /** Minimum new memories since last consolidation before triggering (default: 50) */
+  minNewMemories: number;
+  /** Minimum hours since last consolidation (default: 12) */
+  minHoursSinceLastRun: number;
+  /** Consolidation engine config (cluster/merge thresholds) */
+  consolidation: ConsolidationConfig;
+}
+
+export const DEFAULT_AUTO_CONSOLIDATION_CONFIG: AutoConsolidationConfig = {
+  minNewMemories: 50,
+  minHoursSinceLastRun: 12,
+  consolidation: DEFAULT_CONSOLIDATION_CONFIG,
+};
+
+// ---------------------------------------------------------------------------
+// State
+// ---------------------------------------------------------------------------
+
+let lastRunTimestamp = 0;
+let lastRunMemoryCount = 0;
+
+// ---------------------------------------------------------------------------
+// Core
+// ---------------------------------------------------------------------------
+
+export interface AutoConsolidationResult {
+  triggered: boolean;
+  reason?: string;
+  consolidation?: ConsolidationResult;
+}
+
+/**
+ * Check conditions and run consolidation if thresholds are met.
+ * Returns immediately if conditions not met (no-op).
+ */
+export async function maybeConsolidate(
+  store: MemoryStore,
+  scope: string,
+  config: AutoConsolidationConfig = DEFAULT_AUTO_CONSOLIDATION_CONFIG,
+  /** Tier 3.7: Optional LLM client for semantic consolidation decisions */
+  llm?: LLMClient | null,
+): Promise<AutoConsolidationResult> {
+  const stats = await store.stats([scope]);
+  const currentCount = stats.total ?? 0;
+
+  // Condition 1: enough new memories since last run
+  const newSinceLastRun = currentCount - lastRunMemoryCount;
+  if (newSinceLastRun < config.minNewMemories) {
+    return { triggered: false, reason: "insufficient_new_memories" };
+  }
+
+  // Condition 2: enough time since last run
+  const hoursSinceLastRun = (Date.now() - lastRunTimestamp) / 3_600_000;
+  if (hoursSinceLastRun < config.minHoursSinceLastRun) {
+    return { triggered: false, reason: "too_soon" };
+  }
+
+  // Both conditions met — run consolidation
+  lastRunTimestamp = Date.now();
+  lastRunMemoryCount = currentCount;
+
+  const engine = new ConsolidationEngine(store, config.consolidation);
+  const result = await engine.run(scope);
+
+  // Tier 3.7: LLM post-processing of linked clusters.
+  // When enabled, LLM evaluates "clustered_with" relations and may
+  // upgrade them to version groups (merges) based on semantic judgment.
+  if (isLLMConsolidationEnabled() && llm && result.relationsAdded > 0) {
+    let llmMerges = 0;
+    // Find entries with clustered_with that aren't already in version groups
+    const allEntries = await store.list([scope], undefined, config.consolidation.maxEntriesPerRun ?? 500);
+    const clusterMap = new Map<string, MemoryEntry[]>();
+
+    for (const entry of allEntries) {
+      try {
+        const meta = JSON.parse(entry.metadata || "{}");
+        const members = meta.cluster_members as string[] | undefined;
+        if (Array.isArray(members) && members.length > 0 && !meta.version_group) {
+          const cluster = [entry];
+          for (const memberId of members) {
+            const member = await store.getById(memberId);
+            if (member) cluster.push(member);
+          }
+          if (cluster.length >= 2) {
+            clusterMap.set(entry.id, cluster);
+          }
+        }
+      } catch { /* skip */ }
+    }
+
+    for (const [, cluster] of clusterMap) {
+      const decision = await evaluateCluster(llm, cluster);
+      const merges = await executeMergeDecisions(store, cluster, decision, scope);
+      llmMerges += merges;
+    }
+
+    if (llmMerges > 0) {
+      result.mergedCount += llmMerges;
+    }
+  }
+
+  return { triggered: true, consolidation: result };
+}
+
+/**
+ * Reset state (for testing).
+ */
+export function resetConsolidationState(): void {
+  lastRunTimestamp = 0;
+  lastRunMemoryCount = 0;
+}

package/src/auto-gc.ts

@@ -0,0 +1,124 @@
+/**
+ * Auto Garbage Collection — Brain-inspired active forgetting.
+ *
+ * Condition-driven trigger (not cron): runs when memory count exceeds threshold
+ * AND enough time has passed since last GC. Inspired by NREM triple-coupling:
+ * multiple conditions must be satisfied before consolidation fires.
+ *
+ * Wraps existing cleanup scripts into a programmatic API.
+ */
+
+import type { MemoryStore } from "./store.js";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface AutoGcConfig {
+  /** Minimum memories before GC can trigger (default: 1000) */
+  minMemoryCount: number;
+  /** Minimum hours since last GC (default: 24) */
+  minHoursSinceLastGc: number;
+  /** Score below which memories are candidates for archival (default: 0.2) */
+  staleScoreThreshold: number;
+  /** Max entries to archive per run (default: 100) */
+  maxArchivePerRun: number;
+}
+
+export const DEFAULT_AUTO_GC_CONFIG: AutoGcConfig = {
+  minMemoryCount: 1000,
+  minHoursSinceLastGc: 24,
+  staleScoreThreshold: 0.2,
+  maxArchivePerRun: 100,
+};
+
+// ---------------------------------------------------------------------------
+// State
+// ---------------------------------------------------------------------------
+
+let lastGcTimestamp = 0;
+
+// ---------------------------------------------------------------------------
+// Core
+// ---------------------------------------------------------------------------
+
+export interface GcResult {
+  triggered: boolean;
+  reason?: string;
+  archivedCount: number;
+  totalChecked: number;
+}
+
+/**
+ * Check conditions and run GC if all thresholds are met.
+ * Returns immediately if conditions not met (no-op).
+ */
+export async function maybeRunGc(
+  store: MemoryStore,
+  config: AutoGcConfig = DEFAULT_AUTO_GC_CONFIG,
+): Promise<GcResult> {
+  const stats = await store.stats();
+  const totalMemories = stats.total ?? 0;
+
+  // Condition 1: enough memories to warrant GC
+  if (totalMemories < config.minMemoryCount) {
+    return { triggered: false, reason: "below_memory_threshold", archivedCount: 0, totalChecked: 0 };
+  }
+
+  // Condition 2: enough time since last GC
+  const hoursSinceLastGc = (Date.now() - lastGcTimestamp) / 3_600_000;
+  if (hoursSinceLastGc < config.minHoursSinceLastGc) {
+    return { triggered: false, reason: "too_soon", archivedCount: 0, totalChecked: 0 };
+  }
+
+  // All conditions met — run GC
+  lastGcTimestamp = Date.now();
+
+  // Find stale peripheral memories with low importance and no recent access
+  const cutoffMs = Date.now() - 60 * 86_400_000; // 60 days ago
+  let archivedCount = 0;
+  let totalChecked = 0;
+
+  // Scan memories by listing them
+  const entries = await store.list({ limit: 5000 });
+  totalChecked = entries.length;
+
+  for (const entry of entries) {
+    if (archivedCount >= config.maxArchivePerRun) break;
+
+    let meta: Record<string, any> = {};
+    try { meta = JSON.parse(entry.metadata || "{}"); } catch { /* skip */ }
+
+    const isArchived = meta.archived === true;
+    if (isArchived) continue;
+
+    const tier = meta.tier || "peripheral";
+    const accessCount = meta.accessCount || 0;
+    const importance = entry.importance ?? 0;
+    const age = Date.now() - entry.timestamp;
+    const ageDays = age / 86_400_000;
+
+    // Archive criteria: peripheral + old + low importance + rarely accessed
+    if (
+      tier === "peripheral" &&
+      ageDays > 60 &&
+      importance < config.staleScoreThreshold &&
+      accessCount < 2
+    ) {
+      meta.archived = true;
+      meta.archivedAt = new Date().toISOString();
+      meta.archivedReason = "auto-gc:stale-peripheral";
+      await store.update(entry.id, { metadata: JSON.stringify(meta) });
+      archivedCount++;
+    }
+  }
+
+  return { triggered: true, archivedCount, totalChecked };
+}
+
+/**
+ * Reset last GC timestamp (for testing).
+ */
+export function resetGcTimestamp(): void {
+  lastGcTimestamp = 0;
+}

package/src/auto-recall.ts

@@ -0,0 +1,95 @@
+import { composeResumeContext, type ResumeContextDeps } from "./context-composer.js";
+import type { DurableMemoryCategory } from "./memory-schema.js";
+import type { RetrievalResult } from "./retriever.js";
+import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
+import type { RetrievalProfileName, ResumeContextResponse } from "./session-schema.js";
+
+export interface AutoRecallRequest {
+  message: string;
+  task?: string;
+  scope?: string;
+  sessionId?: string;
+  allScopes?: boolean;
+  limit?: number;
+  limitPerSection?: number;
+  includeLatestCheckpoint?: boolean;
+  category?: DurableMemoryCategory;
+  profile?: RetrievalProfileName;
+  env?: NodeJS.ProcessEnv;
+  operation?: string;
+}
+
+export interface AutoRecallResponse {
+  mode: "resume-only" | "resume+search";
+  resolvedScope?: string;
+  resume: ResumeContextResponse;
+  results: RetrievalResult[];
+  searchSkippedReason?: string;
+}
+
+export async function runAutoRecall(
+  deps: ResumeContextDeps,
+  request: AutoRecallRequest,
+): Promise<AutoRecallResponse> {
+  const message = request.message.replace(/\s+/g, " ").trim();
+  const task = (request.task || message).replace(/\s+/g, " ").trim();
+
+  if (!message) {
+    throw new Error("message is required");
+  }
+
+  const scopeSelection = resolveScopeSelection({
+    scope: request.scope,
+    sessionId: request.sessionId,
+    allScopes: request.allScopes,
+    operation: request.operation || "auto_recall",
+    env: request.env,
+    allowUnscoped: true,
+  });
+  const resumeScope = scopeSelection.inferredFrom === "sessionId"
+    ? undefined
+    : scopeSelection.resolvedScope;
+
+  const resume = await composeResumeContext(deps, {
+    task,
+    scope: resumeScope,
+    sessionId: request.sessionId,
+    limitPerSection: request.limitPerSection,
+    includeLatestCheckpoint: request.includeLatestCheckpoint,
+    profile: request.profile,
+  });
+
+  const resolvedScope = request.allScopes
+    ? undefined
+    : (resume.resolvedScope || resume.latestCheckpoint?.resolvedScope || scopeSelection.resolvedScope);
+  if (!request.allScopes && !resolvedScope) {
+    return {
+      mode: "resume-only",
+      resolvedScope,
+      resume,
+      results: [],
+      searchSkippedReason: "No explicit or inferred scope available for focused recall; returning resume context only.",
+    };
+  }
+
+  const results = await deps.retriever.retrieve(buildRetrievalContext({
+    query: message,
+    limit: request.limit || 5,
+    category: request.category,
+    scope: resolvedScope,
+    sessionId: request.sessionId,
+    allScopes: request.allScopes,
+    source: "auto-recall",
+  }, {
+    operation: request.operation || "auto_recall",
+    env: request.env,
+    allowUnscoped: request.allScopes === true,
+  }));
+
+  return {
+    mode: "resume+search",
+    resolvedScope,
+    resume,
+    results,
+  };
+}