recallnest 1.4.1 → 2.2.0

package/src/context-composer.ts

@@ -13,7 +13,7 @@ import {
   buildTaskResultSections,
 } from "./context-composer-task-results.js";
 import type { RetrievalContext, RetrievalResult } from "./retriever.js";
-import type { ResumeContextResponse, SessionCheckpointRecord } from "./session-schema.js";
+import type { EssentialContext, ResumeContextResponse, SessionCheckpointRecord } from "./session-schema.js";
 import { ResumeContextRequestSchema, ResumeContextResponseSchema } from "./session-schema.js";
 import { formatCheckpointRecallSummary } from "./session-output.js";
 import {
@@ -24,6 +24,10 @@ import {
 } from "./term-registry.js";
 import { synthesizeSection } from "./result-synthesizer.js";
 import type { LLMClient } from "./llm-client.js";
+import { collapseResults, estimateTokens, type CollapseInput } from "./context-collapse-renderer.js";
+import { filterByRelevance } from "./post-retrieval-filter.js";
+import { reconstruct as runReconstruction } from "./context-reconstructor.js";
+import { parseNarrative, isNarrativeModeEnabled } from "./narrative-schema.js";
 type ResumeCategory = "profile" | "preferences" | "entities" | "patterns" | "cases";
 
 interface ResumeRetriever {
@@ -154,6 +158,142 @@ function buildSummary(params: {
   return cleanText(parts.join(" "), 800);
 }
 
+// ---------------------------------------------------------------------------
+// CC-8: Post-Compact Reconstruction — assemble essential context
+// ---------------------------------------------------------------------------
+
+const ESSENTIAL_CONTEXT_TOKEN_BUDGET = 2000;
+
+function buildEssentialContext(params: {
+  pinAssets: Array<PinAsset & { path: string }>;
+  patternResults: RetrievalResult[];
+  latestCheckpoint: SessionCheckpointRecord | null;
+}): EssentialContext | undefined {
+  let tokensUsed = 0;
+
+  // 1. Pinned memories: take up to 3 most recent pins' summaries
+  const pinnedMemories: string[] = [];
+  for (const pin of params.pinAssets.slice(0, 3)) {
+    const text = pin.summary || pin.title;
+    const cost = estimateTokens(text);
+    if (tokensUsed + cost > ESSENTIAL_CONTEXT_TOKEN_BUDGET) break;
+    pinnedMemories.push(text);
+    tokensUsed += cost;
+  }
+
+  // 2. Active patterns: take top 1-2 by score
+  const activePatterns: string[] = [];
+  const sortedPatterns = [...params.patternResults].sort((a, b) => b.score - a.score);
+  for (const pattern of sortedPatterns.slice(0, 2)) {
+    const text = cleanText(pattern.entry.text, 200);
+    const cost = estimateTokens(text);
+    if (tokensUsed + cost > ESSENTIAL_CONTEXT_TOKEN_BUDGET) break;
+    activePatterns.push(text);
+    tokensUsed += cost;
+  }
+
+  // 3. Open loops from latest checkpoint
+  const openLoops: string[] = [];
+  if (params.latestCheckpoint?.openLoops) {
+    for (const loop of params.latestCheckpoint.openLoops.slice(0, 3)) {
+      const cost = estimateTokens(loop);
+      if (tokensUsed + cost > ESSENTIAL_CONTEXT_TOKEN_BUDGET) break;
+      openLoops.push(loop);
+      tokensUsed += cost;
+    }
+  }
+
+  // Return undefined if nothing was collected
+  if (pinnedMemories.length === 0 && activePatterns.length === 0 && openLoops.length === 0) {
+    return undefined;
+  }
+
+  return {
+    ...(pinnedMemories.length > 0 ? { pinnedMemories } : {}),
+    ...(activePatterns.length > 0 ? { activePatterns } : {}),
+    ...(openLoops.length > 0 ? { openLoops } : {}),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// MP-3: Ultra-Light Wake-up — <300 token resume for low-budget terminals
+// ---------------------------------------------------------------------------
+
+const LIGHT_TOKEN_BUDGET = 300;
+const LIGHT_MEMORY_CHAR_LIMIT = 200; // ~50 tokens
+
+export interface LightResumeResult {
+  text: string;
+  resolvedScope?: string;
+  generatedAt: string;
+}
+
+export async function composeLightResumeContext(
+  deps: ResumeContextDeps,
+  rawInput: unknown,
+): Promise<LightResumeResult> {
+  const input = ResumeContextRequestSchema.parse(rawInput);
+  const latestCheckpoint = await resolveLatestCheckpoint(deps.checkpointStore, {
+    includeLatestCheckpoint: true,
+    sessionId: input.sessionId,
+    scope: input.scope,
+  });
+
+  const resolvedScope = input.scope || latestCheckpoint?.resolvedScope;
+
+  // Retrieve top 3 stable memories (profile + preferences + entities combined)
+  const stableResults = await retrieveCandidates(deps.retriever, {
+    query: input.task || "identity key facts and preferences",
+    limit: 3,
+    scope: resolvedScope,
+  });
+
+  // Pin assets provide high-signal stable context
+  const pinAssets = (deps.listPins || listPinAssets)(3);
+
+  const parts: string[] = [];
+  let tokensUsed = 0;
+
+  // 1. Checkpoint summary (1 sentence, ~30 tokens)
+  if (latestCheckpoint) {
+    const cpLine = `Last session (${latestCheckpoint.sessionId}): ${cleanText(latestCheckpoint.summary, 120)}`;
+    parts.push(cpLine);
+    tokensUsed += estimateTokens(cpLine);
+  }
+
+  // 2. Top pinned memories (most reliable stable context)
+  if (pinAssets.length > 0) {
+    for (const pin of pinAssets) {
+      if (tokensUsed >= LIGHT_TOKEN_BUDGET - 40) break;
+      const line = `- ${cleanText(pin.summary || pin.title, LIGHT_MEMORY_CHAR_LIMIT)}`;
+      parts.push(line);
+      tokensUsed += estimateTokens(line);
+    }
+  }
+
+  // 3. Top stable memories (fill remaining budget)
+  if (stableResults.length > 0 && tokensUsed < LIGHT_TOKEN_BUDGET - 40) {
+    const pinTexts = new Set(pinAssets.map(p => (p.summary || p.title).toLowerCase()));
+    for (const r of stableResults) {
+      if (tokensUsed >= LIGHT_TOKEN_BUDGET - 40) break;
+      const textLower = r.entry.text.toLowerCase().slice(0, 100);
+      if (pinTexts.has(textLower)) continue;
+      const line = `- ${cleanText(r.entry.text, LIGHT_MEMORY_CHAR_LIMIT)}`;
+      parts.push(line);
+      tokensUsed += estimateTokens(line);
+    }
+  }
+
+  // 4. Upgrade hint
+  parts.push("\nFor complete context, call resume_context(mode='full').");
+
+  return {
+    text: parts.join("\n"),
+    resolvedScope,
+    generatedAt: new Date().toISOString(),
+  };
+}
+
 export async function composeResumeContext(
   deps: ResumeContextDeps,
   rawInput: unknown,
@@ -221,6 +361,17 @@ export async function composeResumeContext(
   const preferenceResults = mergeRetrievalResults(preferenceResultSets, Math.max(4, stableLimit * 3));
   const entityResults = mergeRetrievalResults(entityResultSets, Math.max(4, stableLimit * 3));
 
+  // E-2: Post-retrieval LLM relevance filter for pattern/case results.
+  // Only fires when LLM is available; stable context (profile/preferences/entities) skipped
+  // because those are identity-level and already curated by the selection pipeline.
+  const llm = deps.llm ?? null;
+  const [filteredPatterns, filteredCases] = llm && taskSeed
+    ? await Promise.all([
+        filterByRelevance(patternResults, taskSeed, llm),
+        filterByRelevance(caseResults, taskSeed, llm),
+      ])
+    : [patternResults, caseResults];
+
   const continuityTask = looksLikeContinuityTask(taskSeed);
   const pinAssets = (deps.listPins || listPinAssets)(Math.max(4, stableLimit * 2));
   const {
@@ -246,8 +397,8 @@ export async function composeResumeContext(
         limit,
         scope,
       }),
-    patternResults,
-    caseResults,
+    patternResults: filteredPatterns,
+    caseResults: filteredCases,
     continuityTask,
     hasLatestCheckpoint: Boolean(latestCheckpoint),
     taskLimit,
@@ -258,7 +409,6 @@ export async function composeResumeContext(
 
   // Tier 3.5: Optionally synthesize sections into coherent narratives via LLM.
   // Only activates when RECALLNEST_SYNTHESIZE=true and llm is provided.
-  const llm = deps.llm ?? null;
   const queryHint = taskSeed || resolvedScope || "general";
   const [synthStable, synthPatterns, synthCases] = await Promise.all([
     synthesizeSection(stableContext, queryHint, llm),
@@ -266,6 +416,83 @@ export async function composeResumeContext(
     synthesizeSection(recentCases, queryHint, llm),
   ]);
 
+  // HP-narrative: Group recalled items by life period for narrative context
+  let narrativeGroups: Array<{ period: string; items: string[] }> | undefined;
+  if (isNarrativeModeEnabled()) {
+    const allNarrativeResults = [...profileResults, ...preferenceResults, ...entityResults, ...filteredPatterns, ...filteredCases];
+    const periodMap = new Map<string, string[]>();
+    for (const r of allNarrativeResults) {
+      const narrative = parseNarrative(r.entry.metadata);
+      if (!narrative) continue;
+      const items = periodMap.get(narrative.lifePeriodLabel) ?? [];
+      items.push(cleanText(r.entry.text, 120));
+      periodMap.set(narrative.lifePeriodLabel, items);
+    }
+    if (periodMap.size > 0) {
+      narrativeGroups = [...periodMap.entries()]
+        .map(([period, items]) => ({ period, items: items.slice(0, 5) }));
+    }
+  }
+
+  // CC-7: Collapse rendering — build mixed-granularity view of all recalled items.
+  // Gathers all retrieval results, deduplicates, and renders at L0/L1/L2 based on score.
+  const allResults: RetrievalResult[] = [];
+  const seenIds = new Set<string>();
+  for (const r of [...profileResults, ...preferenceResults, ...entityResults, ...filteredPatterns, ...filteredCases]) {
+    if (!seenIds.has(r.entry.id)) {
+      seenIds.add(r.entry.id);
+      allResults.push(r);
+    }
+  }
+  const collapseInput: CollapseInput[] = allResults.map(r => ({
+    entryId: r.entry.id,
+    text: r.entry.text,
+    metadata: r.entry.metadata,
+    score: r.score,
+    timestamp: r.entry.timestamp,
+  }));
+  const collapsedItems = collapseInput.length > 0
+    ? collapseResults(collapseInput)
+    : undefined;
+
+  // CC-8: Build essential context from pinned memories, top patterns, and open loops.
+  const essentialContext = buildEssentialContext({
+    pinAssets,
+    patternResults: filteredPatterns,
+    latestCheckpoint,
+  });
+
+  // Phase 4: Constructive retrieval reconstruction for resume context
+  // Pass checkpoint openLoops/nextActions/scope into reconstruction prompt
+  let reconstructedContext: string | undefined;
+  let reconstructionConfidence: number | undefined;
+  let reconstructionContradictions: Array<{ memoryIds: [string, string]; description: string }> | undefined;
+  const constructiveFlag = process.env.RECALLNEST_CONSTRUCTIVE_RETRIEVAL === "true";
+  if (constructiveFlag && deps.llm?.isAvailable?.()) {
+    const allReconResults = [...profileResults, ...preferenceResults, ...entityResults, ...filteredPatterns, ...filteredCases];
+    if (allReconResults.length >= 3) {
+      try {
+        const taskQuery = latestCheckpoint?.summary ?? taskSeed ?? "general context";
+        const checkpointContext = latestCheckpoint ? {
+          openLoops: latestCheckpoint.openLoops,
+          nextActions: latestCheckpoint.nextActions,
+          scope: resolvedScope,
+        } : resolvedScope ? { scope: resolvedScope } : undefined;
+        const recon = await runReconstruction(
+          { query: taskQuery, results: allReconResults, mode: "resume", maxTokens: 600, checkpointContext },
+          deps.llm,
+        );
+        if (recon.reconstructed) {
+          reconstructedContext = recon.reconstructed;
+          reconstructionConfidence = recon.confidence;
+        }
+        if (recon.contradictions.length > 0) {
+          reconstructionContradictions = recon.contradictions;
+        }
+      } catch { /* silent fallback */ }
+    }
+  }
+
   const response = {
     summary: buildSummary({
       stableContext: synthStable,
@@ -277,6 +504,8 @@ export async function composeResumeContext(
     stableContext: synthStable,
     relevantPatterns: synthPatterns,
     recentCases: synthCases,
+    collapsedItems: collapsedItems && collapsedItems.length > 0 ? collapsedItems : undefined,
+    essentialContext,
     latestCheckpoint: latestCheckpoint
       ? {
         sessionId: latestCheckpoint.sessionId,
@@ -285,6 +514,8 @@ export async function composeResumeContext(
         updatedAt: latestCheckpoint.updatedAt,
       }
       : undefined,
+    injectionHint: "user_attachment" as const,
+    ephemeral: true,
     responseMode: recallOnlyTask ? "recall-only" as const : "default" as const,
     responseGuidance: recallOnlyTask
       ? (
@@ -293,6 +524,10 @@ export async function composeResumeContext(
             : "Recall-only mode: answer only from the recalled stable context items. Keep the reply brief and do not expand into extra rules, examples, or local writing docs unless the user explicitly asks."
         )
       : undefined,
+    reconstructedContext,
+    reconstructionConfidence,
+    reconstructionContradictions,
+    narrativeGroups,
     generatedAt: new Date().toISOString(),
   };