recallnest 1.4.1

package/src/context-composer-text.ts

@@ -0,0 +1,57 @@
+import { normalizeText } from "./term-registry.js";
+
+export function compactWhitespace(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
+export function cleanText(text: string, maxLen: number): string {
+  const compact = compactWhitespace(text);
+  if (compact.length <= maxLen) return compact;
+  return `${compact.slice(0, Math.max(0, maxLen - 3)).trimEnd()}...`;
+}
+
+export function stripConversationMarkers(text: string): string {
+  return text
+    .replace(/<image[^>]*>\s*/gi, "")
+    .replace(/\[(用户|助手|Pinned Asset|Memory Brief)\]\s*/g, "")
+    .replace(/\bSummary:\s*/gi, "")
+    .replace(/\bSnippet:\s*/gi, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+/**
+ * Tier 3.1: Extract the best available summary text from a memory entry.
+ * Priority: core_summary > l1_overview > l0_abstract > raw text.
+ * Falls back to raw text (stripped of conversation markers) when no summaries exist.
+ */
+export function bestSummaryText(text: string, metadata?: string): string {
+  if (metadata) {
+    try {
+      const meta = JSON.parse(metadata);
+      if (typeof meta.core_summary === "string" && meta.core_summary.length > 0) {
+        return meta.core_summary;
+      }
+      if (typeof meta.l1_overview === "string" && meta.l1_overview.length > 0) {
+        return meta.l1_overview;
+      }
+      if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 0) {
+        return meta.l0_abstract;
+      }
+    } catch { /* metadata parse failed, use raw text */ }
+  }
+  return stripConversationMarkers(text);
+}
+
+export function dedupeText(items: string[], limit: number): string[] {
+  const seen = new Set<string>();
+  const output: string[] = [];
+  for (const item of items) {
+    const key = normalizeText(item);
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    output.push(item);
+    if (output.length >= limit) break;
+  }
+  return output;
+}

package/src/context-composer.ts

@@ -0,0 +1,300 @@
+import { listPinAssets, type PinAsset } from "./memory-assets.js";
+import { buildStableContextSections } from "./context-composer-stable.js";
+import {
+  buildAssociativeNestEntityFallbackQuery,
+  buildScopedEntityFallbackQuery,
+  buildStableQuery,
+  buildStylePreferenceFallbackQuery,
+  buildTaskQuery,
+  formatLatestCheckpointHeadline,
+} from "./context-composer-queries.js";
+import { cleanText, dedupeText } from "./context-composer-text.js";
+import {
+  buildTaskResultSections,
+} from "./context-composer-task-results.js";
+import type { RetrievalContext, RetrievalResult } from "./retriever.js";
+import type { ResumeContextResponse, SessionCheckpointRecord } from "./session-schema.js";
+import { ResumeContextRequestSchema, ResumeContextResponseSchema } from "./session-schema.js";
+import { formatCheckpointRecallSummary } from "./session-output.js";
+import {
+  STRONG_WORKFLOW_CUE_TERMS,
+  looksLikeContinuityTask,
+  looksLikeRecallOnlyTask,
+  looksLikeStyleTask,
+} from "./term-registry.js";
+import { synthesizeSection } from "./result-synthesizer.js";
+import type { LLMClient } from "./llm-client.js";
+type ResumeCategory = "profile" | "preferences" | "entities" | "patterns" | "cases";
+
+interface ResumeRetriever {
+  retrieve(context: RetrievalContext): Promise<RetrievalResult[]>;
+}
+
+interface CheckpointLookup {
+  getLatest(query?: { sessionId?: string; scope?: string }): Promise<SessionCheckpointRecord | null>;
+}
+
+export interface ResumeContextDeps {
+  retriever: ResumeRetriever;
+  checkpointStore: CheckpointLookup;
+  listPins?: (limit?: number) => Array<PinAsset & { path: string }>;
+  /** Optional LLM client for Tier 3.5 result synthesis. */
+  llm?: LLMClient | null;
+}
+
+async function retrieveCandidates(
+  retriever: ResumeRetriever,
+  params: {
+    category?: ResumeCategory;
+    query: string;
+    limit: number;
+    scope?: string;
+  },
+): Promise<RetrievalResult[]> {
+  const { category, query, limit, scope } = params;
+
+  if (!scope) {
+    return retriever.retrieve({
+      query,
+      limit,
+      ...(category ? { category } : {}),
+      source: "auto-recall",
+    });
+  }
+
+  const [scoped, global] = await Promise.all([
+    retriever.retrieve({
+      query,
+      limit,
+      ...(category ? { category } : {}),
+      scopeFilter: [scope],
+      source: "auto-recall",
+    }),
+    retriever.retrieve({
+      query,
+      limit: Math.min(10, limit * 2),
+      ...(category ? { category } : {}),
+      source: "auto-recall",
+    }),
+  ]);
+
+  const seen = new Set<string>();
+  const merged: RetrievalResult[] = [];
+  for (const result of [...scoped, ...global]) {
+    if (seen.has(result.entry.id)) continue;
+    seen.add(result.entry.id);
+    merged.push(result);
+    if (merged.length >= Math.min(10, limit * 2)) break;
+  }
+  return merged;
+}
+
+function mergeRetrievalResults(resultSets: RetrievalResult[][], limit: number): RetrievalResult[] {
+  const seen = new Set<string>();
+  const merged: RetrievalResult[] = [];
+  for (const set of resultSets) {
+    for (const result of set) {
+      if (seen.has(result.entry.id)) continue;
+      seen.add(result.entry.id);
+      merged.push(result);
+      if (merged.length >= limit) return merged;
+    }
+  }
+  return merged;
+}
+
+async function resolveLatestCheckpoint(
+  checkpointStore: CheckpointLookup,
+  params: {
+    includeLatestCheckpoint: boolean;
+    sessionId?: string;
+    scope?: string;
+  },
+): Promise<SessionCheckpointRecord | null> {
+  if (!params.includeLatestCheckpoint) return null;
+
+  if (params.sessionId) {
+    const bySession = await checkpointStore.getLatest({ sessionId: params.sessionId });
+    if (bySession) return bySession;
+  }
+
+  if (params.scope) {
+    return checkpointStore.getLatest({ scope: params.scope });
+  }
+
+  return null;
+}
+
+function buildSummary(params: {
+  stableContext: string[];
+  relevantPatterns: string[];
+  recentCases: string[];
+  latestCheckpoint: SessionCheckpointRecord | null;
+}): string {
+  const parts: string[] = [];
+
+  if (params.latestCheckpoint) {
+    parts.push(
+      formatLatestCheckpointHeadline(
+        params.latestCheckpoint.sessionId,
+        params.latestCheckpoint.updatedAt,
+        formatCheckpointRecallSummary(params.latestCheckpoint),
+      ),
+    );
+  }
+
+  if (params.stableContext.length > 0) {
+    parts.push(`Stable context: ${params.stableContext.slice(0, 2).map((item) => cleanText(item, 120)).join(" | ")}`);
+  }
+
+  parts.push(
+    `Loaded ${params.stableContext.length} stable context item(s), ${params.relevantPatterns.length} pattern(s), and ${params.recentCases.length} case(s).`,
+  );
+
+  return cleanText(parts.join(" "), 800);
+}
+
+export async function composeResumeContext(
+  deps: ResumeContextDeps,
+  rawInput: unknown,
+): Promise<ResumeContextResponse> {
+  const input = ResumeContextRequestSchema.parse(rawInput);
+  const latestCheckpoint = await resolveLatestCheckpoint(deps.checkpointStore, {
+    includeLatestCheckpoint: input.includeLatestCheckpoint,
+    sessionId: input.sessionId,
+    scope: input.scope,
+  });
+
+  const resolvedScope = input.scope || latestCheckpoint?.resolvedScope;
+  const taskSeed = input.task || latestCheckpoint?.task || latestCheckpoint?.summary;
+  const stableLimit = input.limitPerSection;
+  const taskLimit = input.limitPerSection;
+  const styleFocusedTask = looksLikeStyleTask(taskSeed) || input.profile === "writing";
+  const recallOnlyTask = looksLikeRecallOnlyTask(taskSeed) && styleFocusedTask;
+
+  const preferenceQueries = dedupeText([
+    buildStableQuery("preferences", taskSeed),
+    ...(styleFocusedTask ? [buildStylePreferenceFallbackQuery(taskSeed)] : []),
+  ], 2);
+  const entityQueries = dedupeText([
+    buildStableQuery("entities", taskSeed),
+    ...(resolvedScope ? [buildScopedEntityFallbackQuery(resolvedScope, taskSeed)] : []),
+    ...(!resolvedScope ? [buildAssociativeNestEntityFallbackQuery(taskSeed)] : []),
+  ], 3);
+
+  const [profileResults, preferenceResultSets, entityResultSets, patternResults, caseResults] = await Promise.all([
+    retrieveCandidates(deps.retriever, {
+      category: "profile",
+      query: buildStableQuery("profile", taskSeed),
+      limit: Math.max(2, stableLimit),
+      scope: resolvedScope,
+    }),
+    Promise.all(preferenceQueries.map((query) =>
+      retrieveCandidates(deps.retriever, {
+        category: "preferences",
+        query,
+        limit: Math.max(2, stableLimit),
+          scope: resolvedScope,
+        })
+    )),
+    Promise.all(entityQueries.map((query) =>
+      retrieveCandidates(deps.retriever, {
+        category: "entities",
+        query,
+        limit: Math.max(2, stableLimit),
+        scope: resolvedScope,
+      })
+    )),
+    retrieveCandidates(deps.retriever, {
+      category: "patterns",
+      query: buildTaskQuery("patterns", taskSeed),
+      limit: taskLimit,
+      scope: resolvedScope,
+    }),
+    retrieveCandidates(deps.retriever, {
+      category: "cases",
+      query: buildTaskQuery("cases", taskSeed),
+      limit: taskLimit,
+      scope: resolvedScope,
+    }),
+  ]);
+  const preferenceResults = mergeRetrievalResults(preferenceResultSets, Math.max(4, stableLimit * 3));
+  const entityResults = mergeRetrievalResults(entityResultSets, Math.max(4, stableLimit * 3));
+
+  const continuityTask = looksLikeContinuityTask(taskSeed);
+  const pinAssets = (deps.listPins || listPinAssets)(Math.max(4, stableLimit * 2));
+  const {
+    preferenceContext,
+    stableContext,
+  } = buildStableContextSections({
+    profileResults,
+    preferenceResults,
+    entityResults,
+    pinAssets,
+    latestCheckpoint,
+    taskSeed,
+    scope: resolvedScope,
+    stableLimit,
+    styleFocusedTask,
+  });
+
+  const { relevantPatterns, recentCases } = await buildTaskResultSections({
+    retrieveCandidates: ({ category, query, limit, scope }) =>
+      retrieveCandidates(deps.retriever, {
+        ...(category ? { category } : {}),
+        query,
+        limit,
+        scope,
+      }),
+    patternResults,
+    caseResults,
+    continuityTask,
+    hasLatestCheckpoint: Boolean(latestCheckpoint),
+    taskLimit,
+    taskSeed,
+    scope: resolvedScope,
+    strongWorkflowCueTerms: STRONG_WORKFLOW_CUE_TERMS,
+  });
+
+  // Tier 3.5: Optionally synthesize sections into coherent narratives via LLM.
+  // Only activates when RECALLNEST_SYNTHESIZE=true and llm is provided.
+  const llm = deps.llm ?? null;
+  const queryHint = taskSeed || resolvedScope || "general";
+  const [synthStable, synthPatterns, synthCases] = await Promise.all([
+    synthesizeSection(stableContext, queryHint, llm),
+    synthesizeSection(relevantPatterns, queryHint, llm),
+    synthesizeSection(recentCases, queryHint, llm),
+  ]);
+
+  const response = {
+    summary: buildSummary({
+      stableContext: synthStable,
+      relevantPatterns: synthPatterns,
+      recentCases: synthCases,
+      latestCheckpoint,
+    }),
+    resolvedScope,
+    stableContext: synthStable,
+    relevantPatterns: synthPatterns,
+    recentCases: synthCases,
+    latestCheckpoint: latestCheckpoint
+      ? {
+        sessionId: latestCheckpoint.sessionId,
+        resolvedScope: latestCheckpoint.resolvedScope,
+        summary: formatCheckpointRecallSummary(latestCheckpoint),
+        updatedAt: latestCheckpoint.updatedAt,
+      }
+      : undefined,
+    responseMode: recallOnlyTask ? "recall-only" as const : "default" as const,
+    responseGuidance: recallOnlyTask
+      ? (
+          stableContext.length <= 1
+            ? "Recall-only mode: answer from the recalled stable context item only. Restate it briefly and do not expand into extra rules, examples, or local writing docs unless the user explicitly asks."
+            : "Recall-only mode: answer only from the recalled stable context items. Keep the reply brief and do not expand into extra rules, examples, or local writing docs unless the user explicitly asks."
+        )
+      : undefined,
+    generatedAt: new Date().toISOString(),
+  };
+
+  return ResumeContextResponseSchema.parse(response);
+}

package/src/context-renderer.ts

@@ -0,0 +1,179 @@
+/**
+ * Context Renderer — reorder recall results by contextual relevance.
+ *
+ * Borrowed from UltraMemory's context-renderer.ts, adapted for RecallNest:
+ * - "verbatim" mode: pass-through (default, backward-compatible)
+ * - "highlight" mode: reorder by 60% vector score + 40% term overlap
+ * - "synthesize" mode: reserved for future LLM-based rendering (falls back to highlight)
+ *
+ * Pure functions, zero dependencies on store/embedder/LLM.
+ */
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type RenderMode = "verbatim" | "highlight" | "synthesize";
+
+export interface RenderableMemory {
+  id: string;
+  text: string;
+  score: number;
+  category: string;
+}
+
+export interface RenderedMemory {
+  id: string;
+  text: string;
+  category: string;
+  /** Contextual relevance score (0–1), combining vector score + term overlap. */
+  relevance: number;
+}
+
+export interface RenderResult {
+  mode: RenderMode;
+  memories: RenderedMemory[];
+}
+
+// ---------------------------------------------------------------------------
+// Stop words (EN + ZH function words)
+// ---------------------------------------------------------------------------
+
+const STOP_WORDS = new Set([
+  // English
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "shall", "can", "need", "to", "of", "in",
+  "for", "on", "with", "at", "by", "from", "as", "into", "through",
+  "during", "before", "after", "about", "between", "under", "above",
+  "this", "that", "these", "those", "it", "its", "i", "me", "my",
+  "we", "our", "you", "your", "he", "she", "they", "them", "and",
+  "or", "but", "if", "then", "so", "just", "also", "not", "no",
+  // Chinese function words
+  "的", "了", "在", "是", "我", "有", "和", "就", "不", "人",
+  "都", "一", "一个", "上", "也", "很", "到", "说", "要", "去",
+  "你", "会", "着", "没有", "看", "好", "自己", "这",
+]);
+
+// ---------------------------------------------------------------------------
+// Term extraction
+// ---------------------------------------------------------------------------
+
+/** Extract significant terms from text (lowercase, deduped, stop words removed). */
+export function extractTerms(text: string): Set<string> {
+  const words = text
+    .toLowerCase()
+    .replace(/[^\w\s\u4e00-\u9fff]/g, " ")
+    .split(/\s+/);
+  return new Set(words.filter(w => w.length > 1 && !STOP_WORDS.has(w)));
+}
+
+/** Compute Jaccard-like overlap between two term sets. */
+export function computeTermOverlap(a: Set<string>, b: Set<string>): number {
+  if (a.size === 0 || b.size === 0) return 0;
+  let overlap = 0;
+  for (const term of a) {
+    if (b.has(term)) overlap++;
+  }
+  return overlap / Math.max(a.size, b.size);
+}
+
+// ---------------------------------------------------------------------------
+// Temporal helpers
+// ---------------------------------------------------------------------------
+
+/** Detect whether a query asks for a specific date/time. */
+export function isTemporalQuery(query: string): boolean {
+  return /^when\b|what (date|time|day|year|month)\b|how long ago\b/i.test(query);
+}
+
+/** Extract a bracketed date anchor from memory text, e.g. "[1:56 pm on 7 May, 2023]". */
+function extractAnchorDate(text: string): { dateStr: string; year: number; month: number } | null {
+  const m = text.match(/\[(?:\d{1,2}:\d{2}\s*(?:am|pm)\s+on\s+)?(\d{1,2})\s+(\w+),?\s+(\d{4})\]/i);
+  if (!m) return null;
+  const monthMap: Record<string, number> = {
+    january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
+    july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
+  };
+  const month = monthMap[m[2].toLowerCase()];
+  if (!month) return null;
+  return { dateStr: m[0], year: parseInt(m[3], 10), month };
+}
+
+/**
+ * Annotate relative time expressions with absolute dates.
+ * E.g. "last year" in text dated [7 May, 2023] → "last year (i.e., 2022)"
+ */
+export function resolveRelativeDates(text: string): string {
+  const anchor = extractAnchorDate(text);
+  if (!anchor) return text;
+
+  const { year, month } = anchor;
+  const MONTHS = ["", "January", "February", "March", "April", "May", "June",
+    "July", "August", "September", "October", "November", "December"];
+
+  return text
+    .replace(/\blast year\b/gi, `last year (i.e., ${year - 1})`)
+    .replace(/\bnext year\b/gi, `next year (i.e., ${year + 1})`)
+    .replace(/\blast month\b/gi, `last month (i.e., ${MONTHS[month === 1 ? 12 : month - 1]} ${month === 1 ? year - 1 : year})`)
+    .replace(/\bnext month\b/gi, `next month (i.e., ${MONTHS[month === 12 ? 1 : month + 1]} ${month === 12 ? year + 1 : year})`)
+    .replace(/\b(\d+)\s+years?\s+ago\b/gi, (match, n) => `${match} (i.e., ${year - parseInt(n, 10)})`)
+    .replace(/\ba few years ago\b/gi, `a few years ago (i.e., around ${year - 3})`)
+    .replace(/\blast week\b/gi, `last week (i.e., the week before ${anchor.dateStr})`)
+    .replace(/\blast saturday\b/gi, `last Saturday (i.e., the Saturday before ${anchor.dateStr})`);
+}
+
+// ---------------------------------------------------------------------------
+// Renderer
+// ---------------------------------------------------------------------------
+
+/**
+ * Render recalled memories adapted to the current query context.
+ *
+ * - verbatim: return as-is in original order (default)
+ * - highlight: reorder by contextual relevance (60% vector score + 40% term overlap)
+ * - synthesize: highlight + temporal date annotation for "when" queries
+ */
+export function renderMemories(
+  memories: RenderableMemory[],
+  query: string,
+  mode: RenderMode = "verbatim",
+  taskContext?: string,
+): RenderResult {
+  if (mode === "verbatim" || memories.length === 0) {
+    return {
+      mode: "verbatim",
+      memories: memories.map(m => ({
+        id: m.id,
+        text: m.text,
+        category: m.category,
+        relevance: m.score,
+      })),
+    };
+  }
+
+  const temporal = isTemporalQuery(query);
+  const queryTerms = extractTerms(query + (taskContext ? " " + taskContext : ""));
+
+  const scored = memories.map(m => {
+    const memTerms = extractTerms(m.text);
+    const overlap = computeTermOverlap(queryTerms, memTerms);
+    const relevance = 0.6 * m.score + 0.4 * overlap;
+
+    // In synthesize mode for temporal queries, annotate relative dates
+    const renderedText = (mode === "synthesize" && temporal)
+      ? resolveRelativeDates(m.text)
+      : m.text;
+
+    return {
+      id: m.id,
+      text: renderedText,
+      category: m.category,
+      relevance: Math.round(relevance * 1000) / 1000,
+    };
+  });
+
+  scored.sort((a, b) => b.relevance - a.relevance);
+
+  return { mode, memories: scored };
+}