recallnest 1.4.1

package/src/query-entity-detector.ts

@@ -0,0 +1,207 @@
+/**
+ * Query Entity Detector
+ * Extracts entities from search queries for KG graph traversal.
+ * Combines heuristic extraction with KG entity matching.
+ */
+
+import { normalizeEntity } from "./kg-extractor.js";
+import type { KGStore } from "./kg-store.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface DetectedEntities {
+  /** Entities found in the query */
+  entities: string[];
+  /** Whether the query signals multi-hop intent */
+  isMultiHop: boolean;
+  /** Detected multi-hop predicate (e.g. "friend_of", "uses") */
+  hopPredicate?: string;
+}
+
+// ============================================================================
+// Multi-hop signal patterns
+// ============================================================================
+
+interface HopPattern {
+  pattern: RegExp;
+  predicate: string;
+}
+
+const MULTI_HOP_PATTERNS: HopPattern[] = [
+  // English patterns
+  { pattern: /(\w+)'s\s+friends?/i, predicate: "friend_of" },
+  { pattern: /friends?\s+of\s+(\w+)/i, predicate: "friend_of" },
+  { pattern: /who\s+(?:does|did)\s+(\w+)\s+(?:work|collaborate)\s+with/i, predicate: "works_with" },
+  { pattern: /what\s+does\s+(\w+)\s+use/i, predicate: "uses" },
+  { pattern: /what\s+(?:tools?|tech|technologies?)\s+does\s+(\w+)\s+use/i, predicate: "uses" },
+  { pattern: /who\s+(?:uses?|works?\s+with)\s+(\w+)/i, predicate: "uses" },
+  { pattern: /who\s+(?:created|built|wrote|made)\s+(\w+)/i, predicate: "created_by" },
+  { pattern: /who\s+(?:knows?|is\s+familiar\s+with)\s+(\w+)/i, predicate: "knows" },
+  { pattern: /what\s+depends\s+on\s+(\w+)/i, predicate: "depends_on" },
+  { pattern: /(\w+)\s+depend(?:s|encies)/i, predicate: "depends_on" },
+  { pattern: /who\s+manages?\s+(\w+)/i, predicate: "manages" },
+  { pattern: /(\w+)'s\s+team/i, predicate: "member_of" },
+  // Chinese patterns
+  { pattern: /(.+?)的朋友/u, predicate: "friend_of" },
+  { pattern: /谁和(.+?)一起/u, predicate: "works_with" },
+  { pattern: /(.+?)用(?:了|什么)/u, predicate: "uses" },
+  { pattern: /谁(?:创建|开发|写)了(.+)/u, predicate: "created_by" },
+  { pattern: /(.+?)(?:依赖|需要)什么/u, predicate: "depends_on" },
+  { pattern: /谁(?:认识|了解)(.+)/u, predicate: "knows" },
+  { pattern: /(.+?)的(?:团队|成员)/u, predicate: "member_of" },
+  { pattern: /谁管理(.+)/u, predicate: "manages" },
+];
+
+// ============================================================================
+// Heuristic entity extraction
+// ============================================================================
+
+/**
+ * Extract candidate entities from query text using heuristics:
+ * - Quoted strings: "RecallNest", 'LanceDB'
+ * - Capitalized words/phrases: Alice, OpenAI, Claude Code
+ * - CamelCase compound words: RecallNest, LanceDB
+ */
+function extractCandidates(query: string): string[] {
+  const candidates: string[] = [];
+
+  // 1. Quoted strings (single, double, backtick)
+  const quoted = query.match(/["'`]([^"'`]+)["'`]/g);
+  if (quoted) {
+    for (const q of quoted) {
+      const inner = q.slice(1, -1).trim();
+      if (inner.length >= 2) candidates.push(inner);
+    }
+  }
+
+  // 2. Capitalized words/phrases (English)
+  // Match sequences of capitalized words: "Claude Code", "Open AI", "LanceDB"
+  const capPattern = /\b([A-Z][a-zA-Z]*(?:[\s-][A-Z][a-zA-Z]*)*)\b/g;
+  let match;
+  while ((match = capPattern.exec(query)) !== null) {
+    const word = match[1].trim();
+    // Skip common English words that happen to be at start of sentence
+    if (STOP_WORDS.has(word.toLowerCase())) continue;
+    if (word.length >= 2) candidates.push(word);
+  }
+
+  // 3. CamelCase / PascalCase compound words: "RecallNest", "LanceDB"
+  const camelPattern = /\b([A-Z][a-z]+(?:[A-Z][a-z]*)+)\b/g;
+  while ((match = camelPattern.exec(query)) !== null) {
+    candidates.push(match[1]);
+  }
+
+  return candidates;
+}
+
+const STOP_WORDS = new Set([
+  "the", "a", "an", "is", "are", "was", "were", "be", "been",
+  "do", "does", "did", "have", "has", "had", "will", "would",
+  "can", "could", "should", "may", "might", "must", "shall",
+  "what", "who", "where", "when", "why", "how", "which",
+  "this", "that", "these", "those", "it", "its",
+  "not", "no", "nor", "but", "and", "or", "if", "then",
+  "use", "uses", "used", "using",
+  "important", "search", "find", "get", "set", "let", "new",
+  "all", "any", "some", "each", "every", "many", "much",
+  "my", "your", "his", "her", "our", "their",
+  "i", "you", "he", "she", "we", "they", "me", "him", "us", "them",
+]);
+
+// ============================================================================
+// Main detection
+// ============================================================================
+
+/**
+ * Detect entities in a query for KG graph traversal.
+ * Optionally validates against KG-known entities.
+ */
+export async function detectEntities(
+  query: string,
+  kgStore?: KGStore,
+  scope?: string,
+): Promise<DetectedEntities> {
+  // Check for multi-hop signals
+  let isMultiHop = false;
+  let hopPredicate: string | undefined;
+
+  for (const hp of MULTI_HOP_PATTERNS) {
+    if (hp.pattern.test(query)) {
+      isMultiHop = true;
+      hopPredicate = hp.predicate;
+      break;
+    }
+  }
+
+  // Extract candidates
+  const raw = extractCandidates(query);
+
+  // Normalize
+  const normalized = raw
+    .map(normalizeEntity)
+    .filter(e => e.length >= 2);
+
+  // Deduplicate (case-insensitive)
+  const seen = new Set<string>();
+  const unique = normalized.filter(e => {
+    const key = e.toLowerCase();
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  // If KG store available, validate entities exist in KG
+  if (kgStore && unique.length > 0) {
+    const validated: string[] = [];
+    const kgEntities = await kgStore.getAllEntities(scope);
+    const kgEntitySet = new Set(kgEntities.map(e => e.toLowerCase()));
+
+    for (const entity of unique) {
+      if (kgEntitySet.has(entity.toLowerCase())) {
+        // Find the KG-canonical casing
+        const canonical = kgEntities.find(e => e.toLowerCase() === entity.toLowerCase());
+        validated.push(canonical ?? entity);
+      }
+    }
+
+    // If validation eliminated everything, fall back to heuristic candidates
+    if (validated.length === 0 && unique.length > 0) {
+      return { entities: unique, isMultiHop, hopPredicate };
+    }
+
+    return { entities: validated, isMultiHop, hopPredicate };
+  }
+
+  return { entities: unique, isMultiHop, hopPredicate };
+}
+
+/**
+ * Synchronous version without KG validation — for testing or lightweight use.
+ */
+export function detectEntitiesSync(query: string): DetectedEntities {
+  let isMultiHop = false;
+  let hopPredicate: string | undefined;
+
+  for (const hp of MULTI_HOP_PATTERNS) {
+    if (hp.pattern.test(query)) {
+      isMultiHop = true;
+      hopPredicate = hp.predicate;
+      break;
+    }
+  }
+
+  const raw = extractCandidates(query);
+  const normalized = raw.map(normalizeEntity).filter(e => e.length >= 2);
+
+  const seen = new Set<string>();
+  const unique = normalized.filter(e => {
+    const key = e.toLowerCase();
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  return { entities: unique, isMultiHop, hopPredicate };
+}

package/src/query-expander.ts

@@ -0,0 +1,98 @@
+/**
+ * Lightweight Chinese query expansion via static synonym dictionary.
+ * Expands colloquial/fuzzy terms into technical equivalents for BM25 boost.
+ * No API calls — pure local dictionary lookup.
+ *
+ * v1.1: Word-boundary matching for English triggers (prevents "download" matching "down").
+ *       MAX_EXPANSION_TERMS cap to prevent query bloat hurting BM25 precision.
+ *       Structured synonym entries separating CN (substring) from EN (word-boundary).
+ * v1.2: Entity resolution pre-pass normalizes aliases before synonym expansion.
+ */
+
+import { resolveQueryEntities } from "./entity-resolver.js";
+
+interface SynonymEntry {
+  /** Chinese triggers: matched by substring (CJK has no word boundaries) */
+  cn: string[];
+  /** English triggers: matched by word boundary (\b) to prevent false positives */
+  en: string[];
+  /** Terms to append when triggered */
+  expansions: string[];
+}
+
+const SYNONYM_MAP: SynonymEntry[] = [
+  // --- Status / Failure ---
+  { cn: ["挂了", "挂掉", "宕机"], en: ["shutdown", "crashed"], expansions: ["崩溃", "crash", "error", "报错", "挂了", "宕机", "失败"] },
+  { cn: ["卡住", "卡死", "没反应"], en: ["hang", "stuck"], expansions: ["hang", "timeout", "超时", "卡住", "无响应", "stuck"] },
+  { cn: ["炸了", "爆了"], en: ["OOM"], expansions: ["崩溃", "crash", "OOM", "内存溢出", "error"] },
+
+  // --- AI / Consciousness ---
+  { cn: ["感受", "感觉", "情感"], en: ["feeling", "emotion"], expansions: ["意识", "consciousness", "experiencing", "感受", "情感", "qualia"] },
+  { cn: ["有没有意识", "是否有意识"], en: ["consciousness", "sentience"], expansions: ["consciousness", "意识", "sentience", "感知", "自我意识"] },
+  { cn: ["自由意志"], en: ["free will"], expansions: ["free will", "自由意志", "决定论", "determinism"] },
+
+  // --- Config / Deploy ---
+  { cn: ["配置", "设置"], en: ["config", "configuration", "settings"], expansions: ["配置", "config", "configuration", "settings", "设置"] },
+  { cn: ["部署", "上线"], en: ["deploy"], expansions: ["deploy", "部署", "上线", "发布", "release"] },
+  { cn: ["容器"], en: ["docker", "container"], expansions: ["Docker", "容器", "container", "docker-compose"] },
+
+  // --- Code / Debug ---
+  { cn: ["报错", "出错", "错误"], en: ["error", "exception"], expansions: ["error", "报错", "exception", "错误", "失败", "bug"] },
+  { cn: ["修复", "修了", "修好"], en: ["fix", "patch"], expansions: ["fix", "修复", "patch", "修了", "解决"] },
+  { cn: ["踩坑", "坑"], en: ["troubleshoot"], expansions: ["踩坑", "bug", "问题", "教训", "排查", "troubleshoot"] },
+  { cn: ["日志"], en: ["log", "logging"], expansions: ["日志", "log", "logging", "输出", "stdout", "stderr"] },
+  { cn: ["权限"], en: ["permission", "access"], expansions: ["权限", "permission", "access", "授权", "认证"] },
+
+  // --- Writing / Content ---
+  { cn: ["配图", "插图"], en: [], expansions: ["配图", "封面", "style-catalog", "风格", "图片", "image"] },
+  { cn: ["排版", "版式"], en: ["layout"], expansions: ["排版", "layout", "主题", "theme", "样式"] },
+  { cn: ["风格"], en: ["style"], expansions: ["风格", "style", "轮换", "catalog"] },
+  { cn: ["写作", "写文章"], en: ["writing"], expansions: ["写作", "writing", "文章", "公众号", "content-alchemy"] },
+
+  // --- Infrastructure ---
+  { cn: ["机器人"], en: ["bot", "agent"], expansions: ["bot", "机器人", "OpenClaw", "agent", "gateway"] },
+  { cn: ["推送"], en: ["push"], expansions: ["push", "推送", "git push", "commit"] },
+  { cn: ["记忆"], en: ["memory"], expansions: ["记忆", "memory", "记忆系统", "LanceDB", "索引"] },
+  { cn: ["搜索", "查找", "找"], en: ["search", "retrieval"], expansions: ["搜索", "search", "retrieval", "检索", "查找"] },
+];
+
+/** Maximum expansion terms to prevent query bloat hurting BM25 precision */
+const MAX_EXPANSION_TERMS = 5;
+
+/**
+ * Expand a query by appending synonym terms from the dictionary.
+ * Returns the original query with additional terms appended.
+ * Idempotent — already-precise queries pass through unchanged.
+ */
+export function expandQuery(query: string): string {
+  if (!query || query.trim().length < 2) return query;
+
+  // Entity resolution pre-pass: normalize aliases before synonym expansion
+  const resolved = resolveQueryEntities(query);
+  const lower = resolved.toLowerCase();
+  const additions = new Set<string>();
+
+  for (const entry of SYNONYM_MAP) {
+    // Chinese triggers: substring match (CJK has no word boundaries)
+    const cnMatch = entry.cn.some(t => lower.includes(t.toLowerCase()));
+    // English triggers: word-boundary match to prevent false positives
+    const enMatch = entry.en.length > 0 && entry.en.some(t => {
+      const regex = new RegExp(`\\b${t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'i');
+      return regex.test(lower);
+    });
+
+    if (cnMatch || enMatch) {
+      for (const exp of entry.expansions) {
+        if (!lower.includes(exp.toLowerCase())) {
+          additions.add(exp);
+        }
+      }
+    }
+  }
+
+  if (additions.size === 0) return resolved;
+
+  // Cap expansion terms to prevent query bloat
+  const limited = [...additions].slice(0, MAX_EXPANSION_TERMS);
+  return `${resolved} ${limited.join(" ")}`;
+}

package/src/result-synthesizer.ts

@@ -0,0 +1,114 @@
+/**
+ * Result Synthesizer — Tier 3.5
+ *
+ * Converts multiple retrieval fragments into a single coherent narrative
+ * using LLM. Falls back to original fragments when LLM is unavailable.
+ *
+ * Integration point: context-composer calls synthesize() on section outputs
+ * (stableContext, relevantPatterns, recentCases) when enabled.
+ *
+ * Enable: RECALLNEST_SYNTHESIZE=true environment variable.
+ */
+
+import type { LLMClient } from "./llm-client.js";
+import { logWarn } from "./stderr-log.js";
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface SynthesizerConfig {
+  /** Minimum fragments required to trigger synthesis (default: 3).
+   *  Below this count, fragments are returned as-is. */
+  minFragments: number;
+  /** Maximum output length in chars (default: 500). */
+  maxOutputChars: number;
+}
+
+export const DEFAULT_SYNTHESIZER_CONFIG: SynthesizerConfig = {
+  minFragments: 3,
+  maxOutputChars: 500,
+};
+
+// ============================================================================
+// Core
+// ============================================================================
+
+/**
+ * Check if synthesis is enabled via environment variable.
+ */
+export function isSynthesisEnabled(): boolean {
+  return process.env.RECALLNEST_SYNTHESIZE === "true";
+}
+
+/**
+ * Synthesize an array of text fragments into a coherent narrative.
+ *
+ * @param fragments  Array of text snippets (e.g. from retrieval results)
+ * @param query      The original query/task that produced these fragments
+ * @param llm        LLM client instance (null = fallback to pass-through)
+ * @param config     Optional configuration overrides
+ * @returns Object with synthesized text (or original fragments on fallback)
+ */
+export async function synthesize(
+  fragments: string[],
+  query: string,
+  llm: LLMClient | null,
+  config: SynthesizerConfig = DEFAULT_SYNTHESIZER_CONFIG,
+): Promise<SynthesisResult> {
+  // Not enough fragments to justify synthesis
+  if (fragments.length < config.minFragments) {
+    return { text: null, fragments, synthesized: false, reason: "below-threshold" };
+  }
+
+  // No LLM available
+  if (!llm) {
+    return { text: null, fragments, synthesized: false, reason: "no-llm" };
+  }
+
+  try {
+    const result = await llm.synthesizeFragments(fragments, query, config.maxOutputChars);
+    if (result && result.length > 0) {
+      return { text: result, fragments, synthesized: true, reason: "ok" };
+    }
+    return { text: null, fragments, synthesized: false, reason: "empty-response" };
+  } catch (err) {
+    logWarn("Result synthesis failed, using raw fragments:", err);
+    return { text: null, fragments, synthesized: false, reason: "error" };
+  }
+}
+
+/**
+ * Synthesize a section of context-composer output.
+ * If synthesis succeeds, returns a single-element array with the narrative.
+ * Otherwise returns the original array unchanged.
+ */
+export async function synthesizeSection(
+  sectionItems: string[],
+  query: string,
+  llm: LLMClient | null,
+  config: SynthesizerConfig = DEFAULT_SYNTHESIZER_CONFIG,
+): Promise<string[]> {
+  if (!isSynthesisEnabled()) return sectionItems;
+
+  const result = await synthesize(sectionItems, query, llm, config);
+  if (result.synthesized && result.text) {
+    return [result.text];
+  }
+  return sectionItems;
+}
+
+// ============================================================================
+// Types
+// ============================================================================
+
+export interface SynthesisResult {
+  /** Synthesized narrative text, or null if synthesis was skipped/failed */
+  text: string | null;
+  /** Original fragments (preserved for fallback) */
+  fragments: string[];
+  /** Whether synthesis was actually performed */
+  synthesized: boolean;
+  /** Reason for the outcome */
+  reason: "ok" | "below-threshold" | "no-llm" | "empty-response" | "error";
+}

package/src/retrieval-profiles.ts

@@ -0,0 +1,101 @@
+import type { RetrievalConfig } from "./retriever.js";
+
+export type RetrievalProfileName = "default" | "writing" | "debug" | "fact-check";
+
+export interface RetrievalProfileDefinition {
+  name: RetrievalProfileName;
+  label: string;
+  description: string;
+  overrides: Partial<RetrievalConfig>;
+}
+
+export const RETRIEVAL_PROFILES: Record<RetrievalProfileName, RetrievalProfileDefinition> = {
+  default: {
+    name: "default",
+    label: "General Recall",
+    description: "Balanced hybrid retrieval for everyday memory search.",
+    overrides: {},
+  },
+  writing: {
+    name: "writing",
+    label: "Writing Brief",
+    description: "Prefer broader semantic recall and tolerate older but reusable material.",
+    overrides: {
+      vectorWeight: 0.8,
+      bm25Weight: 0.2,
+      candidatePoolSize: 28,
+      recencyHalfLifeDays: 90,
+      recencyWeight: 0.06,
+      timeDecayHalfLifeDays: 365,
+      hardMinScore: 0.24,
+      lengthNormAnchor: 800,
+    },
+  },
+  debug: {
+    name: "debug",
+    label: "Debug Trace",
+    description: "Favor recent, exact-match operational details such as errors, commands, and fixes.",
+    overrides: {
+      vectorWeight: 0.55,
+      bm25Weight: 0.45,
+      candidatePoolSize: 24,
+      recencyHalfLifeDays: 7,
+      recencyWeight: 0.15,
+      timeDecayHalfLifeDays: 30,
+      hardMinScore: 0.34,
+      lengthNormAnchor: 420,
+    },
+  },
+  "fact-check": {
+    name: "fact-check",
+    label: "Fact Check",
+    description: "Favor exact evidence and fresher records; tighten the cutoff to reduce weak matches.",
+    overrides: {
+      vectorWeight: 0.5,
+      bm25Weight: 0.5,
+      candidatePoolSize: 26,
+      recencyHalfLifeDays: 21,
+      recencyWeight: 0.08,
+      timeDecayHalfLifeDays: 45,
+      hardMinScore: 0.38,
+      lengthNormAnchor: 360,
+    },
+  },
+};
+
+export function normalizeRetrievalProfile(input?: string): RetrievalProfileName {
+  const normalized = (input || "default").trim().toLowerCase();
+  if (!normalized) return "default";
+  if (normalized === "factcheck" || normalized === "fact_check") return "fact-check";
+  if (normalized === "general") return "default";
+  if (normalized in RETRIEVAL_PROFILES) {
+    return normalized as RetrievalProfileName;
+  }
+  throw new Error(
+    `Unknown retrieval profile: ${input}. Available: ${Object.keys(RETRIEVAL_PROFILES).join(", ")}`,
+  );
+}
+
+export function applyRetrievalProfile(
+  baseConfig: RetrievalConfig,
+  profileName?: string,
+): { profile: RetrievalProfileDefinition; config: RetrievalConfig } {
+  const resolvedName = normalizeRetrievalProfile(profileName);
+  const profile = RETRIEVAL_PROFILES[resolvedName];
+  return {
+    profile,
+    config: {
+      ...baseConfig,
+      ...profile.overrides,
+    },
+  };
+}
+
+export function listRetrievalProfiles(): RetrievalProfileDefinition[] {
+  return [
+    RETRIEVAL_PROFILES.default,
+    RETRIEVAL_PROFILES.writing,
+    RETRIEVAL_PROFILES.debug,
+    RETRIEVAL_PROFILES["fact-check"],
+  ];
+}

package/src/retrieval-trace.ts

@@ -0,0 +1,94 @@
+/**
+ * Retrieval Trace — per-stage observability for the retrieval pipeline.
+ *
+ * Usage: pass an optional TraceCollector into retrieve(); each scoring stage
+ * calls startStage / endStage. After retrieve() returns, call summarize()
+ * for a human-readable breakdown or toJSON() for machine consumption.
+ */
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+export interface RetrievalStageResult {
+  name: string;
+  inputCount: number;
+  outputCount: number;
+  droppedCount: number;
+  scoreRange: [number, number] | null;
+  durationMs: number;
+}
+
+export interface RetrievalTrace {
+  query: string;
+  mode: "hybrid" | "vector";
+  stages: RetrievalStageResult[];
+  finalCount: number;
+  totalMs: number;
+}
+
+// ── TraceCollector ───────────────────────────────────────────────────────────
+
+export class TraceCollector {
+  private stages: RetrievalStageResult[] = [];
+  private stageStart = 0;
+  private currentStageName = "";
+  private currentInputCount = 0;
+  private traceStart = 0;
+
+  constructor() {
+    this.traceStart = Date.now();
+  }
+
+  startStage(name: string, inputCount: number): void {
+    this.currentStageName = name;
+    this.currentInputCount = inputCount;
+    this.stageStart = Date.now();
+  }
+
+  endStage(outputCount: number, scores?: number[]): void {
+    const durationMs = Date.now() - this.stageStart;
+    let scoreRange: [number, number] | null = null;
+    if (scores && scores.length > 0) {
+      const finite = scores.filter(Number.isFinite);
+      if (finite.length > 0) {
+        scoreRange = [Math.min(...finite), Math.max(...finite)];
+      }
+    }
+    this.stages.push({
+      name: this.currentStageName,
+      inputCount: this.currentInputCount,
+      outputCount,
+      droppedCount: this.currentInputCount - outputCount,
+      scoreRange,
+      durationMs,
+    });
+  }
+
+  finalize(query: string, mode: "hybrid" | "vector"): RetrievalTrace {
+    const finalCount = this.stages.length > 0
+      ? this.stages[this.stages.length - 1].outputCount
+      : 0;
+    return {
+      query,
+      mode,
+      stages: this.stages,
+      finalCount,
+      totalMs: Date.now() - this.traceStart,
+    };
+  }
+
+  summarize(query: string, mode: "hybrid" | "vector"): string {
+    const trace = this.finalize(query, mode);
+    const lines: string[] = [];
+    lines.push(`[retrieve] query="${trace.query}" mode=${trace.mode} total=${trace.totalMs}ms`);
+    for (const s of trace.stages) {
+      const range = s.scoreRange
+        ? ` [${s.scoreRange[0].toFixed(3)}–${s.scoreRange[1].toFixed(3)}]`
+        : "";
+      lines.push(
+        `  ${s.name}: ${s.inputCount} → ${s.outputCount} (${s.droppedCount} dropped)${range} ${s.durationMs}ms`,
+      );
+    }
+    lines.push(`  final: ${trace.finalCount} results`);
+    return lines.join("\n");
+  }
+}