ralph-hero-knowledge-index 0.1.21 → 0.1.23

package/src/index.ts

@@ -22,12 +22,60 @@ function resolveEnv(name: string): string | undefined {
   return val;
 }
 
-export function createServer(dbPath: string) {
+/**
+ * True when the `chunks` table exists in the schema (v3+). When absent,
+ * `knowledge_memory_stats` reports 0 chunks-per-doc percentiles.
+ */
+function chunksTableExists(db: KnowledgeDB): boolean {
+  const row = db.db
+    .prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='chunks'",
+    )
+    .get();
+  return row !== undefined;
+}
+
+/**
+ * True when the `documents.memory_tier` column exists (v3+). Used to decide
+ * whether tier-level stats can be produced from the schema at all.
+ */
+function memoryTierColumnExists(db: KnowledgeDB): boolean {
+  const rows = db.db
+    .prepare("PRAGMA table_info(documents)")
+    .all() as Array<{ name: string }>;
+  return rows.some((r) => r.name === "memory_tier");
+}
+
+/**
+ * Percentile helper using nearest-rank. For n sorted values returns the value
+ * at index `floor(n * p)` clamped to [0, n-1]. Returns 0 on empty input.
+ * Matches the spec in Phase 8 Task 8.4: "pick index at floor(n*0.5)".
+ */
+function percentile(sortedValues: number[], p: number): number {
+  if (sortedValues.length === 0) return 0;
+  const idx = Math.min(
+    sortedValues.length - 1,
+    Math.max(0, Math.floor(sortedValues.length * p)),
+  );
+  return sortedValues[idx];
+}
+
+/**
+ * Options for `createServer`. When `embedFn` is provided it replaces the
+ * production `embed` import, allowing tests to bypass the HuggingFace model
+ * download.
+ */
+export interface CreateServerOptions {
+  embedFn?: (text: string) => Promise<Float32Array>;
+}
+
+export function createServer(dbPath: string, opts: CreateServerOptions = {}) {
   const server = new McpServer({ name: "ralph-hero-knowledge", version: "0.1.0" });
   const db = new KnowledgeDB(dbPath);
   const fts = new FtsSearch(db);
   const vec = new VectorSearch(db);
-  const hybrid = new HybridSearch(db, fts, vec, embed);
+  const embedImpl = opts.embedFn ?? embed;
+  const hybrid = new HybridSearch(db, fts, vec, embedImpl);
   const traverser = new Traverser(db);
 
   server.tool(
@@ -40,6 +88,16 @@ export function createServer(dbPath: string) {
       limit: z.number().optional().describe("Max results (default: 10)"),
       includeSuperseded: z.boolean().optional().describe("Include superseded documents (default: false)"),
       brief: z.boolean().optional().describe("Return minimal metadata only (default: false)"),
+      memory_tier: z
+        .enum(["doc", "raw", "reflection", "any"])
+        .optional()
+        .default("any")
+        .describe("Filter by memory tier: 'doc' (curated), 'raw' (dream-loop ingest), 'reflection' (synthesized), 'any' (default)"),
+      return_chunk_meta: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe("Include chunk_index/char_start/char_end/context_prefix in each hit when chunk data is available"),
     },
     async (args) => {
       try {
@@ -48,18 +106,33 @@ export function createServer(dbPath: string) {
           type: args.type,
           limit: args.limit ?? 10,
           includeSuperseded: args.includeSuperseded,
+          memoryTier: args.memory_tier,
         });
-        const enriched = results.map(r => {
-          const base = { ...r, tags: db.getTags(r.id) };
+        const enriched = results.map((r) => {
+          // Start with the camelCase SearchResult shape so existing callers
+          // keep working, then optionally add snake_case aliases for new
+          // chunk fields and strip them when callers didn't opt in.
+          const { chunkIndex, charStart, charEnd, contextPrefix, bestChunkId, ...rest } = r;
+          const base: Record<string, unknown> = { ...rest, tags: db.getTags(r.id) };
+          if (args.return_chunk_meta) {
+            if (chunkIndex !== undefined) base.chunk_index = chunkIndex;
+            if (charStart !== undefined) base.char_start = charStart;
+            if (charEnd !== undefined) base.char_end = charEnd;
+            if (contextPrefix !== undefined) base.context_prefix = contextPrefix;
+            if (bestChunkId !== undefined) base.best_chunk_id = bestChunkId;
+          }
           // SearchResult does not carry githubIssue — fetch from documents table
           const doc = db.getDocument(r.id);
           if (doc?.githubIssue) {
             const outcomes = db.getOutcomeSummary(doc.githubIssue);
-            if (outcomes) return { ...base, outcomes_summary: outcomes };
+            if (outcomes) base.outcomes_summary = outcomes;
           }
           return base;
         });
-        const formatted = formatSearchResults(enriched, args.brief ?? false);
+        const formatted = formatSearchResults(
+          enriched as unknown as Parameters<typeof formatSearchResults>[0],
+          args.brief ?? false,
+        );
         return { content: [{ type: "text" as const, text: JSON.stringify(formatted, null, 2) }] };
       } catch (e) {
         return { content: [{ type: "text" as const, text: `Error: ${(e as Error).message}` }], isError: true };
@@ -76,13 +149,26 @@ export function createServer(dbPath: string) {
       depth: z.number().optional().describe("Max traversal depth (default: 3)"),
       direction: z.enum(["outgoing", "incoming"]).optional().describe("Edge direction (default: outgoing)"),
       brief: z.boolean().optional().describe("Return minimal metadata only (default: false)"),
+      memory_tier: z
+        .enum(["doc", "raw", "reflection", "any"])
+        .optional()
+        .default("any")
+        .describe("Filter traversed nodes by memory tier (default: 'any')"),
     },
     async (args) => {
       try {
         const opts = { type: args.type, depth: args.depth ?? 3 };
-        const results = args.direction === "incoming"
+        let results = args.direction === "incoming"
           ? traverser.traverseIncoming(args.from, opts)
           : traverser.traverse(args.from, opts);
+        if (args.memory_tier && args.memory_tier !== "any") {
+          const wantedTier = args.memory_tier;
+          results = results.filter((r) => {
+            const tier = db.getMemoryTier(r.targetId);
+            // When memory_tier column is absent (pre-v3 DB) treat as "doc"
+            return (tier ?? "doc") === wantedTier;
+          });
+        }
         const formatted = formatTraverseResults(results, (id) => db.getTags(id), args.brief ?? false);
         return { content: [{ type: "text" as const, text: JSON.stringify(formatted, null, 2) }] };
       } catch (e) {
@@ -91,6 +177,115 @@ export function createServer(dbPath: string) {
     },
   );
 
+  server.tool(
+    "knowledge_memory_stats",
+    "Return counts of documents by memory_tier plus chunk percentiles and last-reflection timestamp. Used by the dream-loop to confirm ingest/reflection completion.",
+    {
+      since: z
+        .string()
+        .optional()
+        .describe("ISO timestamp — counts for 'new_since' are computed against this. Defaults to 24 hours ago."),
+    },
+    async (args) => {
+      try {
+        const since = args.since ?? new Date(Date.now() - 24 * 3600 * 1000).toISOString();
+        const hasTier = memoryTierColumnExists(db);
+        const hasChunks = chunksTableExists(db);
+
+        const totalRow = db.db
+          .prepare("SELECT COUNT(*) AS c FROM documents")
+          .get() as { c: number };
+        const totalDocuments = totalRow.c;
+
+        const byTier: Record<"doc" | "raw" | "reflection", number> = {
+          doc: 0,
+          raw: 0,
+          reflection: 0,
+        };
+        const newSince: Record<"doc" | "raw" | "reflection", number> = {
+          doc: 0,
+          raw: 0,
+          reflection: 0,
+        };
+
+        if (hasTier) {
+          const rows = db.db
+            .prepare(
+              `SELECT memory_tier AS tier, COUNT(*) AS c
+               FROM documents GROUP BY memory_tier`,
+            )
+            .all() as Array<{ tier: string; c: number }>;
+          for (const r of rows) {
+            if (r.tier === "doc" || r.tier === "raw" || r.tier === "reflection") {
+              byTier[r.tier] = r.c;
+            }
+          }
+          const newRows = db.db
+            .prepare(
+              `SELECT memory_tier AS tier, COUNT(*) AS c
+               FROM documents
+               WHERE date IS NOT NULL AND date >= @since
+               GROUP BY memory_tier`,
+            )
+            .all({ since }) as Array<{ tier: string; c: number }>;
+          for (const r of newRows) {
+            if (r.tier === "doc" || r.tier === "raw" || r.tier === "reflection") {
+              newSince[r.tier] = r.c;
+            }
+          }
+        } else {
+          // v2 schema — everything treated as "doc"
+          byTier.doc = totalDocuments;
+          const newDocRow = db.db
+            .prepare(
+              "SELECT COUNT(*) AS c FROM documents WHERE date IS NOT NULL AND date >= ?",
+            )
+            .get(since) as { c: number };
+          newSince.doc = newDocRow.c;
+        }
+
+        let chunksPerDocP50 = 0;
+        let chunksPerDocP90 = 0;
+        if (hasChunks) {
+          const perDoc = db.db
+            .prepare(
+              `SELECT COUNT(*) AS c FROM chunks GROUP BY document_id`,
+            )
+            .all() as Array<{ c: number }>;
+          const counts = perDoc.map((r) => r.c).sort((a, b) => a - b);
+          chunksPerDocP50 = percentile(counts, 0.5);
+          chunksPerDocP90 = percentile(counts, 0.9);
+        }
+
+        let lastReflectionAt: string | null = null;
+        if (hasTier) {
+          const row = db.db
+            .prepare(
+              `SELECT date FROM documents
+               WHERE memory_tier = 'reflection' AND date IS NOT NULL
+               ORDER BY date DESC LIMIT 1`,
+            )
+            .get() as { date: string } | undefined;
+          lastReflectionAt = row?.date ?? null;
+        }
+
+        const payload = {
+          total_documents: totalDocuments,
+          by_tier: byTier,
+          new_since: newSince,
+          chunks_per_doc_p50: chunksPerDocP50,
+          chunks_per_doc_p90: chunksPerDocP90,
+          last_reflection_at: lastReflectionAt,
+          since,
+        };
+
+        return { content: [{ type: "text" as const, text: JSON.stringify(payload, null, 2) }] };
+      } catch (e) {
+        return { content: [{ type: "text" as const, text: `Error: ${(e as Error).message}` }], isError: true };
+      }
+    },
+  );
+
   server.tool(
     "knowledge_record_outcome",
     "Record a pipeline outcome event (research, plan, phase, validation, etc.)",

package/src/llm-client.ts

@@ -0,0 +1,136 @@
+/**
+ * Minimal OpenAI-compatible LLM client for Contextual Retrieval.
+ *
+ * Probes `${baseUrl}/v1/models` for availability and calls
+ * `${baseUrl}/v1/chat/completions` to generate short context prefixes for
+ * document chunks. Uses native `fetch` + `AbortController` — no SDK dependency.
+ *
+ * Fail-open semantics: network errors, timeouts, non-200 responses, and
+ * malformed JSON all resolve without throwing. `available()` returns `false`;
+ * `contextualize()` returns an empty string. The caller is expected to treat
+ * an empty context prefix as "no context available" and continue.
+ *
+ * Defaults target `gemma-lab` at `http://localhost:8000` with the Gemma 4 26B
+ * MXFP8 model. Override via `RALPH_LLM_URL` / `RALPH_LLM_MODEL` env vars or
+ * explicit options.
+ */
+
+export interface LlmClientOptions {
+  /** Base URL for the OpenAI-compatible endpoint. Default: RALPH_LLM_URL env or http://localhost:8000. */
+  baseUrl?: string;
+  /** Model identifier sent in chat completion requests. Default: RALPH_LLM_MODEL env or mlx-community/gemma-4-26b-a4b-it-mxfp8. */
+  model?: string;
+  /** Timeout for contextualize() in milliseconds. Default: 30000. */
+  timeoutMs?: number;
+}
+
+export interface LlmClient {
+  /**
+   * Probe the endpoint for availability.
+   * Returns `true` iff `${baseUrl}/v1/models` responds with HTTP 200 within 2000ms.
+   * Returns `false` on timeout, connection refused, non-200, or any thrown exception.
+   */
+  available(): Promise<boolean>;
+
+  /**
+   * Generate a short (≤100 token) context prefix situating `chunkContent`
+   * within `fullDocument`, using the Anthropic Contextual Retrieval prompt.
+   *
+   * Returns the trimmed content string on success, or `""` on any error
+   * (network failure, timeout, non-2xx response, missing choices, malformed JSON).
+   */
+  contextualize(fullDocument: string, chunkContent: string): Promise<string>;
+}
+
+const DEFAULT_BASE_URL = "http://localhost:8000";
+const DEFAULT_MODEL = "mlx-community/gemma-4-26b-a4b-it-mxfp8";
+const DEFAULT_TIMEOUT_MS = 30000;
+const AVAILABLE_PROBE_TIMEOUT_MS = 2000;
+const MAX_CONTEXT_TOKENS = 120;
+
+/**
+ * Anthropic Contextual Retrieval prompt, verbatim from the parent plan Phase 2.
+ * Placeholders `{fullDocument}` and `{chunkContent}` are filled at call time.
+ */
+function buildContextualizePrompt(fullDocument: string, chunkContent: string): string {
+  return `<document>
+${fullDocument}
+</document>
+
+Here is the chunk we want to situate within the whole document:
+
+<chunk>
+${chunkContent}
+</chunk>
+
+Please give a short succinct context to situate this chunk within the overall
+document for the purposes of improving search retrieval of the chunk. Answer only
+with the succinct context and nothing else.`;
+}
+
+interface ChatCompletionResponse {
+  choices?: Array<{
+    message?: {
+      content?: string;
+    };
+  }>;
+}
+
+export function createLlmClient(opts: LlmClientOptions = {}): LlmClient {
+  const baseUrl = opts.baseUrl ?? process.env.RALPH_LLM_URL ?? DEFAULT_BASE_URL;
+  const model = opts.model ?? process.env.RALPH_LLM_MODEL ?? DEFAULT_MODEL;
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+  async function available(): Promise<boolean> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), AVAILABLE_PROBE_TIMEOUT_MS);
+    try {
+      const response = await fetch(`${baseUrl}/v1/models`, {
+        method: "GET",
+        signal: controller.signal,
+      });
+      return response.status === 200;
+    } catch {
+      return false;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async function contextualize(fullDocument: string, chunkContent: string): Promise<string> {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const prompt = buildContextualizePrompt(fullDocument, chunkContent);
+      const response = await fetch(`${baseUrl}/v1/chat/completions`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          model,
+          messages: [{ role: "user", content: prompt }],
+          max_tokens: MAX_CONTEXT_TOKENS,
+        }),
+        signal: controller.signal,
+      });
+
+      if (!response.ok) {
+        return "";
+      }
+
+      const data = (await response.json()) as ChatCompletionResponse;
+      const content = data?.choices?.[0]?.message?.content;
+      if (typeof content !== "string") {
+        return "";
+      }
+      return content.trim();
+    } catch {
+      return "";
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  return { available, contextualize };
+}

package/src/reindex.ts

@@ -8,8 +8,15 @@ import { embed, prepareTextForEmbedding } from "./embedder.js";
 import { parseDocument, type ParsedDocument } from "./parser.js";
 import { findMarkdownFiles } from "./file-scanner.js";
 import { generateIndexes } from "./generate-indexes.js";
+import { loadConfig, type KnowledgeConfig } from "./config.js";
+import { loadIgnoreForRoot } from "./ignore.js";
 
-export async function reindex(dirs: string[], dbPath: string, generate: boolean = false): Promise<void> {
+export async function reindex(
+  dirs: string[],
+  dbPath: string,
+  generate: boolean = false,
+  ignorePatterns?: string[],
+): Promise<void> {
   console.log(`Indexing ${dirs.join(", ")} -> ${dbPath}`);
 
   const db = new KnowledgeDB(dbPath);
@@ -32,7 +39,8 @@ export async function reindex(dirs: string[], dbPath: string, generate: boolean
   // Phase 1: Discover files on disk
   const filesOnDisk: string[] = [];
   for (const dir of dirs) {
-    const found = findMarkdownFiles(dir);
+    const matcher = loadIgnoreForRoot(dir, ignorePatterns);
+    const found = findMarkdownFiles(dir, matcher);
     console.log(`  ${dir}: ${found.length} files`);
     filesOnDisk.push(...found);
   }
@@ -182,31 +190,94 @@ export async function reindex(dirs: string[], dbPath: string, generate: boolean
 
 const DEFAULT_DB_PATH = join(homedir(), ".ralph-hero", "knowledge.db");
 
-export function resolveDirs(): { dirs: string[]; dbPath: string; generate: boolean } {
+export type ResolvedDirsSource = "cli" | "env" | "config" | "fallback";
+
+export interface ResolvedDirs {
+  dirs: string[];
+  dbPath: string;
+  generate: boolean;
+  source: ResolvedDirsSource;
+  config: KnowledgeConfig;
+}
+
+/**
+ * Resolve the set of roots, database path, and generate flag for a reindex
+ * run. Precedence (highest to lowest):
+ *   1. CLI positional args
+ *   2. `RALPH_KNOWLEDGE_DIRS` environment variable
+ *   3. `config.roots` from `~/.ralph/knowledge.config.json`
+ *   4. `"../../thoughts"` fallback
+ *
+ * `dbPath` precedence is independent: CLI `.db` positional > `RALPH_KNOWLEDGE_DB`
+ * env var > `config.dbPath` > {@link DEFAULT_DB_PATH}.
+ *
+ * The returned `config` is forwarded to the caller so `ignorePatterns` can be
+ * threaded into {@link reindex}.
+ */
+export function resolveDirs(): ResolvedDirs {
   const cliArgs = process.argv.slice(2);
   const noGenerate = cliArgs.includes("--no-generate");
   const positional = cliArgs.filter(a => !a.startsWith("--"));
   const cliDb = positional.find(a => a.endsWith(".db"));
   const cliDirs = positional.filter(a => !a.endsWith(".db"));
 
+  const config = loadConfig();
+
+  const resolveDbPath = (): string =>
+    cliDb ??
+    process.env.RALPH_KNOWLEDGE_DB ??
+    config.dbPath ??
+    DEFAULT_DB_PATH;
+
   if (cliDirs.length > 0) {
-    return { dirs: cliDirs, dbPath: cliDb ?? DEFAULT_DB_PATH, generate: !noGenerate };
+    console.log("Using roots from: CLI");
+    return {
+      dirs: cliDirs,
+      dbPath: resolveDbPath(),
+      generate: !noGenerate,
+      source: "cli",
+      config,
+    };
   }
 
   const envDirs = process.env.RALPH_KNOWLEDGE_DIRS;
   if (envDirs) {
+    const parsed = envDirs.split(",").map(d => d.trim()).filter(Boolean);
+    if (parsed.length > 0) {
+      console.log("Using roots from: env");
+      return {
+        dirs: parsed,
+        dbPath: resolveDbPath(),
+        generate: !noGenerate,
+        source: "env",
+        config,
+      };
+    }
+  }
+
+  if (config.roots && config.roots.length > 0) {
+    console.log("Using roots from: config");
     return {
-      dirs: envDirs.split(",").map(d => d.trim()).filter(Boolean),
-      dbPath: cliDb ?? process.env.RALPH_KNOWLEDGE_DB ?? DEFAULT_DB_PATH,
+      dirs: config.roots,
+      dbPath: resolveDbPath(),
       generate: !noGenerate,
+      source: "config",
+      config,
     };
   }
 
-  return { dirs: ["../../thoughts"], dbPath: cliDb ?? DEFAULT_DB_PATH, generate: !noGenerate };
+  console.log("Using roots from: fallback");
+  return {
+    dirs: ["../../thoughts"],
+    dbPath: resolveDbPath(),
+    generate: !noGenerate,
+    source: "fallback",
+    config,
+  };
 }
 
 const isMain = process.argv[1]?.endsWith("reindex.js");
 if (isMain) {
-  const { dirs, dbPath, generate } = resolveDirs();
-  reindex(dirs, dbPath, generate).catch(console.error);
+  const { dirs, dbPath, generate, config } = resolveDirs();
+  reindex(dirs, dbPath, generate, config.ignorePatterns).catch(console.error);
 }

package/src/search.ts

@@ -1,10 +1,13 @@
 import type { KnowledgeDB } from "./db.js";
 
+export type MemoryTier = "doc" | "raw" | "reflection" | "any";
+
 export interface SearchOptions {
   type?: string;
   tags?: string[];
   includeSuperseded?: boolean;
   limit?: number;
+  memoryTier?: MemoryTier;
 }
 
 export interface SearchResult {
@@ -16,6 +19,13 @@ export interface SearchResult {
   date: string | null;
   score: number;
   snippet: string;
+  // Optional chunk-level metadata. Populated when chunk data is available
+  // for the best-scoring chunk of this document.
+  chunkIndex?: number;
+  charStart?: number;
+  charEnd?: number;
+  contextPrefix?: string;
+  bestChunkId?: string;
 }
 
 export class FtsSearch {
@@ -99,8 +109,19 @@ export class FtsSearch {
     return tokens.map(t => '"' + t.replace(/"/g, '""') + '"').join(" ");
   }
 
+  /**
+   * Returns true when the `documents.memory_tier` column exists (schema v3+).
+   * On v2 schemas this is false and the memoryTier filter is silently ignored.
+   */
+  private memoryTierColumnExists(): boolean {
+    const rows = this.db.db
+      .prepare("PRAGMA table_info(documents)")
+      .all() as Array<{ name: string }>;
+    return rows.some((r) => r.name === "memory_tier");
+  }
+
   search(query: string, options: SearchOptions = {}): SearchResult[] {
-    const { type, tags, includeSuperseded = false, limit = 20 } = options;
+    const { type, tags, includeSuperseded = false, limit = 20, memoryTier } = options;
 
     const conditions: string[] = ["documents_fts MATCH @query"];
     const params: Record<string, unknown> = { query: this.escapeFts5Query(query), limit };
@@ -114,6 +135,11 @@ export class FtsSearch {
       params.type = type;
     }
 
+    if (memoryTier && memoryTier !== "any" && this.memoryTierColumnExists()) {
+      conditions.push("d.memory_tier = @memoryTier");
+      params.memoryTier = memoryTier;
+    }
+
     let joinClause = "";
     if (tags && tags.length > 0) {
       joinClause = "JOIN tags t ON t.doc_id = d.id";