ralph-hero-knowledge-index 0.1.21 → 0.1.23

package/src/__tests__/reindex.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 import { mkdtempSync, writeFileSync, mkdirSync, unlinkSync, utimesSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { tmpdir } from "node:os";
@@ -15,7 +15,7 @@ vi.mock("../embedder.js", () => ({
 }));
 
 import { embed } from "../embedder.js";
-import { reindex } from "../reindex.js";
+import { reindex, resolveDirs } from "../reindex.js";
 import { KnowledgeDB } from "../db.js";
 
 const mockedEmbed = vi.mocked(embed);
@@ -353,4 +353,150 @@ describe("incremental reindex", () => {
     expect(results.some(r => r.id === "fresh-doc")).toBe(true);
     db1.close();
   });
+
+  it("scenario 13: reindex honors .ralphignore for file discovery", async () => {
+    writeFileSync(join(dir, "kept.md"), makeDoc("Kept"));
+    writeFileSync(join(dir, "skipped.md"), makeDoc("Skipped"));
+    writeFileSync(join(dir, ".ralphignore"), "skipped.md\n");
+
+    await reindex([dir], dbPath);
+    expect(mockedEmbed).toHaveBeenCalledTimes(1);
+
+    const db = new KnowledgeDB(dbPath);
+    expect(db.getDocument("kept")).toBeTruthy();
+    expect(db.getDocument("skipped")).toBeUndefined();
+    db.close();
+  });
+
+  it("scenario 14: reindex honors caller-supplied ignorePatterns arg", async () => {
+    writeFileSync(join(dir, "kept.md"), makeDoc("Kept"));
+    mkdirSync(join(dir, "drafts"));
+    writeFileSync(join(dir, "drafts", "wip.md"), makeDoc("WIP"));
+
+    await reindex([dir], dbPath, false, ["drafts/**"]);
+    // Only kept.md should have been embedded.
+    expect(mockedEmbed).toHaveBeenCalledTimes(1);
+
+    const db = new KnowledgeDB(dbPath);
+    expect(db.getDocument("kept")).toBeTruthy();
+    expect(db.getDocument("wip")).toBeUndefined();
+    db.close();
+  });
+});
+
+describe("resolveDirs precedence", () => {
+  const ORIGINAL_ARGV = process.argv;
+  const ORIGINAL_ENV = {
+    RALPH_KNOWLEDGE_DIRS: process.env.RALPH_KNOWLEDGE_DIRS,
+    RALPH_KNOWLEDGE_DB: process.env.RALPH_KNOWLEDGE_DB,
+    RALPH_KNOWLEDGE_CONFIG: process.env.RALPH_KNOWLEDGE_CONFIG,
+  };
+  let tmpHome: string;
+  let configDir: string;
+
+  beforeEach(() => {
+    process.argv = ["node", "reindex.js"];
+    delete process.env.RALPH_KNOWLEDGE_DIRS;
+    delete process.env.RALPH_KNOWLEDGE_DB;
+    configDir = mkdtempSync(join(tmpdir(), "resolve-dirs-"));
+    tmpHome = configDir;
+    process.env.RALPH_KNOWLEDGE_CONFIG = join(configDir, "knowledge.config.json");
+  });
+
+  afterEach(() => {
+    process.argv = ORIGINAL_ARGV;
+    for (const key of Object.keys(ORIGINAL_ENV) as (keyof typeof ORIGINAL_ENV)[]) {
+      const orig = ORIGINAL_ENV[key];
+      if (orig === undefined) {
+        delete process.env[key];
+      } else {
+        process.env[key] = orig;
+      }
+    }
+  });
+
+  it("CLI positional args beat env var even when both are set", () => {
+    writeFileSync(
+      process.env.RALPH_KNOWLEDGE_CONFIG!,
+      JSON.stringify({ roots: ["/from/config"] }),
+    );
+    process.argv = ["node", "reindex.js", "/from/cli"];
+    process.env.RALPH_KNOWLEDGE_DIRS = "/from/env";
+    const r = resolveDirs();
+    expect(r.source).toBe("cli");
+    expect(r.dirs).toEqual(["/from/cli"]);
+  });
+
+  it("env var beats config file roots when CLI is empty", () => {
+    writeFileSync(
+      process.env.RALPH_KNOWLEDGE_CONFIG!,
+      JSON.stringify({ roots: ["/from/config"] }),
+    );
+    process.env.RALPH_KNOWLEDGE_DIRS = "/from/env-a,/from/env-b";
+    const r = resolveDirs();
+    expect(r.source).toBe("env");
+    expect(r.dirs).toEqual(["/from/env-a", "/from/env-b"]);
+  });
+
+  it("config file roots beat fallback when CLI and env are absent", () => {
+    writeFileSync(
+      process.env.RALPH_KNOWLEDGE_CONFIG!,
+      JSON.stringify({ roots: ["/from/config-a", "/from/config-b"] }),
+    );
+    const r = resolveDirs();
+    expect(r.source).toBe("config");
+    expect(r.dirs).toEqual(["/from/config-a", "/from/config-b"]);
+  });
+
+  it("falls back to ../../thoughts when no source is configured", () => {
+    // Point env var at a nonexistent config path so loadConfig returns {}.
+    process.env.RALPH_KNOWLEDGE_CONFIG = join(configDir, "missing.json");
+    const r = resolveDirs();
+    expect(r.source).toBe("fallback");
+    expect(r.dirs).toEqual(["../../thoughts"]);
+  });
+
+  it("dbPath precedence: CLI arg > env var > config > default", () => {
+    writeFileSync(
+      process.env.RALPH_KNOWLEDGE_CONFIG!,
+      JSON.stringify({ roots: ["/x"], dbPath: "/from/config.db" }),
+    );
+    // CLI wins
+    process.argv = ["node", "reindex.js", "/cli/root", "/cli/override.db"];
+    process.env.RALPH_KNOWLEDGE_DB = "/from/env.db";
+    expect(resolveDirs().dbPath).toBe("/cli/override.db");
+
+    // Env wins over config when CLI is absent
+    process.argv = ["node", "reindex.js"];
+    process.env.RALPH_KNOWLEDGE_DIRS = "/env/root";
+    process.env.RALPH_KNOWLEDGE_DB = "/from/env.db";
+    expect(resolveDirs().dbPath).toBe("/from/env.db");
+
+    // Config wins when neither CLI nor env set dbPath
+    delete process.env.RALPH_KNOWLEDGE_DB;
+    expect(resolveDirs().dbPath).toBe("/from/config.db");
+  });
+
+  it("forwards config.ignorePatterns on the returned config object", () => {
+    writeFileSync(
+      process.env.RALPH_KNOWLEDGE_CONFIG!,
+      JSON.stringify({
+        roots: ["/r1"],
+        ignorePatterns: ["draft/**", "*.bak"],
+      }),
+    );
+    const r = resolveDirs();
+    expect(r.config.ignorePatterns).toEqual(["draft/**", "*.bak"]);
+  });
+
+  it("treats an empty RALPH_KNOWLEDGE_DIRS as unset and falls through", () => {
+    writeFileSync(
+      process.env.RALPH_KNOWLEDGE_CONFIG!,
+      JSON.stringify({ roots: ["/from/config"] }),
+    );
+    process.env.RALPH_KNOWLEDGE_DIRS = "  ,  ";
+    const r = resolveDirs();
+    expect(r.source).toBe("config");
+    expect(r.dirs).toEqual(["/from/config"]);
+  });
 });

package/src/__tests__/search.test.ts

@@ -205,6 +205,43 @@ describe("FtsSearch", () => {
     });
   });
 
+  describe("memory_tier filter", () => {
+    it("filters by memory_tier when schema has the column", () => {
+      db.db
+        .prepare("UPDATE documents SET memory_tier = ? WHERE id = ?")
+        .run("reflection", "auth-doc");
+      fts.rebuildIndex();
+
+      // auth-doc is "reflection", so search for terms in auth-doc should hit.
+      const reflectionHits = fts.search("authentication", { memoryTier: "reflection" });
+      const ids = reflectionHits.map((r) => r.id);
+      expect(ids).toContain("auth-doc");
+
+      // A "doc" filter should omit the reflection-tagged doc.
+      const docHits = fts.search("authentication", { memoryTier: "doc" });
+      expect(docHits.some((r) => r.id === "auth-doc")).toBe(false);
+    });
+
+    it("ignores memory_tier silently when column is absent (v2 schema)", () => {
+      // beforeEach gives us a v2 schema — column does not exist.
+      const results = fts.search("cache", { memoryTier: "reflection" });
+      // Filter is a no-op on v2; regular FTS results come through.
+      expect(Array.isArray(results)).toBe(true);
+    });
+
+    it("returns all tiers when memoryTier='any'", () => {
+      db.db
+        .prepare("UPDATE documents SET memory_tier = ? WHERE id = ?")
+        .run("reflection", "auth-doc");
+      fts.rebuildIndex();
+
+      const authHits = fts.search("authentication", { memoryTier: "any" });
+      expect(authHits.some((r) => r.id === "auth-doc")).toBe(true);
+      const cacheHits = fts.search("cache", { memoryTier: "any" });
+      expect(cacheHits.some((r) => r.id === "cache-doc")).toBe(true);
+    });
+  });
+
   describe("ensureTable", () => {
     it("creates FTS table if it does not exist", () => {
       // Create a fresh DB without FTS table

package/src/config.ts

@@ -0,0 +1,105 @@
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { homedir } from "node:os";
+
+/**
+ * Shape of the optional `~/.ralph/knowledge.config.json` file.
+ *
+ * All fields are optional. Unknown fields are preserved at parse time but are
+ * not surfaced through this interface — callers should treat the file as
+ * forward-compatible.
+ */
+export interface KnowledgeConfig {
+  /** Absolute or `~`-prefixed directories to index. */
+  roots?: string[];
+  /** Extra gitignore-syntax patterns layered on top of per-root `.ralphignore`. */
+  ignorePatterns?: string[];
+  /** Override for the SQLite database path. */
+  dbPath?: string;
+}
+
+/**
+ * Expand a leading `~` or `~/` segment in a path to the user's home directory.
+ * Paths that do not begin with `~` are returned unchanged.
+ */
+export function expandHome(p: string): string {
+  if (!p) return p;
+  if (p === "~") return homedir();
+  if (p.startsWith("~/") || p.startsWith("~\\")) {
+    return join(homedir(), p.slice(2));
+  }
+  return p;
+}
+
+/**
+ * Resolve the knowledge config file path. Precedence:
+ *   1. `process.env.RALPH_KNOWLEDGE_CONFIG`
+ *   2. `~/.ralph/knowledge.config.json`
+ */
+export function resolveConfigPath(): string {
+  const envPath = process.env.RALPH_KNOWLEDGE_CONFIG;
+  if (envPath && envPath.trim().length > 0) {
+    return expandHome(envPath);
+  }
+  return join(homedir(), ".ralph", "knowledge.config.json");
+}
+
+/**
+ * Load the optional `knowledge.config.json` file. Returns an empty object when
+ * the file is missing or malformed. Tilde-prefixed paths inside `roots` and
+ * `dbPath` are expanded eagerly so callers receive absolute paths.
+ */
+export function loadConfig(): KnowledgeConfig {
+  const configPath = resolveConfigPath();
+  if (!existsSync(configPath)) {
+    return {};
+  }
+
+  let raw: string;
+  try {
+    raw = readFileSync(configPath, "utf-8");
+  } catch (e) {
+    console.warn(
+      `Failed to read knowledge config at ${configPath}: ${(e as Error).message}`,
+    );
+    return {};
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    console.warn(
+      `Malformed JSON in knowledge config at ${configPath}: ${(e as Error).message}`,
+    );
+    return {};
+  }
+
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    console.warn(
+      `Knowledge config at ${configPath} is not a JSON object; ignoring.`,
+    );
+    return {};
+  }
+
+  const obj = parsed as Record<string, unknown>;
+  const out: KnowledgeConfig = {};
+
+  if (Array.isArray(obj.roots)) {
+    out.roots = obj.roots
+      .filter((r): r is string => typeof r === "string" && r.length > 0)
+      .map(expandHome);
+  }
+
+  if (Array.isArray(obj.ignorePatterns)) {
+    out.ignorePatterns = obj.ignorePatterns.filter(
+      (p): p is string => typeof p === "string" && p.length > 0,
+    );
+  }
+
+  if (typeof obj.dbPath === "string" && obj.dbPath.length > 0) {
+    out.dbPath = expandHome(obj.dbPath);
+  }
+
+  return out;
+}

package/src/db.ts

@@ -468,6 +468,23 @@ export class KnowledgeDB {
     return row !== undefined;
   }
 
+  /**
+   * Returns the `memory_tier` for the given document id. Returns `undefined`
+   * when the document does not exist OR when the `memory_tier` column is
+   * absent from the schema (pre-v3 databases). Used by MCP `knowledge_*`
+   * tools that need to post-filter result sets by tier.
+   */
+  getMemoryTier(id: string): string | undefined {
+    const columns = this.db
+      .prepare("PRAGMA table_info(documents)")
+      .all() as Array<{ name: string }>;
+    if (!columns.some((c) => c.name === "memory_tier")) return undefined;
+    const row = this.db
+      .prepare("SELECT memory_tier AS memoryTier FROM documents WHERE id = ?")
+      .get(id) as { memoryTier: string } | undefined;
+    return row?.memoryTier;
+  }
+
   deleteDocument(id: string): void {
     this.db.prepare("DELETE FROM documents WHERE id = ?").run(id);
   }

package/src/file-scanner.ts

@@ -1,14 +1,39 @@
 import { readdirSync } from "node:fs";
-import { join } from "node:path";
+import { join, relative } from "node:path";
+import type { IgnoreMatcher } from "./ignore.js";
 
-export function findMarkdownFiles(dir: string): string[] {
+/**
+ * Recursively find all `.md` files under `dir`.
+ *
+ * Directory names beginning with `.` or `_` and file names beginning with `_`
+ * are always skipped (fast-path). When an {@link IgnoreMatcher} is supplied,
+ * each remaining path is additionally tested against it via its root-relative
+ * form; matches are skipped.
+ *
+ * @param dir root directory to walk
+ * @param matcher optional matcher built via `loadIgnoreForRoot(dir, …)`
+ */
+export function findMarkdownFiles(dir: string, matcher?: IgnoreMatcher): string[] {
   const results: string[] = [];
   function walk(d: string) {
     for (const entry of readdirSync(d, { withFileTypes: true })) {
       const fullPath = join(d, entry.name);
-      if (entry.isDirectory() && !entry.name.startsWith(".") && !entry.name.startsWith("_")) {
+      if (entry.isDirectory()) {
+        // Fast-path: hidden/underscored directories are always skipped.
+        if (entry.name.startsWith(".") || entry.name.startsWith("_")) continue;
+        if (matcher) {
+          // Test both bare and trailing-slash forms so gitignore-style
+          // directory-only patterns (e.g., `dist/`) match even when the
+          // directory itself has not yet been descended.
+          const rel = relative(dir, fullPath);
+          if (matcher.isIgnored(rel) || matcher.isIgnored(`${rel}/`)) continue;
+        }
         walk(fullPath);
       } else if (entry.isFile() && entry.name.endsWith(".md") && !entry.name.startsWith("_")) {
+        if (matcher) {
+          const rel = relative(dir, fullPath);
+          if (matcher.isIgnored(rel)) continue;
+        }
         results.push(fullPath);
       }
     }

package/src/hybrid-search.ts

@@ -4,6 +4,16 @@ import type { VectorSearch } from "./vector-search.js";
 
 export type EmbedFn = (text: string) => Promise<Float32Array>;
 
+interface ChunkRow {
+  id: string;
+  document_id: string;
+  chunk_index: number;
+  char_start: number;
+  char_end: number;
+  context_prefix: string;
+  content: string;
+}
+
 export class HybridSearch {
   private static readonly RRF_K = 60;
 
@@ -14,23 +24,64 @@ export class HybridSearch {
     private readonly embedFn: EmbedFn,
   ) {}
 
+  /**
+   * Returns true when the `chunks` table exists (schema v3+). When absent we
+   * behave as if all vector ids are doc ids (pre-chunking behavior).
+   */
+  private chunksTableExists(): boolean {
+    const row = this.db.db
+      .prepare(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='chunks'",
+      )
+      .get();
+    return row !== undefined;
+  }
+
+  /**
+   * Given a vector-search id, return the `document_id` portion. Chunk ids
+   * follow the pattern `{doc_id}#c{index}` per Shared Constraint #6 of the
+   * GH-0761 plan. Legacy non-chunk ids pass through unchanged.
+   */
+  private docIdFromVecId(vecId: string): string {
+    const marker = vecId.lastIndexOf("#c");
+    if (marker === -1) return vecId;
+    const suffix = vecId.slice(marker + 2);
+    if (suffix.length === 0 || !/^\d+$/.test(suffix)) return vecId;
+    return vecId.slice(0, marker);
+  }
+
+  private fetchChunk(chunkId: string): ChunkRow | undefined {
+    if (!this.chunksTableExists()) return undefined;
+    return this.db.db
+      .prepare(
+        `SELECT id, document_id, chunk_index, char_start, char_end, context_prefix, content
+         FROM chunks WHERE id = ?`,
+      )
+      .get(chunkId) as ChunkRow | undefined;
+  }
+
   async search(
     query: string,
     options: SearchOptions = {},
   ): Promise<SearchResult[]> {
-    const { type, tags, includeSuperseded = false, limit = 20 } = options;
+    const { type, tags, includeSuperseded = false, limit = 20, memoryTier } = options;
 
-    // Run FTS and vector search
+    // Run FTS and vector search (FTS already applies memoryTier filter in SQL
+    // when the schema supports it).
     const ftsResults = this.fts.search(query, {
       includeSuperseded: true,
       limit: limit * 2,
+      memoryTier,
     });
 
     const queryEmbedding = await this.embedFn(query);
     const vecResults = this.vec.search(queryEmbedding, limit * 2);
 
-    // Build RRF score map
+    // Build RRF score map, keyed by document_id. When vec ids are chunk ids
+    // like `{doc}#c{n}`, we collapse to the parent doc for scoring but
+    // remember the best-scoring chunk id per doc for later meta enrichment.
     const scores = new Map<string, number>();
+    const bestChunkByDoc = new Map<string, { chunkId: string; rank: number }>();
 
     for (let i = 0; i < ftsResults.length; i++) {
       const id = ftsResults[i].id;
@@ -39,9 +90,16 @@ export class HybridSearch {
     }
 
     for (let i = 0; i < vecResults.length; i++) {
-      const id = vecResults[i].id;
+      const vecId = vecResults[i].id;
+      const docId = this.docIdFromVecId(vecId);
       const rrfScore = 1 / (HybridSearch.RRF_K + i + 1);
-      scores.set(id, (scores.get(id) ?? 0) + rrfScore);
+      scores.set(docId, (scores.get(docId) ?? 0) + rrfScore);
+      if (vecId !== docId) {
+        const existing = bestChunkByDoc.get(docId);
+        if (!existing || i < existing.rank) {
+          bestChunkByDoc.set(docId, { chunkId: vecId, rank: i });
+        }
+      }
     }
 
     // Build a lookup of FTS results by id for quick access
@@ -98,6 +156,31 @@ export class HybridSearch {
       });
     }
 
+    // Post-filter: memory_tier for vector-only hits that bypassed the FTS
+    // SQL filter. Also covers the case where the FTS stage returned 0 rows
+    // but vec returned chunks from a doc in another tier.
+    if (memoryTier && memoryTier !== "any") {
+      filtered = filtered.filter((r) => {
+        const tier = this.db.getMemoryTier(r.id);
+        // When column absent (v2 schema) treat as "doc"
+        return (tier ?? "doc") === memoryTier;
+      });
+    }
+
+    // Enrich with chunk meta when chunk data is available (best-scoring
+    // chunk per doc).
+    for (const r of filtered) {
+      const best = bestChunkByDoc.get(r.id);
+      if (!best) continue;
+      const chunk = this.fetchChunk(best.chunkId);
+      if (!chunk) continue;
+      r.bestChunkId = chunk.id;
+      r.chunkIndex = chunk.chunk_index;
+      r.charStart = chunk.char_start;
+      r.charEnd = chunk.char_end;
+      r.contextPrefix = chunk.context_prefix;
+    }
+
     return filtered.slice(0, limit);
   }
 }

package/src/ignore.ts

@@ -0,0 +1,82 @@
+import { readFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import ignorePkg, { type Ignore } from "ignore";
+
+// The `ignore` CJS module exposes the factory via `module.exports = factory`
+// with `factory.default = factory` attached. Under `NodeNext` + ESM, depending
+// on the interop mode, the default import can resolve to either the factory
+// itself or the whole namespace. Probe and pick the callable form.
+const ignore: (options?: { ignorecase?: boolean }) => Ignore = (
+  typeof (ignorePkg as unknown) === "function"
+    ? (ignorePkg as unknown as (options?: { ignorecase?: boolean }) => Ignore)
+    : ((ignorePkg as unknown as { default: (options?: { ignorecase?: boolean }) => Ignore }).default)
+);
+
+/**
+ * Default ignore patterns applied to every root even when no `.ralphignore`
+ * file or caller-supplied globals are provided. These target directories and
+ * files that should virtually never be indexed.
+ */
+export const DEFAULT_IGNORE_PATTERNS: string[] = [
+  ".claude/",
+  "node_modules/",
+  "dist/",
+  ".git/",
+  "*.log",
+];
+
+/**
+ * Opaque matcher returned by {@link loadIgnoreForRoot}. Given a path relative
+ * to the root used to construct the matcher, {@link isIgnored} reports whether
+ * the path should be skipped by the scanner.
+ */
+export interface IgnoreMatcher {
+  isIgnored(relativePath: string): boolean;
+}
+
+/**
+ * Build an {@link IgnoreMatcher} for a given root directory. The matcher
+ * combines (in order):
+ *   1. {@link DEFAULT_IGNORE_PATTERNS} — always applied.
+ *   2. `globalPatterns` — caller-supplied patterns (typically from
+ *      `knowledge.config.json`'s `ignorePatterns`).
+ *   3. Contents of `<rootDir>/.ralphignore`, if present.
+ *
+ * All patterns follow gitignore syntax via the `ignore` package.
+ *
+ * @param rootDir absolute path of the root being scanned
+ * @param globalPatterns optional extra patterns applied before the per-root
+ *   `.ralphignore` file
+ */
+export function loadIgnoreForRoot(
+  rootDir: string,
+  globalPatterns?: string[],
+): IgnoreMatcher {
+  const ign: Ignore = ignore();
+  ign.add(DEFAULT_IGNORE_PATTERNS);
+  if (globalPatterns && globalPatterns.length > 0) {
+    ign.add(globalPatterns);
+  }
+
+  const ralphIgnorePath = join(rootDir, ".ralphignore");
+  if (existsSync(ralphIgnorePath)) {
+    try {
+      const contents = readFileSync(ralphIgnorePath, "utf-8");
+      ign.add(contents);
+    } catch (e) {
+      console.warn(
+        `Failed to read .ralphignore at ${ralphIgnorePath}: ${(e as Error).message}`,
+      );
+    }
+  }
+
+  return {
+    isIgnored(relativePath: string): boolean {
+      if (!relativePath) return false;
+      // `ignore` package requires forward-slash paths with no leading slash.
+      const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
+      if (!normalized) return false;
+      return ign.ignores(normalized);
+    },
+  };
+}