readthemanual 0.0.1

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "readthemanual",
+  "version": "0.0.1",
+  "description": "Read the Manual — fast local full-text search for any project's docs",
+  "main": "dist/cli.js",
+  "bin": {
+    "rtm": "dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "dependencies": {
+    "@hono/node-server": "^1.19.9",
+    "@modelcontextprotocol/sdk": "^1.27.1",
+    "better-sqlite3": "^12.6.2",
+    "commander": "^14.0.3",
+    "hono": "^4.12.3",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.3.3",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { ingest } from "./ingest";
+import { startServer } from "./server";
+import { startMcp } from "./mcp";
+import { openDb, search, getDocument, listDocuments } from "./db";
+import {
+  loadConfig, getConfigPath, getDbPath, getDefaultSource,
+  addSource, removeSource, setLastSource, isGlobal,
+} from "./config";
+import path from "path";
+
+const config = loadConfig();
+const defaultDb = getDbPath(config);
+
+const program = new Command();
+
+program
+  .name("rtm")
+  .description("Read the Manual — fast local full-text search for documentation")
+  .version("0.0.1");
+
+program
+  .command("ingest")
+  .description("Fetch and index docs from configured GitHub sources")
+  .option("-d, --db <path>", "SQLite database path", defaultDb)
+  .option("-s, --source <name>", "Ingest only this source")
+  .option("--all-langs", "Index all languages, not just English", false)
+  .option("-v, --verbose", "Verbose output", false)
+  .action(async (opts) => {
+    const result = await ingest({
+      dbPath: opts.db,
+      source: opts.source,
+      allLangs: opts.allLangs,
+      verbose: opts.verbose,
+    });
+    // Update last source if a specific one was ingested
+    if (opts.source) {
+      setLastSource(opts.source);
+    }
+    console.log(
+      `Indexed ${result.filesProcessed} files (${result.sectionsInserted} sections).`
+    );
+  });
+
+program
+  .command("serve")
+  .description("Start the JSON-RPC HTTP server")
+  .option("-d, --db <path>", "SQLite database path", defaultDb)
+  .option("-p, --port <number>", "Port to listen on", "3777")
+  .action((opts) => {
+    startServer({ dbPath: opts.db, port: parseInt(opts.port, 10) });
+  });
+
+program
+  .command("mcp")
+  .description("Start as an MCP server over stdio")
+  .option("-d, --db <path>", "SQLite database path", defaultDb)
+  .action((opts) => {
+    startMcp({ dbPath: opts.db });
+  });
+
+program
+  .command("search <query>")
+  .description("Search the docs index")
+  .option("-d, --db <path>", "SQLite database path", defaultDb)
+  .option("-s, --source <name>", "Search only this source")
+  .option("-l, --lang <language>", "Filter by language")
+  .option("-n, --limit <number>", "Max results", "10")
+  .option("--json", "Output raw JSON", false)
+  .action((query, opts) => {
+    const cfg = loadConfig();
+    const db = openDb(opts.db);
+
+    // Determine source scope: explicit flag > lastSource > all
+    const sourceName = opts.source ?? getDefaultSource(cfg);
+
+    const { results, total } = search(db, query, {
+      language: opts.lang,
+      limit: parseInt(opts.limit, 10),
+      sourcePrefix: sourceName ? `${sourceName}/` : undefined,
+    });
+    db.close();
+    if (opts.json) {
+      console.log(JSON.stringify({ results, total }, null, 2));
+      return;
+    }
+    if (results.length === 0) {
+      console.log("No results found.");
+      return;
+    }
+    for (const r of results) {
+      const fullPath = path.join(cfg.dataDir, r.file_path);
+      const snippet = r.snippet.replace(/<\/?mark>/g, (m) => m === "<mark>" ? "\x1b[1;33m" : "\x1b[0m");
+      console.log(`\x1b[36m[${r.id}]\x1b[0m \x1b[1m${r.heading}\x1b[0m`);
+      console.log(`     ${fullPath} (${r.language})`);
+      console.log(`     ${snippet}`);
+      console.log();
+    }
+    if (sourceName) {
+      console.log(`${results.length} of ${total} match(es) [source: ${sourceName}]`);
+    } else {
+      console.log(`${results.length} of ${total} match(es)`);
+    }
+  });
+
+program
+  .command("get <id>")
+  .description("Get a document section by ID")
+  .option("-d, --db <path>", "SQLite database path", defaultDb)
+  .option("--json", "Output raw JSON", false)
+  .action((id, opts) => {
+    const db = openDb(opts.db);
+    const doc = getDocument(db, parseInt(id, 10));
+    db.close();
+    if (!doc) {
+      console.error(`Document ${id} not found.`);
+      process.exit(1);
+    }
+    if (opts.json) {
+      console.log(JSON.stringify(doc, null, 2));
+      return;
+    }
+    const cfg = loadConfig();
+    const fullPath = path.join(cfg.dataDir, doc.file_path);
+    console.log(`\x1b[1m${doc.heading}\x1b[0m`);
+    console.log(`${fullPath} (${doc.language})`);
+    console.log();
+    console.log(doc.content);
+  });
+
+program
+  .command("list")
+  .description("List indexed documents")
+  .option("-d, --db <path>", "SQLite database path", defaultDb)
+  .option("-l, --lang <language>", "Filter by language")
+  .option("-n, --limit <number>", "Max results", "50")
+  .option("-o, --offset <number>", "Offset", "0")
+  .option("--json", "Output raw JSON", false)
+  .action((opts) => {
+    const db = openDb(opts.db);
+    const result = listDocuments(db, {
+      language: opts.lang,
+      limit: parseInt(opts.limit, 10),
+      offset: parseInt(opts.offset, 10),
+    });
+    db.close();
+    if (opts.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    const cfg = loadConfig();
+    for (const d of result.docs) {
+      const fullPath = path.join(cfg.dataDir, d.file_path);
+      console.log(`\x1b[36m[${d.id}]\x1b[0m ${fullPath} — ${d.heading} (${d.language})`);
+    }
+    console.log(`\nShowing ${result.docs.length} of ${result.total} total sections`);
+  });
+
+program
+  .command("add <name> <repo>")
+  .description("Add a GitHub repo as a doc source (repo = owner/repo)")
+  .option("-b, --branch <branch>", "Branch to index", "main")
+  .option("-p, --prefix <prefix>", "Path prefix for docs in the repo", "docs/")
+  .action((name, repo, opts) => {
+    addSource({ name, repo, branch: opts.branch, docsPrefix: opts.prefix });
+    console.log(`Added source "${name}" → ${repo}@${opts.branch} (prefix: ${opts.prefix})`);
+    console.log(`Run \`rtm ingest -s ${name}\` to index it.`);
+  });
+
+program
+  .command("remove <name>")
+  .description("Remove a doc source")
+  .action((name) => {
+    if (removeSource(name)) {
+      console.log(`Removed source "${name}".`);
+    } else {
+      console.error(`Source "${name}" not found.`);
+      process.exit(1);
+    }
+  });
+
+program
+  .command("use <name>")
+  .description("Set the default source for searches")
+  .action((name) => {
+    const cfg = loadConfig();
+    if (!cfg.sources.some((s) => s.name === name)) {
+      const available = cfg.sources.map((s) => s.name).join(", ");
+      console.error(`Source "${name}" not found. Available: ${available}`);
+      process.exit(1);
+    }
+    setLastSource(name);
+    console.log(`Default source set to "${name}".`);
+  });
+
+program
+  .command("config")
+  .description("Show config file path and contents")
+  .action(() => {
+    const cfg = loadConfig();
+    console.log(`Config:  ${getConfigPath()}`);
+    console.log(`Data:    ${cfg.dataDir}`);
+    console.log(`Install: ${isGlobal() ? "global (/usr/local)" : "local (~/.local)"}`);
+    console.log(`Default: ${getDefaultSource(cfg) ?? "(none)"}`);
+    console.log();
+    console.log(`Sources (${cfg.sources.length}):`);
+    for (const s of cfg.sources) {
+      const marker = s.name === getDefaultSource(cfg) ? " *" : "";
+      console.log(`  - ${s.name}: ${s.repo}@${s.branch ?? "main"} (prefix: ${s.docsPrefix ?? "docs/"})${marker}`);
+    }
+  });
+
+// Default: bare args with no subcommand → search
+const knownCommands = ["ingest", "serve", "mcp", "search", "get", "list", "add", "remove", "use", "config", "help"];
+const firstArg = process.argv[2];
+if (firstArg && !firstArg.startsWith("-") && !knownCommands.includes(firstArg)) {
+  process.argv.splice(2, 0, "search");
+}
+
+program.parse();

package/src/config.ts

@@ -0,0 +1,143 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+function isGlobalInstall(): boolean {
+  return __dirname.startsWith("/usr/local");
+}
+
+// Data always lives in user's home — it's user data, not system data
+const DATA_DIR = path.join(os.homedir(), ".local", "rtm");
+const CONFIG_PATH = path.join(DATA_DIR, "config.json");
+
+export interface SourceConfig {
+  name: string;
+  repo: string;
+  branch?: string;
+  docsPrefix?: string;
+}
+
+export interface Config {
+  sources: SourceConfig[];
+  dataDir: string;
+  lastSource?: string;
+}
+
+const DEFAULT_CONFIG: Config = {
+  sources: [
+    {
+      name: "zeroclaw",
+      repo: "zeroclaw-labs/zeroclaw",
+      branch: "main",
+      docsPrefix: "docs/",
+    },
+  ],
+  dataDir: DATA_DIR,
+  lastSource: "zeroclaw",
+};
+
+export function getConfigPath(): string {
+  return CONFIG_PATH;
+}
+
+export function isGlobal(): boolean {
+  return isGlobalInstall();
+}
+
+function hasWriteAccess(dir: string): boolean {
+  // Walk up to find the first existing ancestor and check write permission
+  let check = dir;
+  while (!fs.existsSync(check)) {
+    const parent = path.dirname(check);
+    if (parent === check) return false;
+    check = parent;
+  }
+  try {
+    fs.accessSync(check, fs.constants.W_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function checkWriteAccess(): void {
+  if (!hasWriteAccess(DATA_DIR)) {
+    console.error(`Error: No write permission to ${DATA_DIR}\nCheck directory permissions.`);
+    process.exit(1);
+  }
+}
+
+export function loadConfig(): Config {
+  if (!fs.existsSync(CONFIG_PATH)) {
+    // Return defaults in memory; only write if we have access
+    if (hasWriteAccess(DATA_DIR) || hasWriteAccess(path.dirname(DATA_DIR))) {
+      fs.mkdirSync(DATA_DIR, { recursive: true });
+      fs.writeFileSync(CONFIG_PATH, JSON.stringify(DEFAULT_CONFIG, null, 2) + "\n", "utf-8");
+    }
+    return DEFAULT_CONFIG;
+  }
+  const raw = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf-8"));
+  return {
+    sources: raw.sources ?? DEFAULT_CONFIG.sources,
+    dataDir: raw.dataDir ?? DATA_DIR,
+    lastSource: raw.lastSource,
+  };
+}
+
+export function getDbPath(config: Config): string {
+  return path.join(config.dataDir, "rtm.db");
+}
+
+export function getDocsDir(config: Config, sourceName: string): string {
+  return path.join(config.dataDir, sourceName);
+}
+
+function saveConfig(config: Config): void {
+  checkWriteAccess();
+  fs.mkdirSync(path.dirname(CONFIG_PATH), { recursive: true });
+  fs.writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2) + "\n", "utf-8");
+}
+
+export function addSource(source: SourceConfig): void {
+  const config = loadConfig();
+  const existing = config.sources.findIndex((s) => s.name === source.name);
+  if (existing >= 0) {
+    config.sources[existing] = source;
+  } else {
+    config.sources.push(source);
+  }
+  config.lastSource = source.name;
+  saveConfig(config);
+}
+
+export function removeSource(name: string): boolean {
+  const config = loadConfig();
+  const before = config.sources.length;
+  config.sources = config.sources.filter((s) => s.name !== name);
+  if (config.sources.length === before) return false;
+  // If we removed the last-used source, reset to the final remaining one
+  if (config.lastSource === name) {
+    config.lastSource = config.sources.length > 0
+      ? config.sources[config.sources.length - 1].name
+      : undefined;
+  }
+  saveConfig(config);
+  return true;
+}
+
+export function setLastSource(name: string): void {
+  const config = loadConfig();
+  config.lastSource = name;
+  saveConfig(config);
+}
+
+export function getDefaultSource(config: Config): string | undefined {
+  // Last-used source, or the last one in the list, or undefined
+  if (config.lastSource && config.sources.some((s) => s.name === config.lastSource)) {
+    return config.lastSource;
+  }
+  if (config.sources.length > 0) {
+    return config.sources[config.sources.length - 1].name;
+  }
+  return undefined;
+}

package/src/db.ts

@@ -0,0 +1,207 @@
+import Database from "better-sqlite3";
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+const DEFAULT_DB_PATH = path.join(os.homedir(), ".local", "rtm", "rtm.db");
+
+export interface DocSection {
+  id: number;
+  file_path: string;
+  heading: string;
+  content: string;
+  language: string;
+}
+
+export interface SearchResult extends DocSection {
+  rank: number;
+  snippet: string;
+}
+
+export function openDb(dbPath: string = DEFAULT_DB_PATH): Database.Database {
+  const dir = path.dirname(dbPath);
+  if (!fs.existsSync(dir)) {
+    try {
+      fs.mkdirSync(dir, { recursive: true });
+    } catch (err: any) {
+      if (err.code === "EACCES") {
+        console.error(
+          `Error: No write permission to ${dir}\n` +
+          `If installed globally, run with sudo or install locally.`
+        );
+        process.exit(1);
+      }
+      throw err;
+    }
+  }
+  const db = new Database(dbPath);
+  db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
+  initSchema(db);
+  return db;
+}
+
+function initSchema(db: Database.Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS docs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      file_path TEXT NOT NULL,
+      heading TEXT NOT NULL,
+      content TEXT NOT NULL,
+      language TEXT NOT NULL DEFAULT 'en'
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_docs_path ON docs(file_path);
+    CREATE INDEX IF NOT EXISTS idx_docs_lang ON docs(language);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(
+      heading,
+      content,
+      content='docs',
+      content_rowid='id',
+      tokenize='porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs BEGIN
+      INSERT INTO docs_fts(rowid, heading, content)
+      VALUES (new.id, new.heading, new.content);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS docs_ad AFTER DELETE ON docs BEGIN
+      INSERT INTO docs_fts(docs_fts, rowid, heading, content)
+      VALUES ('delete', old.id, old.heading, old.content);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS docs_au AFTER UPDATE ON docs BEGIN
+      INSERT INTO docs_fts(docs_fts, rowid, heading, content)
+      VALUES ('delete', old.id, old.heading, old.content);
+      INSERT INTO docs_fts(rowid, heading, content)
+      VALUES (new.id, new.heading, new.content);
+    END;
+  `);
+}
+
+export function clearDocs(db: Database.Database): void {
+  db.exec("DELETE FROM docs");
+}
+
+export function insertSection(
+  db: Database.Database,
+  section: Omit<DocSection, "id">
+): number {
+  const stmt = db.prepare(
+    "INSERT INTO docs (file_path, heading, content, language) VALUES (?, ?, ?, ?)"
+  );
+  const result = stmt.run(
+    section.file_path,
+    section.heading,
+    section.content,
+    section.language
+  );
+  return Number(result.lastInsertRowid);
+}
+
+export interface SearchResponse {
+  results: SearchResult[];
+  total: number;
+}
+
+function sanitizeFtsQuery(query: string): string {
+  const tokens = query
+    .replace(/"/g, "")
+    .split(/\s+/)
+    .filter(Boolean);
+  if (tokens.length === 0) return '""';
+  return tokens.map((t) => `"${t}"`).join(" ");
+}
+
+export function search(
+  db: Database.Database,
+  query: string,
+  opts: { language?: string; limit?: number; sourcePrefix?: string } = {}
+): SearchResponse {
+  const limit = opts.limit ?? 20;
+  const ftsQuery = sanitizeFtsQuery(query);
+
+  let countSql = `
+    SELECT COUNT(*) as total
+    FROM docs_fts
+    JOIN docs d ON d.id = docs_fts.rowid
+    WHERE docs_fts MATCH ?
+  `;
+  let sql = `
+    SELECT
+      d.id, d.file_path, d.heading, d.content, d.language,
+      docs_fts.rank AS rank,
+      snippet(docs_fts, 1, '<mark>', '</mark>', '...', 40) AS snippet
+    FROM docs_fts
+    JOIN docs d ON d.id = docs_fts.rowid
+    WHERE docs_fts MATCH ?
+  `;
+  const params: (string | number)[] = [ftsQuery];
+  const countParams: (string | number)[] = [ftsQuery];
+
+  if (opts.sourcePrefix) {
+    countSql += " AND d.file_path LIKE ?";
+    sql += " AND d.file_path LIKE ?";
+    params.push(opts.sourcePrefix + "%");
+    countParams.push(opts.sourcePrefix + "%");
+  }
+
+  if (opts.language) {
+    countSql += " AND d.language = ?";
+    sql += " AND d.language = ?";
+    params.push(opts.language);
+    countParams.push(opts.language);
+  }
+
+  sql += " ORDER BY docs_fts.rank LIMIT ?";
+  params.push(limit);
+
+  const { total } = db.prepare(countSql).get(...countParams) as { total: number };
+  const results = db.prepare(sql).all(...params) as SearchResult[];
+
+  return { results, total };
+}
+
+export function getDocument(
+  db: Database.Database,
+  id: number
+): DocSection | null {
+  return (
+    (db.prepare("SELECT * FROM docs WHERE id = ?").get(id) as
+      | DocSection
+      | undefined) ?? null
+  );
+}
+
+export function listDocuments(
+  db: Database.Database,
+  opts: { language?: string; limit?: number; offset?: number } = {}
+): { docs: Pick<DocSection, "id" | "file_path" | "heading" | "language">[]; total: number } {
+  const limit = opts.limit ?? 100;
+  const offset = opts.offset ?? 0;
+
+  let countSql = "SELECT COUNT(*) as total FROM docs";
+  let sql = "SELECT id, file_path, heading, language FROM docs";
+  const params: (string | number)[] = [];
+  const countParams: string[] = [];
+
+  if (opts.language) {
+    countSql += " WHERE language = ?";
+    sql += " WHERE language = ?";
+    params.push(opts.language);
+    countParams.push(opts.language);
+  }
+
+  sql += " ORDER BY file_path, id LIMIT ? OFFSET ?";
+  params.push(limit, offset);
+
+  const { total } = db.prepare(countSql).get(...countParams) as { total: number };
+  const docs = db.prepare(sql).all(...params) as Pick<
+    DocSection,
+    "id" | "file_path" | "heading" | "language"
+  >[];
+
+  return { docs, total };
+}