readthemanual 0.0.1

package/docs-search/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "docs-search",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/cli.js",
+  "bin": {
+    "docs-search": "dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "dependencies": {
+    "@hono/node-server": "^1.19.9",
+    "better-sqlite3": "^12.6.2",
+    "commander": "^14.0.3",
+    "hono": "^4.12.3"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.3.3",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}

package/docs-search/src/cli.ts

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { ingest } from "./ingest";
+import { startServer } from "./server";
+import { openDb, search, getDocument, listDocuments, DEFAULT_DB_PATH } from "./db";
+
+const program = new Command();
+
+program
+  .name("docs-search")
+  .description("SQLite FTS5 search service for ZeroClaw docs")
+  .version("1.0.0");
+
+program
+  .command("ingest")
+  .description("Fetch and index ZeroClaw docs from GitHub")
+  .option("-d, --db <path>", "SQLite database path", DEFAULT_DB_PATH)
+  .option("--all-langs", "Index all languages, not just English", false)
+  .option("-v, --verbose", "Verbose output", false)
+  .action(async (opts) => {
+    const result = await ingest({
+      dbPath: opts.db,
+      allLangs: opts.allLangs,
+      verbose: opts.verbose,
+    });
+    console.log(
+      `Indexed ${result.filesProcessed} files (${result.sectionsInserted} sections).`
+    );
+  });
+
+program
+  .command("serve")
+  .description("Start the JSON-RPC HTTP server")
+  .option("-d, --db <path>", "SQLite database path", DEFAULT_DB_PATH)
+  .option("-p, --port <number>", "Port to listen on", "3777")
+  .action((opts) => {
+    startServer({ dbPath: opts.db, port: parseInt(opts.port, 10) });
+  });
+
+program
+  .command("search <query>")
+  .description("Search the docs index")
+  .option("-d, --db <path>", "SQLite database path", DEFAULT_DB_PATH)
+  .option("-l, --lang <language>", "Filter by language")
+  .option("-n, --limit <number>", "Max results", "10")
+  .option("--json", "Output raw JSON", false)
+  .action((query, opts) => {
+    const db = openDb(opts.db);
+    const { results, total } = search(db, query, {
+      language: opts.lang,
+      limit: parseInt(opts.limit, 10),
+    });
+    db.close();
+    if (opts.json) {
+      console.log(JSON.stringify({ results, total }, null, 2));
+      return;
+    }
+    if (results.length === 0) {
+      console.log("No results found.");
+      return;
+    }
+    for (const r of results) {
+      const snippet = r.snippet.replace(/<\/?mark>/g, (m) => m === "<mark>" ? "\x1b[1;33m" : "\x1b[0m");
+      console.log(`\x1b[36m[${r.id}]\x1b[0m \x1b[1m${r.heading}\x1b[0m`);
+      console.log(`     ${r.file_path} (${r.language})`);
+      console.log(`     ${snippet}`);
+      console.log();
+    }
+    console.log(`${results.length} of ${total} match(es)`);
+  });
+
+program
+  .command("get <id>")
+  .description("Get a document section by ID")
+  .option("-d, --db <path>", "SQLite database path", DEFAULT_DB_PATH)
+  .option("--json", "Output raw JSON", false)
+  .action((id, opts) => {
+    const db = openDb(opts.db);
+    const doc = getDocument(db, parseInt(id, 10));
+    db.close();
+    if (!doc) {
+      console.error(`Document ${id} not found.`);
+      process.exit(1);
+    }
+    if (opts.json) {
+      console.log(JSON.stringify(doc, null, 2));
+      return;
+    }
+    console.log(`\x1b[1m${doc.heading}\x1b[0m`);
+    console.log(`${doc.file_path} (${doc.language})`);
+    console.log();
+    console.log(doc.content);
+  });
+
+program
+  .command("list")
+  .description("List indexed documents")
+  .option("-d, --db <path>", "SQLite database path", DEFAULT_DB_PATH)
+  .option("-l, --lang <language>", "Filter by language")
+  .option("-n, --limit <number>", "Max results", "50")
+  .option("-o, --offset <number>", "Offset", "0")
+  .option("--json", "Output raw JSON", false)
+  .action((opts) => {
+    const db = openDb(opts.db);
+    const result = listDocuments(db, {
+      language: opts.lang,
+      limit: parseInt(opts.limit, 10),
+      offset: parseInt(opts.offset, 10),
+    });
+    db.close();
+    if (opts.json) {
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+    for (const d of result.docs) {
+      console.log(`\x1b[36m[${d.id}]\x1b[0m ${d.file_path} — ${d.heading} (${d.language})`);
+    }
+    console.log(`\nShowing ${result.docs.length} of ${result.total} total sections`);
+  });
+
+// Default: bare args with no subcommand → search
+const knownCommands = ["ingest", "serve", "search", "get", "list", "help"];
+const firstArg = process.argv[2];
+if (firstArg && !firstArg.startsWith("-") && !knownCommands.includes(firstArg)) {
+  // Rewrite argv to inject "search" before the query
+  process.argv.splice(2, 0, "search");
+}
+
+program.parse();

package/docs-search/src/db.ts

@@ -0,0 +1,192 @@
+import Database from "better-sqlite3";
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+const DEFAULT_DB_DIR = path.join(os.homedir(), ".local", "docs-search");
+const DEFAULT_DB_PATH = path.join(DEFAULT_DB_DIR, "docs-search.db");
+
+export interface DocSection {
+  id: number;
+  file_path: string;
+  heading: string;
+  content: string;
+  language: string;
+}
+
+export interface SearchResult extends DocSection {
+  rank: number;
+  snippet: string;
+}
+
+export { DEFAULT_DB_PATH };
+
+export function openDb(dbPath: string = DEFAULT_DB_PATH): Database.Database {
+  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+  const db = new Database(dbPath);
+  db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
+  initSchema(db);
+  return db;
+}
+
+function initSchema(db: Database.Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS docs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      file_path TEXT NOT NULL,
+      heading TEXT NOT NULL,
+      content TEXT NOT NULL,
+      language TEXT NOT NULL DEFAULT 'en'
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_docs_path ON docs(file_path);
+    CREATE INDEX IF NOT EXISTS idx_docs_lang ON docs(language);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(
+      heading,
+      content,
+      content='docs',
+      content_rowid='id',
+      tokenize='porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs BEGIN
+      INSERT INTO docs_fts(rowid, heading, content)
+      VALUES (new.id, new.heading, new.content);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS docs_ad AFTER DELETE ON docs BEGIN
+      INSERT INTO docs_fts(docs_fts, rowid, heading, content)
+      VALUES ('delete', old.id, old.heading, old.content);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS docs_au AFTER UPDATE ON docs BEGIN
+      INSERT INTO docs_fts(docs_fts, rowid, heading, content)
+      VALUES ('delete', old.id, old.heading, old.content);
+      INSERT INTO docs_fts(rowid, heading, content)
+      VALUES (new.id, new.heading, new.content);
+    END;
+  `);
+}
+
+export function clearDocs(db: Database.Database): void {
+  db.exec("DELETE FROM docs");
+}
+
+export function insertSection(
+  db: Database.Database,
+  section: Omit<DocSection, "id">
+): number {
+  const stmt = db.prepare(
+    "INSERT INTO docs (file_path, heading, content, language) VALUES (?, ?, ?, ?)"
+  );
+  const result = stmt.run(
+    section.file_path,
+    section.heading,
+    section.content,
+    section.language
+  );
+  return Number(result.lastInsertRowid);
+}
+
+export interface SearchResponse {
+  results: SearchResult[];
+  total: number;
+}
+
+function sanitizeFtsQuery(query: string): string {
+  // Split into tokens and wrap each in double quotes so FTS5 treats them as
+  // literals (handles dashes, slashes, backslashes, etc.).  Any embedded
+  // double-quote characters are removed to avoid breaking the quoting.
+  const tokens = query
+    .replace(/"/g, "")
+    .split(/\s+/)
+    .filter(Boolean);
+  if (tokens.length === 0) return '""';
+  return tokens.map((t) => `"${t}"`).join(" ");
+}
+
+export function search(
+  db: Database.Database,
+  query: string,
+  opts: { language?: string; limit?: number } = {}
+): SearchResponse {
+  const limit = opts.limit ?? 20;
+  const ftsQuery = sanitizeFtsQuery(query);
+
+  let countSql = `
+    SELECT COUNT(*) as total
+    FROM docs_fts
+    JOIN docs d ON d.id = docs_fts.rowid
+    WHERE docs_fts MATCH ?
+  `;
+  let sql = `
+    SELECT
+      d.id, d.file_path, d.heading, d.content, d.language,
+      docs_fts.rank AS rank,
+      snippet(docs_fts, 1, '<mark>', '</mark>', '...', 40) AS snippet
+    FROM docs_fts
+    JOIN docs d ON d.id = docs_fts.rowid
+    WHERE docs_fts MATCH ?
+  `;
+  const params: (string | number)[] = [ftsQuery];
+  const countParams: (string | number)[] = [ftsQuery];
+
+  if (opts.language) {
+    countSql += " AND d.language = ?";
+    sql += " AND d.language = ?";
+    params.push(opts.language);
+    countParams.push(opts.language);
+  }
+
+  sql += " ORDER BY docs_fts.rank LIMIT ?";
+  params.push(limit);
+
+  const { total } = db.prepare(countSql).get(...countParams) as { total: number };
+  const results = db.prepare(sql).all(...params) as SearchResult[];
+
+  return { results, total };
+}
+
+export function getDocument(
+  db: Database.Database,
+  id: number
+): DocSection | null {
+  return (
+    (db.prepare("SELECT * FROM docs WHERE id = ?").get(id) as
+      | DocSection
+      | undefined) ?? null
+  );
+}
+
+export function listDocuments(
+  db: Database.Database,
+  opts: { language?: string; limit?: number; offset?: number } = {}
+): { docs: Pick<DocSection, "id" | "file_path" | "heading" | "language">[]; total: number } {
+  const limit = opts.limit ?? 100;
+  const offset = opts.offset ?? 0;
+
+  let countSql = "SELECT COUNT(*) as total FROM docs";
+  let sql = "SELECT id, file_path, heading, language FROM docs";
+  const params: (string | number)[] = [];
+  const countParams: string[] = [];
+
+  if (opts.language) {
+    countSql += " WHERE language = ?";
+    sql += " WHERE language = ?";
+    params.push(opts.language);
+    countParams.push(opts.language);
+  }
+
+  sql += " ORDER BY file_path, id LIMIT ? OFFSET ?";
+  params.push(limit, offset);
+
+  const { total } = db.prepare(countSql).get(...countParams) as { total: number };
+  const docs = db.prepare(sql).all(...params) as Pick<
+    DocSection,
+    "id" | "file_path" | "heading" | "language"
+  >[];
+
+  return { docs, total };
+}

package/docs-search/src/ingest.ts

@@ -0,0 +1,168 @@
+import { openDb, clearDocs, insertSection } from "./db";
+import type Database from "better-sqlite3";
+import fs from "fs";
+import os from "os";
+import path from "path";
+
+const DOCS_DIR = path.join(os.homedir(), ".local", "docs-search", "docs");
+
+const REPO = "zeroclaw-labs/zeroclaw";
+const BRANCH = "main";
+const DOCS_PREFIX = "docs/";
+
+interface TreeEntry {
+  path: string;
+  type: string;
+  url: string;
+}
+
+async function fetchTree(): Promise<TreeEntry[]> {
+  const url = `https://api.github.com/repos/${REPO}/git/trees/${BRANCH}?recursive=1`;
+  const res = await fetch(url, {
+    headers: {
+      Accept: "application/vnd.github.v3+json",
+      ...(process.env.GITHUB_TOKEN
+        ? { Authorization: `token ${process.env.GITHUB_TOKEN}` }
+        : {}),
+    },
+  });
+  if (!res.ok) throw new Error(`GitHub tree API: ${res.status} ${res.statusText}`);
+  const data = (await res.json()) as { tree: TreeEntry[] };
+  return data.tree;
+}
+
+async function fetchRawFile(filePath: string): Promise<string> {
+  const url = `https://raw.githubusercontent.com/${REPO}/${BRANCH}/${filePath}`;
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Failed to fetch ${filePath}: ${res.status}`);
+  return res.text();
+}
+
+function detectLanguage(filePath: string): string {
+  const match = filePath.match(/^docs\/i18n\/([a-z]{2}(?:-[A-Z]{2})?)\//);
+  if (match) return match[1];
+  return "en";
+}
+
+interface Section {
+  heading: string;
+  content: string;
+}
+
+function splitByHeadings(markdown: string): Section[] {
+  const lines = markdown.split("\n");
+  const sections: Section[] = [];
+  let currentHeading = "(intro)";
+  let currentLines: string[] = [];
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,3})\s+(.+)/);
+    if (headingMatch) {
+      if (currentLines.length > 0) {
+        const content = currentLines.join("\n").trim();
+        if (content.length > 0) {
+          sections.push({ heading: currentHeading, content });
+        }
+      }
+      currentHeading = headingMatch[2];
+      currentLines = [];
+    } else {
+      currentLines.push(line);
+    }
+  }
+
+  if (currentLines.length > 0) {
+    const content = currentLines.join("\n").trim();
+    if (content.length > 0) {
+      sections.push({ heading: currentHeading, content });
+    }
+  }
+
+  return sections;
+}
+
+export async function ingest(opts: {
+  dbPath?: string;
+  allLangs?: boolean;
+  verbose?: boolean;
+}): Promise<{ filesProcessed: number; sectionsInserted: number }> {
+  const db = openDb(opts.dbPath);
+  const log = opts.verbose ? console.log : () => {};
+
+  log("Fetching file tree from GitHub...");
+  const tree = await fetchTree();
+
+  const mdFiles = tree.filter((entry) => {
+    if (entry.type !== "blob") return false;
+    if (!entry.path.startsWith(DOCS_PREFIX)) return false;
+    if (!entry.path.endsWith(".md")) return false;
+    // Skip images and non-text
+    if (/\.(jpg|png|svg|gif)$/i.test(entry.path)) return false;
+    // Filter by language
+    if (!opts.allLangs) {
+      const lang = detectLanguage(entry.path);
+      if (lang !== "en") return false;
+    }
+    return true;
+  });
+
+  log(`Found ${mdFiles.length} markdown files to index.`);
+  log(`Saving docs to ${DOCS_DIR}`);
+
+  // Clean and recreate local docs directory
+  fs.rmSync(DOCS_DIR, { recursive: true, force: true });
+  fs.mkdirSync(DOCS_DIR, { recursive: true });
+
+  clearDocs(db);
+
+  const insertMany = db.transaction(
+    (sections: { file_path: string; heading: string; content: string; language: string }[]) => {
+      for (const s of sections) {
+        insertSection(db, s);
+      }
+    }
+  );
+
+  let filesProcessed = 0;
+  let sectionsInserted = 0;
+  const allSections: { file_path: string; heading: string; content: string; language: string }[] = [];
+
+  // Fetch in batches of 10 to avoid rate limits
+  const batchSize = 10;
+  for (let i = 0; i < mdFiles.length; i += batchSize) {
+    const batch = mdFiles.slice(i, i + batchSize);
+    const results = await Promise.all(
+      batch.map(async (entry) => {
+        const content = await fetchRawFile(entry.path);
+        return { path: entry.path, content };
+      })
+    );
+
+    for (const { path: filePath, content } of results) {
+      // Save raw markdown to ~/.local/docs-search/docs/
+      const localPath = path.join(DOCS_DIR, filePath);
+      fs.mkdirSync(path.dirname(localPath), { recursive: true });
+      fs.writeFileSync(localPath, content, "utf-8");
+
+      const lang = detectLanguage(filePath);
+      const sections = splitByHeadings(content);
+      for (const section of sections) {
+        allSections.push({
+          file_path: filePath,
+          heading: section.heading,
+          content: section.content,
+          language: lang,
+        });
+      }
+      filesProcessed++;
+      log(`  [${filesProcessed}/${mdFiles.length}] ${filePath} (${sections.length} sections)`);
+    }
+  }
+
+  insertMany(allSections);
+  sectionsInserted = allSections.length;
+
+  db.close();
+  log(`Done. ${filesProcessed} files, ${sectionsInserted} sections indexed.`);
+  return { filesProcessed, sectionsInserted };
+}

package/docs-search/src/server.ts

@@ -0,0 +1,145 @@
+import { Hono } from "hono";
+import { serve } from "@hono/node-server";
+import { openDb, search, getDocument, listDocuments } from "./db";
+import type Database from "better-sqlite3";
+
+interface JsonRpcRequest {
+  jsonrpc: "2.0";
+  id: string | number | null;
+  method: string;
+  params?: Record<string, unknown>;
+}
+
+interface JsonRpcResponse {
+  jsonrpc: "2.0";
+  id: string | number | null;
+  result?: unknown;
+  error?: { code: number; message: string; data?: unknown };
+}
+
+function rpcError(
+  id: string | number | null,
+  code: number,
+  message: string
+): JsonRpcResponse {
+  return { jsonrpc: "2.0", id, error: { code, message } };
+}
+
+function rpcResult(id: string | number | null, result: unknown): JsonRpcResponse {
+  return { jsonrpc: "2.0", id, result };
+}
+
+function handleRpc(db: Database.Database, req: JsonRpcRequest): JsonRpcResponse {
+  const { id, method, params } = req;
+
+  switch (method) {
+    case "search": {
+      const query = params?.query;
+      if (typeof query !== "string" || query.trim().length === 0) {
+        return rpcError(id, -32602, "params.query is required and must be a non-empty string");
+      }
+      const language = typeof params?.language === "string" ? params.language : undefined;
+      const limit = typeof params?.limit === "number" ? params.limit : undefined;
+      const { results, total } = search(db, query, { language, limit });
+      return rpcResult(id, { results, total, count: results.length });
+    }
+
+    case "get_document": {
+      const docId = params?.id;
+      if (typeof docId !== "number") {
+        return rpcError(id, -32602, "params.id is required and must be a number");
+      }
+      const doc = getDocument(db, docId);
+      if (!doc) {
+        return rpcError(id, -32602, `Document with id ${docId} not found`);
+      }
+      return rpcResult(id, doc);
+    }
+
+    case "list_documents": {
+      const language = typeof params?.language === "string" ? params.language : undefined;
+      const limit = typeof params?.limit === "number" ? params.limit : undefined;
+      const offset = typeof params?.offset === "number" ? params.offset : undefined;
+      const result = listDocuments(db, { language, limit, offset });
+      return rpcResult(id, result);
+    }
+
+    default:
+      return rpcError(id, -32601, `Method '${method}' not found`);
+  }
+}
+
+export function createApp(db: Database.Database): Hono {
+  const app = new Hono();
+
+  app.post("/rpc", async (c) => {
+    let body: unknown;
+    try {
+      body = await c.req.json();
+    } catch {
+      return c.json(rpcError(null, -32700, "Parse error"), 200);
+    }
+
+    // Batch support
+    if (Array.isArray(body)) {
+      const responses = body.map((req) => {
+        if (!isValidRequest(req)) {
+          return rpcError(req?.id ?? null, -32600, "Invalid JSON-RPC request");
+        }
+        return handleRpc(db, req);
+      });
+      return c.json(responses, 200);
+    }
+
+    if (!isValidRequest(body)) {
+      return c.json(
+        rpcError((body as any)?.id ?? null, -32600, "Invalid JSON-RPC request"),
+        200
+      );
+    }
+
+    return c.json(handleRpc(db, body), 200);
+  });
+
+  // Health check
+  app.get("/health", (c) => c.json({ status: "ok" }));
+
+  return app;
+}
+
+function isValidRequest(obj: unknown): obj is JsonRpcRequest {
+  if (typeof obj !== "object" || obj === null) return false;
+  const r = obj as Record<string, unknown>;
+  return r.jsonrpc === "2.0" && typeof r.method === "string";
+}
+
+import net from "net";
+
+function isPortInUse(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once("error", () => resolve(true));
+    server.once("listening", () => {
+      server.close();
+      resolve(false);
+    });
+    server.listen(port);
+  });
+}
+
+export async function startServer(opts: { dbPath?: string; port?: number }): Promise<void> {
+  const port = opts.port ?? 3777;
+
+  if (await isPortInUse(port)) {
+    console.error(`Port ${port} is already in use. Another instance may be running.`);
+    process.exit(1);
+  }
+
+  const db = openDb(opts.dbPath);
+  const app = createApp(db);
+
+  serve({ fetch: app.fetch, port }, () => {
+    console.log(`docs-search server listening on http://localhost:${port}`);
+    console.log(`JSON-RPC endpoint: POST http://localhost:${port}/rpc`);
+  });
+}

package/docs-search/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "commonjs",
+    "lib": ["ES2022"],
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}