readthemanual 0.0.1

package/README.md

@@ -0,0 +1,196 @@
+# rtm
+
+**Read the Manual.** Fast, local full-text search for any project's documentation.
+
+Point it at a GitHub repo, and it pulls down the markdown, indexes every section with SQLite FTS5, and gives you instant search from the terminal, an HTTP API, or an MCP server your AI tools can talk to.
+
+## Why
+
+Documentation lives in repos. Searching it means context-switching to a browser, waiting for GitHub's search, and losing your flow. `rtm` keeps a local, searchable copy of any project's docs — one command to ingest, instant results from your terminal.
+
+## Install
+
+```bash
+git clone https://github.com/rgr4y/rtm.git
+cd rtm
+npm install && npm run build
+npm link   # makes `rtm` available globally
+```
+
+## Quick start
+
+```bash
+# Add a source (any public GitHub repo with markdown docs)
+rtm add react facebook/react -p docs/
+
+# Pull and index the docs
+rtm ingest -s react -v
+
+# Search
+rtm "concurrent rendering"
+```
+
+That's it. Results come back with highlighted snippets, section headings, and file paths.
+
+## How it works
+
+1. **Ingest** fetches the repo tree via the GitHub API, downloads every `.md` file under the docs prefix, and saves them locally to `~/.local/rtm/<source>/`
+2. Each file is split by headings into sections
+3. Sections are indexed into SQLite with [FTS5](https://www.sqlite.org/fts5.html) using the `porter unicode61` tokenizer — the same stemming and Unicode handling you'd get from a dedicated search engine, in a single file
+4. **Search** runs FTS5 `MATCH` queries with ranked results and highlighted snippets
+
+## Usage
+
+### Search
+
+```bash
+# Bare query (no subcommand needed)
+rtm "hooks lifecycle"
+
+# Scope to a source
+rtm search "routing" -s nextjs
+
+# JSON output for scripting
+rtm search "middleware" --json
+
+# Limit results
+rtm search "state management" -n 5
+```
+
+### Manage sources
+
+```bash
+# Add a source
+rtm add <name> <owner/repo> [-b branch] [-p docs-prefix]
+
+# Examples
+rtm add nextjs vercel/next.js -p docs/
+rtm add django django/django -p docs/ -b main
+rtm add myproject myorg/myproject -p documentation/
+
+# Remove a source
+rtm remove nextjs
+
+# Switch default source for searches
+rtm use django
+
+# See all configured sources
+rtm config
+```
+
+### Ingest
+
+```bash
+# Ingest all sources
+rtm ingest -v
+
+# Ingest one source
+rtm ingest -s react -v
+
+# Include non-English docs
+rtm ingest --all-langs
+```
+
+### Read a full section
+
+```bash
+# Get a section by its ID (shown in search results)
+rtm get 42
+```
+
+### List indexed sections
+
+```bash
+rtm list
+rtm list -l en -n 100
+```
+
+## MCP server
+
+Run as an [MCP](https://modelcontextprotocol.io/) stdio server so AI tools can search your docs:
+
+```bash
+rtm mcp
+```
+
+Exposes four tools: `search`, `get_document`, `list_sources`, `list_documents`.
+
+Add it to any MCP-compatible client. Example for a `config.toml`:
+
+```toml
+[mcp]
+enabled = true
+
+[[mcp.servers]]
+name = "rtm"
+command = "rtm"
+args = ["mcp"]
+```
+
+Or for Claude Code's `settings.json`:
+
+```json
+{
+  "mcpServers": {
+    "rtm": {
+      "command": "rtm",
+      "args": ["mcp"]
+    }
+  }
+}
+```
+
+## HTTP server
+
+For network access or integration with other tools:
+
+```bash
+rtm serve -p 3777
+```
+
+JSON-RPC endpoint at `POST /rpc`:
+
+```bash
+curl -s http://localhost:3777/rpc \
+  -H 'Content-Type: application/json' \
+  -d '{"jsonrpc":"2.0","id":1,"method":"search","params":{"query":"authentication"}}' | jq
+```
+
+Methods: `search`, `get_document`, `list_documents`.
+
+## Configuration
+
+Config lives at `~/.local/rtm/config.json`. Downloaded docs and the search database are stored alongside it. You can edit it directly or use the CLI commands above.
+
+```json
+{
+  "sources": [
+    {
+      "name": "react",
+      "repo": "facebook/react",
+      "branch": "main",
+      "docsPrefix": "docs/"
+    }
+  ],
+  "dataDir": "/home/you/.local/rtm",
+  "lastSource": "react"
+}
+```
+
+Set `GITHUB_TOKEN` for private repos or to avoid rate limits:
+
+```bash
+export GITHUB_TOKEN=ghp_...
+```
+
+## Stack
+
+- **[SQLite FTS5](https://www.sqlite.org/fts5.html)** — full-text search with porter stemming
+- **[better-sqlite3](https://github.com/WiseLibs/better-sqlite3)** — synchronous, fast SQLite bindings
+- **[Commander](https://github.com/tj/commander.js)** — CLI framework
+- **[Hono](https://hono.dev/)** — lightweight HTTP server
+- **[@modelcontextprotocol/sdk](https://github.com/modelcontextprotocol/typescript-sdk)** — MCP server implementation
+
+## License
+
+ISC

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const ingest_1 = require("./ingest");
+const server_1 = require("./server");
+const mcp_1 = require("./mcp");
+const db_1 = require("./db");
+const config_1 = require("./config");
+const path_1 = __importDefault(require("path"));
+const config = (0, config_1.loadConfig)();
+const defaultDb = (0, config_1.getDbPath)(config);
+const program = new commander_1.Command();
+program
+    .name("rtm")
+    .description("Read the Manual — fast local full-text search for documentation")
+    .version("0.0.1");
+program
+    .command("ingest")
+    .description("Fetch and index docs from configured GitHub sources")
+    .option("-d, --db <path>", "SQLite database path", defaultDb)
+    .option("-s, --source <name>", "Ingest only this source")
+    .option("--all-langs", "Index all languages, not just English", false)
+    .option("-v, --verbose", "Verbose output", false)
+    .action(async (opts) => {
+    const result = await (0, ingest_1.ingest)({
+        dbPath: opts.db,
+        source: opts.source,
+        allLangs: opts.allLangs,
+        verbose: opts.verbose,
+    });
+    // Update last source if a specific one was ingested
+    if (opts.source) {
+        (0, config_1.setLastSource)(opts.source);
+    }
+    console.log(`Indexed ${result.filesProcessed} files (${result.sectionsInserted} sections).`);
+});
+program
+    .command("serve")
+    .description("Start the JSON-RPC HTTP server")
+    .option("-d, --db <path>", "SQLite database path", defaultDb)
+    .option("-p, --port <number>", "Port to listen on", "3777")
+    .action((opts) => {
+    (0, server_1.startServer)({ dbPath: opts.db, port: parseInt(opts.port, 10) });
+});
+program
+    .command("mcp")
+    .description("Start as an MCP server over stdio")
+    .option("-d, --db <path>", "SQLite database path", defaultDb)
+    .action((opts) => {
+    (0, mcp_1.startMcp)({ dbPath: opts.db });
+});
+program
+    .command("search <query>")
+    .description("Search the docs index")
+    .option("-d, --db <path>", "SQLite database path", defaultDb)
+    .option("-s, --source <name>", "Search only this source")
+    .option("-l, --lang <language>", "Filter by language")
+    .option("-n, --limit <number>", "Max results", "10")
+    .option("--json", "Output raw JSON", false)
+    .action((query, opts) => {
+    const cfg = (0, config_1.loadConfig)();
+    const db = (0, db_1.openDb)(opts.db);
+    // Determine source scope: explicit flag > lastSource > all
+    const sourceName = opts.source ?? (0, config_1.getDefaultSource)(cfg);
+    const { results, total } = (0, db_1.search)(db, query, {
+        language: opts.lang,
+        limit: parseInt(opts.limit, 10),
+        sourcePrefix: sourceName ? `${sourceName}/` : undefined,
+    });
+    db.close();
+    if (opts.json) {
+        console.log(JSON.stringify({ results, total }, null, 2));
+        return;
+    }
+    if (results.length === 0) {
+        console.log("No results found.");
+        return;
+    }
+    for (const r of results) {
+        const fullPath = path_1.default.join(cfg.dataDir, r.file_path);
+        const snippet = r.snippet.replace(/<\/?mark>/g, (m) => m === "<mark>" ? "\x1b[1;33m" : "\x1b[0m");
+        console.log(`\x1b[36m[${r.id}]\x1b[0m \x1b[1m${r.heading}\x1b[0m`);
+        console.log(`     ${fullPath} (${r.language})`);
+        console.log(`     ${snippet}`);
+        console.log();
+    }
+    if (sourceName) {
+        console.log(`${results.length} of ${total} match(es) [source: ${sourceName}]`);
+    }
+    else {
+        console.log(`${results.length} of ${total} match(es)`);
+    }
+});
+program
+    .command("get <id>")
+    .description("Get a document section by ID")
+    .option("-d, --db <path>", "SQLite database path", defaultDb)
+    .option("--json", "Output raw JSON", false)
+    .action((id, opts) => {
+    const db = (0, db_1.openDb)(opts.db);
+    const doc = (0, db_1.getDocument)(db, parseInt(id, 10));
+    db.close();
+    if (!doc) {
+        console.error(`Document ${id} not found.`);
+        process.exit(1);
+    }
+    if (opts.json) {
+        console.log(JSON.stringify(doc, null, 2));
+        return;
+    }
+    const cfg = (0, config_1.loadConfig)();
+    const fullPath = path_1.default.join(cfg.dataDir, doc.file_path);
+    console.log(`\x1b[1m${doc.heading}\x1b[0m`);
+    console.log(`${fullPath} (${doc.language})`);
+    console.log();
+    console.log(doc.content);
+});
+program
+    .command("list")
+    .description("List indexed documents")
+    .option("-d, --db <path>", "SQLite database path", defaultDb)
+    .option("-l, --lang <language>", "Filter by language")
+    .option("-n, --limit <number>", "Max results", "50")
+    .option("-o, --offset <number>", "Offset", "0")
+    .option("--json", "Output raw JSON", false)
+    .action((opts) => {
+    const db = (0, db_1.openDb)(opts.db);
+    const result = (0, db_1.listDocuments)(db, {
+        language: opts.lang,
+        limit: parseInt(opts.limit, 10),
+        offset: parseInt(opts.offset, 10),
+    });
+    db.close();
+    if (opts.json) {
+        console.log(JSON.stringify(result, null, 2));
+        return;
+    }
+    const cfg = (0, config_1.loadConfig)();
+    for (const d of result.docs) {
+        const fullPath = path_1.default.join(cfg.dataDir, d.file_path);
+        console.log(`\x1b[36m[${d.id}]\x1b[0m ${fullPath} — ${d.heading} (${d.language})`);
+    }
+    console.log(`\nShowing ${result.docs.length} of ${result.total} total sections`);
+});
+program
+    .command("add <name> <repo>")
+    .description("Add a GitHub repo as a doc source (repo = owner/repo)")
+    .option("-b, --branch <branch>", "Branch to index", "main")
+    .option("-p, --prefix <prefix>", "Path prefix for docs in the repo", "docs/")
+    .action((name, repo, opts) => {
+    (0, config_1.addSource)({ name, repo, branch: opts.branch, docsPrefix: opts.prefix });
+    console.log(`Added source "${name}" → ${repo}@${opts.branch} (prefix: ${opts.prefix})`);
+    console.log(`Run \`rtm ingest -s ${name}\` to index it.`);
+});
+program
+    .command("remove <name>")
+    .description("Remove a doc source")
+    .action((name) => {
+    if ((0, config_1.removeSource)(name)) {
+        console.log(`Removed source "${name}".`);
+    }
+    else {
+        console.error(`Source "${name}" not found.`);
+        process.exit(1);
+    }
+});
+program
+    .command("use <name>")
+    .description("Set the default source for searches")
+    .action((name) => {
+    const cfg = (0, config_1.loadConfig)();
+    if (!cfg.sources.some((s) => s.name === name)) {
+        const available = cfg.sources.map((s) => s.name).join(", ");
+        console.error(`Source "${name}" not found. Available: ${available}`);
+        process.exit(1);
+    }
+    (0, config_1.setLastSource)(name);
+    console.log(`Default source set to "${name}".`);
+});
+program
+    .command("config")
+    .description("Show config file path and contents")
+    .action(() => {
+    const cfg = (0, config_1.loadConfig)();
+    console.log(`Config:  ${(0, config_1.getConfigPath)()}`);
+    console.log(`Data:    ${cfg.dataDir}`);
+    console.log(`Install: ${(0, config_1.isGlobal)() ? "global (/usr/local)" : "local (~/.local)"}`);
+    console.log(`Default: ${(0, config_1.getDefaultSource)(cfg) ?? "(none)"}`);
+    console.log();
+    console.log(`Sources (${cfg.sources.length}):`);
+    for (const s of cfg.sources) {
+        const marker = s.name === (0, config_1.getDefaultSource)(cfg) ? " *" : "";
+        console.log(`  - ${s.name}: ${s.repo}@${s.branch ?? "main"} (prefix: ${s.docsPrefix ?? "docs/"})${marker}`);
+    }
+});
+// Default: bare args with no subcommand → search
+const knownCommands = ["ingest", "serve", "mcp", "search", "get", "list", "add", "remove", "use", "config", "help"];
+const firstArg = process.argv[2];
+if (firstArg && !firstArg.startsWith("-") && !knownCommands.includes(firstArg)) {
+    process.argv.splice(2, 0, "search");
+}
+program.parse();

package/dist/config.d.ts

@@ -0,0 +1,21 @@
+export interface SourceConfig {
+    name: string;
+    repo: string;
+    branch?: string;
+    docsPrefix?: string;
+}
+export interface Config {
+    sources: SourceConfig[];
+    dataDir: string;
+    lastSource?: string;
+}
+export declare function getConfigPath(): string;
+export declare function isGlobal(): boolean;
+export declare function checkWriteAccess(): void;
+export declare function loadConfig(): Config;
+export declare function getDbPath(config: Config): string;
+export declare function getDocsDir(config: Config, sourceName: string): string;
+export declare function addSource(source: SourceConfig): void;
+export declare function removeSource(name: string): boolean;
+export declare function setLastSource(name: string): void;
+export declare function getDefaultSource(config: Config): string | undefined;

package/dist/config.js

@@ -0,0 +1,134 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getConfigPath = getConfigPath;
+exports.isGlobal = isGlobal;
+exports.checkWriteAccess = checkWriteAccess;
+exports.loadConfig = loadConfig;
+exports.getDbPath = getDbPath;
+exports.getDocsDir = getDocsDir;
+exports.addSource = addSource;
+exports.removeSource = removeSource;
+exports.setLastSource = setLastSource;
+exports.getDefaultSource = getDefaultSource;
+const fs_1 = __importDefault(require("fs"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+function isGlobalInstall() {
+    return __dirname.startsWith("/usr/local");
+}
+// Data always lives in user's home — it's user data, not system data
+const DATA_DIR = path_1.default.join(os_1.default.homedir(), ".local", "rtm");
+const CONFIG_PATH = path_1.default.join(DATA_DIR, "config.json");
+const DEFAULT_CONFIG = {
+    sources: [
+        {
+            name: "zeroclaw",
+            repo: "zeroclaw-labs/zeroclaw",
+            branch: "main",
+            docsPrefix: "docs/",
+        },
+    ],
+    dataDir: DATA_DIR,
+    lastSource: "zeroclaw",
+};
+function getConfigPath() {
+    return CONFIG_PATH;
+}
+function isGlobal() {
+    return isGlobalInstall();
+}
+function hasWriteAccess(dir) {
+    // Walk up to find the first existing ancestor and check write permission
+    let check = dir;
+    while (!fs_1.default.existsSync(check)) {
+        const parent = path_1.default.dirname(check);
+        if (parent === check)
+            return false;
+        check = parent;
+    }
+    try {
+        fs_1.default.accessSync(check, fs_1.default.constants.W_OK);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function checkWriteAccess() {
+    if (!hasWriteAccess(DATA_DIR)) {
+        console.error(`Error: No write permission to ${DATA_DIR}\nCheck directory permissions.`);
+        process.exit(1);
+    }
+}
+function loadConfig() {
+    if (!fs_1.default.existsSync(CONFIG_PATH)) {
+        // Return defaults in memory; only write if we have access
+        if (hasWriteAccess(DATA_DIR) || hasWriteAccess(path_1.default.dirname(DATA_DIR))) {
+            fs_1.default.mkdirSync(DATA_DIR, { recursive: true });
+            fs_1.default.writeFileSync(CONFIG_PATH, JSON.stringify(DEFAULT_CONFIG, null, 2) + "\n", "utf-8");
+        }
+        return DEFAULT_CONFIG;
+    }
+    const raw = JSON.parse(fs_1.default.readFileSync(CONFIG_PATH, "utf-8"));
+    return {
+        sources: raw.sources ?? DEFAULT_CONFIG.sources,
+        dataDir: raw.dataDir ?? DATA_DIR,
+        lastSource: raw.lastSource,
+    };
+}
+function getDbPath(config) {
+    return path_1.default.join(config.dataDir, "rtm.db");
+}
+function getDocsDir(config, sourceName) {
+    return path_1.default.join(config.dataDir, sourceName);
+}
+function saveConfig(config) {
+    checkWriteAccess();
+    fs_1.default.mkdirSync(path_1.default.dirname(CONFIG_PATH), { recursive: true });
+    fs_1.default.writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2) + "\n", "utf-8");
+}
+function addSource(source) {
+    const config = loadConfig();
+    const existing = config.sources.findIndex((s) => s.name === source.name);
+    if (existing >= 0) {
+        config.sources[existing] = source;
+    }
+    else {
+        config.sources.push(source);
+    }
+    config.lastSource = source.name;
+    saveConfig(config);
+}
+function removeSource(name) {
+    const config = loadConfig();
+    const before = config.sources.length;
+    config.sources = config.sources.filter((s) => s.name !== name);
+    if (config.sources.length === before)
+        return false;
+    // If we removed the last-used source, reset to the final remaining one
+    if (config.lastSource === name) {
+        config.lastSource = config.sources.length > 0
+            ? config.sources[config.sources.length - 1].name
+            : undefined;
+    }
+    saveConfig(config);
+    return true;
+}
+function setLastSource(name) {
+    const config = loadConfig();
+    config.lastSource = name;
+    saveConfig(config);
+}
+function getDefaultSource(config) {
+    // Last-used source, or the last one in the list, or undefined
+    if (config.lastSource && config.sources.some((s) => s.name === config.lastSource)) {
+        return config.lastSource;
+    }
+    if (config.sources.length > 0) {
+        return config.sources[config.sources.length - 1].name;
+    }
+    return undefined;
+}

package/dist/db.d.ts

@@ -0,0 +1,33 @@
+import Database from "better-sqlite3";
+export interface DocSection {
+    id: number;
+    file_path: string;
+    heading: string;
+    content: string;
+    language: string;
+}
+export interface SearchResult extends DocSection {
+    rank: number;
+    snippet: string;
+}
+export declare function openDb(dbPath?: string): Database.Database;
+export declare function clearDocs(db: Database.Database): void;
+export declare function insertSection(db: Database.Database, section: Omit<DocSection, "id">): number;
+export interface SearchResponse {
+    results: SearchResult[];
+    total: number;
+}
+export declare function search(db: Database.Database, query: string, opts?: {
+    language?: string;
+    limit?: number;
+    sourcePrefix?: string;
+}): SearchResponse;
+export declare function getDocument(db: Database.Database, id: number): DocSection | null;
+export declare function listDocuments(db: Database.Database, opts?: {
+    language?: string;
+    limit?: number;
+    offset?: number;
+}): {
+    docs: Pick<DocSection, "id" | "file_path" | "heading" | "language">[];
+    total: number;
+};