readthemanual 0.0.1

package/src/ingest.ts

@@ -0,0 +1,200 @@
+import { openDb, clearDocs, insertSection } from "./db";
+import { loadConfig, getDbPath, getDocsDir, type SourceConfig } from "./config";
+import type Database from "better-sqlite3";
+import fs from "fs";
+import path from "path";
+
+interface TreeEntry {
+  path: string;
+  type: string;
+  url: string;
+}
+
+async function fetchTree(repo: string, branch: string): Promise<TreeEntry[]> {
+  const url = `https://api.github.com/repos/${repo}/git/trees/${branch}?recursive=1`;
+  const res = await fetch(url, {
+    headers: {
+      Accept: "application/vnd.github.v3+json",
+      ...(process.env.GITHUB_TOKEN
+        ? { Authorization: `token ${process.env.GITHUB_TOKEN}` }
+        : {}),
+    },
+  });
+  if (!res.ok) throw new Error(`GitHub tree API: ${res.status} ${res.statusText}`);
+  const data = (await res.json()) as { tree: TreeEntry[] };
+  return data.tree;
+}
+
+async function fetchRawFile(repo: string, branch: string, filePath: string): Promise<string> {
+  const url = `https://raw.githubusercontent.com/${repo}/${branch}/${filePath}`;
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Failed to fetch ${filePath}: ${res.status}`);
+  return res.text();
+}
+
+function detectLanguage(filePath: string): string {
+  const match = filePath.match(/^docs\/i18n\/([a-z]{2}(?:-[A-Z]{2})?)\//);
+  if (match) return match[1];
+  return "en";
+}
+
+interface Section {
+  heading: string;
+  content: string;
+}
+
+function splitByHeadings(markdown: string): Section[] {
+  const lines = markdown.split("\n");
+  const sections: Section[] = [];
+  let currentHeading = "(intro)";
+  let currentLines: string[] = [];
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,3})\s+(.+)/);
+    if (headingMatch) {
+      if (currentLines.length > 0) {
+        const content = currentLines.join("\n").trim();
+        if (content.length > 0) {
+          sections.push({ heading: currentHeading, content });
+        }
+      }
+      currentHeading = headingMatch[2];
+      currentLines = [];
+    } else {
+      currentLines.push(line);
+    }
+  }
+
+  if (currentLines.length > 0) {
+    const content = currentLines.join("\n").trim();
+    if (content.length > 0) {
+      sections.push({ heading: currentHeading, content });
+    }
+  }
+
+  return sections;
+}
+
+async function ingestSource(
+  db: Database.Database,
+  source: SourceConfig,
+  opts: { allLangs?: boolean; verbose?: boolean }
+): Promise<{ filesProcessed: number; sectionsInserted: number }> {
+  const log = opts.verbose ? console.log : () => {};
+  const repo = source.repo;
+  const branch = source.branch ?? "main";
+  const docsPrefix = source.docsPrefix ?? "docs/";
+
+  const config = loadConfig();
+  const docsDir = getDocsDir(config, source.name);
+
+  log(`[${source.name}] Fetching tree from ${repo}@${branch}...`);
+  const tree = await fetchTree(repo, branch);
+
+  const mdFiles = tree.filter((entry) => {
+    if (entry.type !== "blob") return false;
+    if (!entry.path.startsWith(docsPrefix)) return false;
+    if (!entry.path.endsWith(".md")) return false;
+    if (/\.(jpg|png|svg|gif)$/i.test(entry.path)) return false;
+    if (!opts.allLangs) {
+      const lang = detectLanguage(entry.path);
+      if (lang !== "en") return false;
+    }
+    return true;
+  });
+
+  log(`[${source.name}] Found ${mdFiles.length} markdown files.`);
+
+  // Save docs locally
+  fs.rmSync(docsDir, { recursive: true, force: true });
+  fs.mkdirSync(docsDir, { recursive: true });
+
+  let filesProcessed = 0;
+  let sectionsInserted = 0;
+  const allSections: { file_path: string; heading: string; content: string; language: string }[] = [];
+
+  const batchSize = 10;
+  for (let i = 0; i < mdFiles.length; i += batchSize) {
+    const batch = mdFiles.slice(i, i + batchSize);
+    const results = await Promise.all(
+      batch.map(async (entry) => {
+        const content = await fetchRawFile(repo, branch, entry.path);
+        return { path: entry.path, content };
+      })
+    );
+
+    for (const { path: filePath, content } of results) {
+      // Strip docsPrefix so we don't get docs/docs/ nesting
+      const relativePath = filePath.startsWith(docsPrefix)
+        ? filePath.slice(docsPrefix.length)
+        : filePath;
+
+      // Write raw markdown to disk
+      const localPath = path.join(docsDir, relativePath);
+      fs.mkdirSync(path.dirname(localPath), { recursive: true });
+      fs.writeFileSync(localPath, content, "utf-8");
+
+      const lang = detectLanguage(filePath);
+      const sections = splitByHeadings(content);
+      for (const section of sections) {
+        allSections.push({
+          file_path: `${source.name}/${relativePath}`,
+          heading: section.heading,
+          content: section.content,
+          language: lang,
+        });
+      }
+      filesProcessed++;
+      log(`  [${filesProcessed}/${mdFiles.length}] ${filePath} (${sections.length} sections)`);
+    }
+  }
+
+  const insertMany = db.transaction(
+    (sections: { file_path: string; heading: string; content: string; language: string }[]) => {
+      for (const s of sections) {
+        insertSection(db, s);
+      }
+    }
+  );
+  insertMany(allSections);
+  sectionsInserted = allSections.length;
+
+  return { filesProcessed, sectionsInserted };
+}
+
+export async function ingest(opts: {
+  dbPath?: string;
+  source?: string;
+  allLangs?: boolean;
+  verbose?: boolean;
+}): Promise<{ filesProcessed: number; sectionsInserted: number }> {
+  const config = loadConfig();
+  const dbPath = opts.dbPath ?? getDbPath(config);
+  const db = openDb(dbPath);
+  const log = opts.verbose ? console.log : () => {};
+
+  // Filter to a single source if requested
+  const sources = opts.source
+    ? config.sources.filter((s) => s.name === opts.source)
+    : config.sources;
+
+  if (sources.length === 0) {
+    const available = config.sources.map((s) => s.name).join(", ");
+    throw new Error(`Source "${opts.source}" not found in config. Available: ${available}`);
+  }
+
+  clearDocs(db);
+
+  let totalFiles = 0;
+  let totalSections = 0;
+
+  for (const source of sources) {
+    const result = await ingestSource(db, source, opts);
+    totalFiles += result.filesProcessed;
+    totalSections += result.sectionsInserted;
+  }
+
+  db.close();
+  log(`Done. ${totalFiles} files, ${totalSections} sections indexed from ${sources.length} source(s).`);
+  return { filesProcessed: totalFiles, sectionsInserted: totalSections };
+}

package/src/mcp.ts

@@ -0,0 +1,124 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+import { openDb, search, getDocument, listDocuments } from "./db";
+import { loadConfig, getDbPath, getDefaultSource } from "./config";
+import path from "path";
+
+export async function startMcp(opts: { dbPath?: string }): Promise<void> {
+  const config = loadConfig();
+  const dbPath = opts.dbPath ?? getDbPath(config);
+  const db = openDb(dbPath);
+
+  const server = new McpServer({
+    name: "rtm",
+    version: "0.0.1",
+  });
+
+  server.registerTool(
+    "search",
+    {
+      title: "Search docs",
+      description: "Full-text search across indexed documentation. Returns matching sections with snippets.",
+      inputSchema: {
+        query: z.string().describe("Search query"),
+        source: z.string().optional().describe("Source name to scope search (default: last used source)"),
+        language: z.string().optional().describe("Filter by language code (e.g. 'en')"),
+        limit: z.number().optional().describe("Max results (default: 10)"),
+      },
+    },
+    async ({ query, source, language, limit }) => {
+      const cfg = loadConfig();
+      const sourceName = source ?? getDefaultSource(cfg);
+      const { results, total } = search(db, query, {
+        language,
+        limit: limit ?? 10,
+        sourcePrefix: sourceName ? `${sourceName}/` : undefined,
+      });
+
+      const text = results.length === 0
+        ? "No results found."
+        : results.map((r) => {
+            const fullPath = path.join(cfg.dataDir, r.file_path);
+            const clean = r.snippet.replace(/<\/?mark>/g, "");
+            return `[${r.id}] ${r.heading}\n  ${fullPath}\n  ${clean}`;
+          }).join("\n\n") + `\n\n${results.length} of ${total} match(es)`;
+
+      return { content: [{ type: "text" as const, text }] };
+    }
+  );
+
+  server.registerTool(
+    "get_document",
+    {
+      title: "Get document",
+      description: "Retrieve a full document section by its ID.",
+      inputSchema: {
+        id: z.number().describe("Document section ID"),
+      },
+    },
+    async ({ id }) => {
+      const doc = getDocument(db, id);
+      if (!doc) {
+        return { content: [{ type: "text" as const, text: `Document ${id} not found.` }] };
+      }
+      const cfg = loadConfig();
+      const fullPath = path.join(cfg.dataDir, doc.file_path);
+      return {
+        content: [{
+          type: "text" as const,
+          text: `# ${doc.heading}\n${fullPath} (${doc.language})\n\n${doc.content}`,
+        }],
+      };
+    }
+  );
+
+  server.registerTool(
+    "list_sources",
+    {
+      title: "List sources",
+      description: "List configured documentation sources.",
+      inputSchema: {},
+    },
+    async () => {
+      const cfg = loadConfig();
+      const defaultSrc = getDefaultSource(cfg);
+      const lines = cfg.sources.map((s) => {
+        const marker = s.name === defaultSrc ? " (default)" : "";
+        return `- ${s.name}: ${s.repo}@${s.branch ?? "main"}${marker}`;
+      });
+      return { content: [{ type: "text" as const, text: lines.join("\n") }] };
+    }
+  );
+
+  server.registerTool(
+    "list_documents",
+    {
+      title: "List documents",
+      description: "List indexed document sections.",
+      inputSchema: {
+        language: z.string().optional().describe("Filter by language"),
+        limit: z.number().optional().describe("Max results (default: 50)"),
+        offset: z.number().optional().describe("Offset for pagination"),
+      },
+    },
+    async ({ language, limit, offset }) => {
+      const result = listDocuments(db, {
+        language,
+        limit: limit ?? 50,
+        offset: offset ?? 0,
+      });
+      const cfg = loadConfig();
+      const lines = result.docs.map((d) => {
+        const fullPath = path.join(cfg.dataDir, d.file_path);
+        return `[${d.id}] ${fullPath} — ${d.heading}`;
+      });
+      const text = lines.join("\n") + `\n\n${result.docs.length} of ${result.total} total`;
+      return { content: [{ type: "text" as const, text }] };
+    }
+  );
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error("rtm MCP server running on stdio");
+}

package/src/server.ts

@@ -0,0 +1,145 @@
+import { Hono } from "hono";
+import { serve } from "@hono/node-server";
+import { openDb, search, getDocument, listDocuments } from "./db";
+import type Database from "better-sqlite3";
+
+interface JsonRpcRequest {
+  jsonrpc: "2.0";
+  id: string | number | null;
+  method: string;
+  params?: Record<string, unknown>;
+}
+
+interface JsonRpcResponse {
+  jsonrpc: "2.0";
+  id: string | number | null;
+  result?: unknown;
+  error?: { code: number; message: string; data?: unknown };
+}
+
+function rpcError(
+  id: string | number | null,
+  code: number,
+  message: string
+): JsonRpcResponse {
+  return { jsonrpc: "2.0", id, error: { code, message } };
+}
+
+function rpcResult(id: string | number | null, result: unknown): JsonRpcResponse {
+  return { jsonrpc: "2.0", id, result };
+}
+
+function handleRpc(db: Database.Database, req: JsonRpcRequest): JsonRpcResponse {
+  const { id, method, params } = req;
+
+  switch (method) {
+    case "search": {
+      const query = params?.query;
+      if (typeof query !== "string" || query.trim().length === 0) {
+        return rpcError(id, -32602, "params.query is required and must be a non-empty string");
+      }
+      const language = typeof params?.language === "string" ? params.language : undefined;
+      const limit = typeof params?.limit === "number" ? params.limit : undefined;
+      const { results, total } = search(db, query, { language, limit });
+      return rpcResult(id, { results, total, count: results.length });
+    }
+
+    case "get_document": {
+      const docId = params?.id;
+      if (typeof docId !== "number") {
+        return rpcError(id, -32602, "params.id is required and must be a number");
+      }
+      const doc = getDocument(db, docId);
+      if (!doc) {
+        return rpcError(id, -32602, `Document with id ${docId} not found`);
+      }
+      return rpcResult(id, doc);
+    }
+
+    case "list_documents": {
+      const language = typeof params?.language === "string" ? params.language : undefined;
+      const limit = typeof params?.limit === "number" ? params.limit : undefined;
+      const offset = typeof params?.offset === "number" ? params.offset : undefined;
+      const result = listDocuments(db, { language, limit, offset });
+      return rpcResult(id, result);
+    }
+
+    default:
+      return rpcError(id, -32601, `Method '${method}' not found`);
+  }
+}
+
+export function createApp(db: Database.Database): Hono {
+  const app = new Hono();
+
+  app.post("/rpc", async (c) => {
+    let body: unknown;
+    try {
+      body = await c.req.json();
+    } catch {
+      return c.json(rpcError(null, -32700, "Parse error"), 200);
+    }
+
+    // Batch support
+    if (Array.isArray(body)) {
+      const responses = body.map((req) => {
+        if (!isValidRequest(req)) {
+          return rpcError(req?.id ?? null, -32600, "Invalid JSON-RPC request");
+        }
+        return handleRpc(db, req);
+      });
+      return c.json(responses, 200);
+    }
+
+    if (!isValidRequest(body)) {
+      return c.json(
+        rpcError((body as any)?.id ?? null, -32600, "Invalid JSON-RPC request"),
+        200
+      );
+    }
+
+    return c.json(handleRpc(db, body), 200);
+  });
+
+  // Health check
+  app.get("/health", (c) => c.json({ status: "ok" }));
+
+  return app;
+}
+
+function isValidRequest(obj: unknown): obj is JsonRpcRequest {
+  if (typeof obj !== "object" || obj === null) return false;
+  const r = obj as Record<string, unknown>;
+  return r.jsonrpc === "2.0" && typeof r.method === "string";
+}
+
+import net from "net";
+
+function isPortInUse(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once("error", () => resolve(true));
+    server.once("listening", () => {
+      server.close();
+      resolve(false);
+    });
+    server.listen(port);
+  });
+}
+
+export async function startServer(opts: { dbPath?: string; port?: number }): Promise<void> {
+  const port = opts.port ?? 3777;
+
+  if (await isPortInUse(port)) {
+    console.error(`Port ${port} is already in use. Another instance may be running.`);
+    process.exit(1);
+  }
+
+  const db = openDb(opts.dbPath);
+  const app = createApp(db);
+
+  serve({ fetch: app.fetch, port }, () => {
+    console.log(`rtm server listening on http://localhost:${port}`);
+    console.log(`JSON-RPC endpoint: POST http://localhost:${port}/rpc`);
+  });
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "commonjs",
+    "lib": ["ES2022"],
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true
+  },
+  "include": ["src"],
+  "exclude": ["node_modules", "dist"]
+}