ragpipe 0.0.1 → 0.1.0-alpha.1

package/README.md

@@ -0,0 +1,109 @@
+# ragpipe
+
+Pluggable TypeScript RAG toolkit — `defineConfig()` one file, embed → search → generate.
+
+## Install
+
+```bash
+pnpm add ragpipe
+```
+
+## Quick Start
+
+### CLI
+
+```bash
+# Scaffold a ragpipe.config.ts
+npx ragpipe init
+
+# Ingest documents
+npx ragpipe ingest ./docs
+
+# Ask a question
+npx ragpipe ask "What is the refund policy?"
+```
+
+### Programmatic
+
+```ts
+import { loadConfig, createPipeline } from "ragpipe";
+
+const config = await loadConfig();
+const rag = createPipeline(config);
+
+await rag.ingest(markdownText, "docs/guide.md");
+
+const result = await rag.ask("How does authentication work?");
+console.log(result.answer);
+console.log(result.sources.map((s) => s.source));
+```
+
+## Configuration
+
+Create a `ragpipe.config.ts` at your project root:
+
+```ts
+import { defineConfig } from "ragpipe";
+import { geminiEmbedding, geminiGeneration } from "@ragpipe/plugin-gemini";
+import { supabaseVectorStore } from "@ragpipe/plugin-supabase";
+
+export default defineConfig({
+  embedding: geminiEmbedding({
+    apiKey: process.env.GEMINI_API_KEY ?? "",
+  }),
+  vectorStore: supabaseVectorStore({
+    databaseUrl: process.env.DATABASE_URL ?? "",
+  }),
+  generation: geminiGeneration({
+    apiKey: process.env.GEMINI_API_KEY ?? "",
+  }),
+});
+```
+
+## API
+
+### `defineConfig(config)`
+
+Identity helper that provides type-safe autocompletion for your config file.
+
+### `loadConfig(overrides?)`
+
+Loads `ragpipe.config.ts` from the project root using [c12](https://github.com/unjs/c12). Validates that `embedding`, `vectorStore`, and `generation` plugins are present.
+
+### `createPipeline(config)`
+
+Returns a pipeline with three methods:
+
+| Method | Description |
+|---|---|
+| `ingest(text, source)` | Chunk text, embed each chunk, and store vectors. Returns chunk count. |
+| `search(query, topK?)` | Embed the query and return the top-K matching documents. |
+| `ask(query, topK?)` | Search for context, then generate an answer. Returns `{ answer, sources }`. |
+
+### `defaultChunker(options?)`
+
+Built-in paragraph-based chunker. Options: `chunkSize` (default 500), `overlap` (default 50).
+
+### `createRateLimitedEmbedder(plugin)`
+
+Wraps an `EmbeddingPlugin` with throttling based on its `rateLimit.delayMs`.
+
+## Plugin Interfaces
+
+Implement any of these to create a custom plugin:
+
+- **`EmbeddingPlugin`** — `embed(text)`, optional `embedMany(texts)`, `rateLimit`
+- **`VectorStorePlugin`** — `search(vector, topK)`, `upsert(source, content, vector)`, optional `clear()`, `disconnect()`
+- **`GenerationPlugin`** — `generate(question, context, options?)`, optional `generateStream()`
+- **`ChunkerPlugin`** — `chunk(text, source)`
+
+## Official Plugins
+
+| Package | Description |
+|---|---|
+| [`@ragpipe/plugin-gemini`](https://www.npmjs.com/package/@ragpipe/plugin-gemini) | Google Gemini embedding + generation |
+| [`@ragpipe/plugin-supabase`](https://www.npmjs.com/package/@ragpipe/plugin-supabase) | Supabase pgvector store |
+
+## License
+
+MIT

package/dist/cli.js

@@ -0,0 +1,372 @@
+#!/usr/bin/env node
+
+// src/cli.ts
+import { defineCommand as defineCommand4, runMain } from "citty";
+
+// src/cli/ask.ts
+import { defineCommand } from "citty";
+import consola from "consola";
+
+// src/config.ts
+import { loadConfig as c12LoadConfig } from "c12";
+async function loadConfig(overrides) {
+  const { config } = await c12LoadConfig({
+    name: "ragpipe",
+    defaults: overrides
+  });
+  if (!config) {
+    throw new Error(
+      "No ragpipe config found. Create a ragpipe.config.ts or pass config directly."
+    );
+  }
+  const cfg = config;
+  if (!cfg.embedding) {
+    throw new Error("ragpipe config is missing 'embedding' plugin.");
+  }
+  if (!cfg.vectorStore) {
+    throw new Error("ragpipe config is missing 'vectorStore' plugin.");
+  }
+  if (!cfg.generation) {
+    throw new Error("ragpipe config is missing 'generation' plugin.");
+  }
+  return cfg;
+}
+
+// src/chunker.ts
+var DEFAULT_CHUNK_SIZE = 400;
+var DEFAULT_OVERLAP = 50;
+function defaultChunker(options) {
+  const chunkSize = options?.chunkSize ?? DEFAULT_CHUNK_SIZE;
+  const overlap = options?.overlap ?? DEFAULT_OVERLAP;
+  return {
+    name: "default",
+    chunk(text, source) {
+      const paragraphs = text.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+      const chunks = [];
+      let current = "";
+      for (const paragraph of paragraphs) {
+        if (current && current.length + paragraph.length + 1 > chunkSize) {
+          chunks.push({ source, content: current });
+          const overlapSlice = current.slice(-overlap);
+          current = overlapSlice + paragraph;
+        } else {
+          current = current ? `${current}
+${paragraph}` : paragraph;
+        }
+      }
+      if (current.trim()) {
+        chunks.push({ source, content: current });
+      }
+      if (chunks.length === 0 && text.trim()) {
+        chunks.push({ source, content: text.trim() });
+      }
+      return chunks;
+    }
+  };
+}
+
+// src/rate-limiter.ts
+function sleep(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
+function createRateLimitedEmbedder(plugin, delayMs) {
+  const delay = delayMs ?? plugin.rateLimit?.delayMs ?? 0;
+  let lastCall = 0;
+  async function throttle() {
+    if (delay <= 0) return;
+    const now = Date.now();
+    const elapsed = now - lastCall;
+    if (elapsed < delay) {
+      await sleep(delay - elapsed);
+    }
+    lastCall = Date.now();
+  }
+  return {
+    name: plugin.name,
+    dimensions: plugin.dimensions,
+    rateLimit: plugin.rateLimit,
+    async embed(text) {
+      await throttle();
+      return plugin.embed(text);
+    },
+    async embedMany(texts) {
+      if (plugin.embedMany) {
+        await throttle();
+        return plugin.embedMany(texts);
+      }
+      const results = [];
+      for (const text of texts) {
+        await throttle();
+        results.push(await plugin.embed(text));
+      }
+      return results;
+    }
+  };
+}
+
+// src/pipeline.ts
+function createPipeline(config) {
+  const chunker = config.chunker ?? defaultChunker();
+  const embedder = config.embedding.rateLimit ? createRateLimitedEmbedder(config.embedding) : config.embedding;
+  return {
+    async ingest(text, source) {
+      const chunks = chunker.chunk(text, source);
+      for (const chunk of chunks) {
+        const vector = await embedder.embed(chunk.content);
+        await config.vectorStore.upsert(chunk.source, chunk.content, vector);
+      }
+      return chunks.length;
+    },
+    async search(query, topK = 5) {
+      const vector = await embedder.embed(query);
+      return config.vectorStore.search(vector, topK);
+    },
+    async ask(query, topK = 5) {
+      const chunks = await this.search(query, topK);
+      const context = chunks.map((c) => `[${c.source}]
+${c.content}`).join("\n\n---\n\n");
+      const answer = await config.generation.generate(query, context, {
+        systemPrompt: config.systemPrompt
+      });
+      return { answer, sources: chunks };
+    }
+  };
+}
+
+// src/cli/ask.ts
+var askCommand = defineCommand({
+  meta: {
+    name: "ask",
+    description: "Ask a question against ingested documents"
+  },
+  args: {
+    query: {
+      type: "positional",
+      description: "The question to ask",
+      required: true
+    },
+    topK: {
+      type: "string",
+      description: "Number of context chunks to retrieve",
+      default: "5"
+    }
+  },
+  async run({ args }) {
+    const query = args.query;
+    const topK = Number.parseInt(args.topK, 10);
+    consola.start(`Asking: "${query}"`);
+    const config = await loadConfig();
+    const pipeline = createPipeline(config);
+    const result = await pipeline.ask(query, topK);
+    console.log(`
+${result.answer}
+`);
+    if (result.sources.length > 0) {
+      consola.info("Sources:");
+      for (const source of result.sources) {
+        consola.log(`  \u2022 ${source.source} (score: ${source.score.toFixed(3)})`);
+      }
+    }
+    if (config.vectorStore.disconnect) {
+      await config.vectorStore.disconnect();
+    }
+  }
+});
+
+// src/cli/ingest.ts
+import { readFile, readdir, stat } from "fs/promises";
+import { extname, join, relative } from "path";
+import { defineCommand as defineCommand2 } from "citty";
+import consola2 from "consola";
+var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".md",
+  ".mdx",
+  ".txt",
+  ".csv",
+  ".json",
+  ".html",
+  ".htm",
+  ".xml",
+  ".yaml",
+  ".yml",
+  ".rst",
+  ".adoc",
+  ".tex",
+  ".log"
+]);
+async function collectFiles(dir) {
+  const files = [];
+  const entries = await readdir(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name.startsWith(".") || entry.name === "node_modules") {
+        continue;
+      }
+      files.push(...await collectFiles(full));
+    } else if (TEXT_EXTENSIONS.has(extname(entry.name).toLowerCase())) {
+      files.push(full);
+    }
+  }
+  return files;
+}
+var ingestCommand = defineCommand2({
+  meta: {
+    name: "ingest",
+    description: "Ingest documents into the vector store"
+  },
+  args: {
+    path: {
+      type: "positional",
+      description: "File or directory path to ingest",
+      required: true
+    }
+  },
+  async run({ args }) {
+    const target = args.path;
+    consola2.start(`Ingesting from ${target}...`);
+    const config = await loadConfig();
+    const pipeline = createPipeline(config);
+    const targetStat = await stat(target);
+    const files = targetStat.isDirectory() ? await collectFiles(target) : [target];
+    if (files.length === 0) {
+      consola2.warn("No text files found to ingest.");
+      return;
+    }
+    consola2.info(`Found ${files.length} file(s)`);
+    let totalChunks = 0;
+    for (const file of files) {
+      const content = await readFile(file, "utf-8");
+      const source = relative(process.cwd(), file);
+      const chunks = await pipeline.ingest(content, source);
+      totalChunks += chunks;
+      consola2.success(`${source} \u2192 ${chunks} chunks`);
+    }
+    consola2.success(`Done! ${totalChunks} total chunks ingested.`);
+    if (config.vectorStore.disconnect) {
+      await config.vectorStore.disconnect();
+    }
+  }
+});
+
+// src/cli/init.ts
+import { writeFile } from "fs/promises";
+import { resolve } from "path";
+import { defineCommand as defineCommand3 } from "citty";
+import consola3 from "consola";
+var EMBEDDING_PROVIDERS = [
+  {
+    label: "Gemini (Google)",
+    value: "gemini",
+    package: "@ragpipe/plugin-gemini",
+    importName: "geminiEmbedding"
+  }
+];
+var VECTORSTORE_PROVIDERS = [
+  {
+    label: "Supabase (pgvector)",
+    value: "supabase",
+    package: "@ragpipe/plugin-supabase",
+    importName: "supabaseVectorStore"
+  }
+];
+var GENERATION_PROVIDERS = [
+  {
+    label: "Gemini",
+    value: "gemini",
+    package: "@ragpipe/plugin-gemini",
+    importName: "geminiGeneration"
+  }
+];
+function generateConfig(embedding, vectorStore, generation) {
+  const imports = /* @__PURE__ */ new Map();
+  for (const p of [embedding, vectorStore, generation]) {
+    const existing = imports.get(p.package) ?? [];
+    if (!existing.includes(p.importName)) {
+      existing.push(p.importName);
+    }
+    imports.set(p.package, existing);
+  }
+  const importLines = [
+    'import { defineConfig } from "ragpipe";',
+    ...Array.from(imports.entries()).map(
+      ([pkg, names]) => `import { ${names.join(", ")} } from "${pkg}";`
+    )
+  ].join("\n");
+  return `${importLines}
+
+export default defineConfig({
+	embedding: ${embedding.importName}({
+		apiKey: process.env.${embedding.value === "gemini" ? "GEMINI_API_KEY" : "API_KEY"}!,
+	}),
+	vectorStore: ${vectorStore.importName}({
+		${vectorStore.value === "supabase" ? "databaseUrl: process.env.DATABASE_URL!," : ""}
+	}),
+	generation: ${generation.importName}({
+		apiKey: process.env.${generation.value === "gemini" ? "GEMINI_API_KEY" : "API_KEY"}!,
+	}),
+});
+`;
+}
+async function selectProvider(label, providers) {
+  if (providers.length === 1) {
+    consola3.info(`${label}: ${providers[0].label} (only available option)`);
+    return providers[0];
+  }
+  const selected = await consola3.prompt(label, {
+    type: "select",
+    options: providers.map((p) => p.label)
+  });
+  if (typeof selected === "symbol") {
+    throw new Error("Selection cancelled");
+  }
+  return providers.find((p) => p.label === selected) ?? providers[0];
+}
+var initCommand = defineCommand3({
+  meta: {
+    name: "init",
+    description: "Initialize a ragpipe project \u2014 scaffold ragpipe.config.ts"
+  },
+  async run() {
+    consola3.start("Initializing ragpipe project...\n");
+    const embedding = await selectProvider(
+      "Select an Embedding provider",
+      EMBEDDING_PROVIDERS
+    );
+    const vectorStore = await selectProvider(
+      "Select a VectorStore",
+      VECTORSTORE_PROVIDERS
+    );
+    const generation = await selectProvider(
+      "Select a Generation LLM",
+      GENERATION_PROVIDERS
+    );
+    const configContent = generateConfig(embedding, vectorStore, generation);
+    const configPath = resolve(process.cwd(), "ragpipe.config.ts");
+    await writeFile(configPath, configContent, "utf-8");
+    consola3.success("ragpipe.config.ts created");
+    const packages = /* @__PURE__ */ new Set();
+    for (const p of [embedding, vectorStore, generation]) {
+      packages.add(p.package);
+    }
+    consola3.info(
+      `Required packages: ragpipe, ${Array.from(packages).join(", ")}`
+    );
+    consola3.info(`Run: pnpm add ragpipe ${Array.from(packages).join(" ")}`);
+  }
+});
+
+// src/cli.ts
+var main = defineCommand4({
+  meta: {
+    name: "ragpipe",
+    description: "Pluggable TypeScript RAG toolkit",
+    version: "0.0.1"
+  },
+  subCommands: {
+    init: initCommand,
+    ingest: ingestCommand,
+    ask: askCommand
+  }
+});
+runMain(main);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "ragpipe",
-	"version": "0.0.1",
+	"version": "0.1.0-alpha.1",
 	"description": "Pluggable TypeScript RAG toolkit — defineConfig() one file, embed → search → generate.",
 	"type": "module",
 	"license": "MIT",
@@ -27,10 +27,13 @@
 			"require": "./dist/index.cjs"
 		}
 	},
+	"bin": {
+		"ragpipe": "./dist/cli.js"
+	},
 	"main": "./dist/index.cjs",
 	"module": "./dist/index.js",
 	"types": "./dist/index.d.ts",
-	"files": ["dist"],
+	"files": ["dist", "README.md"],
 	"scripts": {
 		"build": "tsup",
 		"dev": "tsup --watch",
@@ -55,6 +58,8 @@
 		"vitest": "^3.1.1"
 	},
 	"dependencies": {
-		"c12": "^2.0.4"
+		"c12": "^2.0.4",
+		"citty": "^0.1.6",
+		"consola": "^3.4.0"
 	}
 }