ragpipe 0.0.1

package/dist/index.cjs

@@ -0,0 +1,166 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  createPipeline: () => createPipeline,
+  createRateLimitedEmbedder: () => createRateLimitedEmbedder,
+  defaultChunker: () => defaultChunker,
+  defineConfig: () => defineConfig,
+  loadConfig: () => loadConfig
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/config.ts
+var import_c12 = require("c12");
+function defineConfig(config) {
+  return config;
+}
+async function loadConfig(overrides) {
+  const { config } = await (0, import_c12.loadConfig)({
+    name: "ragpipe",
+    defaults: overrides
+  });
+  if (!config) {
+    throw new Error(
+      "No ragpipe config found. Create a ragpipe.config.ts or pass config directly."
+    );
+  }
+  const cfg = config;
+  if (!cfg.embedding) {
+    throw new Error("ragpipe config is missing 'embedding' plugin.");
+  }
+  if (!cfg.vectorStore) {
+    throw new Error("ragpipe config is missing 'vectorStore' plugin.");
+  }
+  if (!cfg.generation) {
+    throw new Error("ragpipe config is missing 'generation' plugin.");
+  }
+  return cfg;
+}
+
+// src/chunker.ts
+var DEFAULT_CHUNK_SIZE = 400;
+var DEFAULT_OVERLAP = 50;
+function defaultChunker(options) {
+  const chunkSize = options?.chunkSize ?? DEFAULT_CHUNK_SIZE;
+  const overlap = options?.overlap ?? DEFAULT_OVERLAP;
+  return {
+    name: "default",
+    chunk(text, source) {
+      const paragraphs = text.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+      const chunks = [];
+      let current = "";
+      for (const paragraph of paragraphs) {
+        if (current && current.length + paragraph.length + 1 > chunkSize) {
+          chunks.push({ source, content: current });
+          const overlapSlice = current.slice(-overlap);
+          current = overlapSlice + paragraph;
+        } else {
+          current = current ? `${current}
+${paragraph}` : paragraph;
+        }
+      }
+      if (current.trim()) {
+        chunks.push({ source, content: current });
+      }
+      if (chunks.length === 0 && text.trim()) {
+        chunks.push({ source, content: text.trim() });
+      }
+      return chunks;
+    }
+  };
+}
+
+// src/rate-limiter.ts
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function createRateLimitedEmbedder(plugin, delayMs) {
+  const delay = delayMs ?? plugin.rateLimit?.delayMs ?? 0;
+  let lastCall = 0;
+  async function throttle() {
+    if (delay <= 0) return;
+    const now = Date.now();
+    const elapsed = now - lastCall;
+    if (elapsed < delay) {
+      await sleep(delay - elapsed);
+    }
+    lastCall = Date.now();
+  }
+  return {
+    name: plugin.name,
+    dimensions: plugin.dimensions,
+    rateLimit: plugin.rateLimit,
+    async embed(text) {
+      await throttle();
+      return plugin.embed(text);
+    },
+    async embedMany(texts) {
+      if (plugin.embedMany) {
+        await throttle();
+        return plugin.embedMany(texts);
+      }
+      const results = [];
+      for (const text of texts) {
+        await throttle();
+        results.push(await plugin.embed(text));
+      }
+      return results;
+    }
+  };
+}
+
+// src/pipeline.ts
+function createPipeline(config) {
+  const chunker = config.chunker ?? defaultChunker();
+  const embedder = config.embedding.rateLimit ? createRateLimitedEmbedder(config.embedding) : config.embedding;
+  return {
+    async ingest(text, source) {
+      const chunks = chunker.chunk(text, source);
+      for (const chunk of chunks) {
+        const vector = await embedder.embed(chunk.content);
+        await config.vectorStore.upsert(chunk.source, chunk.content, vector);
+      }
+      return chunks.length;
+    },
+    async search(query, topK = 5) {
+      const vector = await embedder.embed(query);
+      return config.vectorStore.search(vector, topK);
+    },
+    async ask(query, topK = 5) {
+      const chunks = await this.search(query, topK);
+      const context = chunks.map((c) => `[${c.source}]
+${c.content}`).join("\n\n---\n\n");
+      const answer = await config.generation.generate(query, context, {
+        systemPrompt: config.systemPrompt
+      });
+      return { answer, sources: chunks };
+    }
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  createPipeline,
+  createRateLimitedEmbedder,
+  defaultChunker,
+  defineConfig,
+  loadConfig
+});

package/dist/index.d.cts

@@ -0,0 +1,79 @@
+interface SearchResult {
+    source: string;
+    content: string;
+    score: number;
+}
+interface DocumentChunk {
+    source: string;
+    content: string;
+}
+interface AskResult {
+    answer: string;
+    sources: SearchResult[];
+}
+interface EmbeddingPlugin {
+    readonly name: string;
+    readonly dimensions: number;
+    embed(text: string): Promise<number[]>;
+    embedMany?(texts: string[]): Promise<number[][]>;
+    rateLimit?: {
+        delayMs: number;
+    };
+}
+interface VectorStorePlugin {
+    readonly name: string;
+    search(vector: number[], topK: number): Promise<SearchResult[]>;
+    upsert(source: string, content: string, vector: number[]): Promise<void>;
+    clear?(): Promise<void>;
+    disconnect?(): Promise<void>;
+}
+interface GenerationPlugin {
+    readonly name: string;
+    generate(question: string, context: string, options?: {
+        history?: string;
+        systemPrompt?: string;
+    }): Promise<string>;
+    generateStream?(question: string, context: string, options?: {
+        history?: string;
+        systemPrompt?: string;
+    }): AsyncIterable<string>;
+}
+interface ChunkerPlugin {
+    readonly name: string;
+    chunk(text: string, source: string): DocumentChunk[];
+}
+interface RagpipeConfig {
+    embedding: EmbeddingPlugin;
+    vectorStore: VectorStorePlugin;
+    generation: GenerationPlugin;
+    chunker?: ChunkerPlugin;
+    systemPrompt?: string;
+}
+
+declare function defineConfig(config: RagpipeConfig): RagpipeConfig;
+declare function loadConfig(overrides?: Partial<RagpipeConfig>): Promise<RagpipeConfig>;
+
+interface Pipeline {
+    ingest(text: string, source: string): Promise<number>;
+    search(query: string, topK?: number): Promise<SearchResult[]>;
+    ask(query: string, topK?: number): Promise<AskResult>;
+}
+declare function createPipeline(config: RagpipeConfig): Pipeline;
+
+interface DefaultChunkerOptions {
+    chunkSize?: number;
+    overlap?: number;
+}
+/**
+ * Splits text into chunks by paragraph boundaries, respecting a max character size.
+ * Adjacent chunks overlap by `overlap` characters to preserve context at boundaries.
+ */
+declare function defaultChunker(options?: DefaultChunkerOptions): ChunkerPlugin;
+
+/**
+ * Wraps an EmbeddingPlugin to enforce rate limiting between calls.
+ * Uses the plugin's `rateLimit.delayMs` if present, or a custom delay.
+ */
+declare function createRateLimitedEmbedder(plugin: EmbeddingPlugin, delayMs?: number): EmbeddingPlugin;
+
+export { type AskResult, type ChunkerPlugin, type DefaultChunkerOptions, type DocumentChunk, type EmbeddingPlugin, type GenerationPlugin, type Pipeline, type RagpipeConfig, type SearchResult, type VectorStorePlugin, createPipeline, createRateLimitedEmbedder, defaultChunker, defineConfig, loadConfig };

package/dist/index.d.ts

@@ -0,0 +1,79 @@
+interface SearchResult {
+    source: string;
+    content: string;
+    score: number;
+}
+interface DocumentChunk {
+    source: string;
+    content: string;
+}
+interface AskResult {
+    answer: string;
+    sources: SearchResult[];
+}
+interface EmbeddingPlugin {
+    readonly name: string;
+    readonly dimensions: number;
+    embed(text: string): Promise<number[]>;
+    embedMany?(texts: string[]): Promise<number[][]>;
+    rateLimit?: {
+        delayMs: number;
+    };
+}
+interface VectorStorePlugin {
+    readonly name: string;
+    search(vector: number[], topK: number): Promise<SearchResult[]>;
+    upsert(source: string, content: string, vector: number[]): Promise<void>;
+    clear?(): Promise<void>;
+    disconnect?(): Promise<void>;
+}
+interface GenerationPlugin {
+    readonly name: string;
+    generate(question: string, context: string, options?: {
+        history?: string;
+        systemPrompt?: string;
+    }): Promise<string>;
+    generateStream?(question: string, context: string, options?: {
+        history?: string;
+        systemPrompt?: string;
+    }): AsyncIterable<string>;
+}
+interface ChunkerPlugin {
+    readonly name: string;
+    chunk(text: string, source: string): DocumentChunk[];
+}
+interface RagpipeConfig {
+    embedding: EmbeddingPlugin;
+    vectorStore: VectorStorePlugin;
+    generation: GenerationPlugin;
+    chunker?: ChunkerPlugin;
+    systemPrompt?: string;
+}
+
+declare function defineConfig(config: RagpipeConfig): RagpipeConfig;
+declare function loadConfig(overrides?: Partial<RagpipeConfig>): Promise<RagpipeConfig>;
+
+interface Pipeline {
+    ingest(text: string, source: string): Promise<number>;
+    search(query: string, topK?: number): Promise<SearchResult[]>;
+    ask(query: string, topK?: number): Promise<AskResult>;
+}
+declare function createPipeline(config: RagpipeConfig): Pipeline;
+
+interface DefaultChunkerOptions {
+    chunkSize?: number;
+    overlap?: number;
+}
+/**
+ * Splits text into chunks by paragraph boundaries, respecting a max character size.
+ * Adjacent chunks overlap by `overlap` characters to preserve context at boundaries.
+ */
+declare function defaultChunker(options?: DefaultChunkerOptions): ChunkerPlugin;
+
+/**
+ * Wraps an EmbeddingPlugin to enforce rate limiting between calls.
+ * Uses the plugin's `rateLimit.delayMs` if present, or a custom delay.
+ */
+declare function createRateLimitedEmbedder(plugin: EmbeddingPlugin, delayMs?: number): EmbeddingPlugin;
+
+export { type AskResult, type ChunkerPlugin, type DefaultChunkerOptions, type DocumentChunk, type EmbeddingPlugin, type GenerationPlugin, type Pipeline, type RagpipeConfig, type SearchResult, type VectorStorePlugin, createPipeline, createRateLimitedEmbedder, defaultChunker, defineConfig, loadConfig };

package/dist/index.js

@@ -0,0 +1,135 @@
+// src/config.ts
+import { loadConfig as c12LoadConfig } from "c12";
+function defineConfig(config) {
+  return config;
+}
+async function loadConfig(overrides) {
+  const { config } = await c12LoadConfig({
+    name: "ragpipe",
+    defaults: overrides
+  });
+  if (!config) {
+    throw new Error(
+      "No ragpipe config found. Create a ragpipe.config.ts or pass config directly."
+    );
+  }
+  const cfg = config;
+  if (!cfg.embedding) {
+    throw new Error("ragpipe config is missing 'embedding' plugin.");
+  }
+  if (!cfg.vectorStore) {
+    throw new Error("ragpipe config is missing 'vectorStore' plugin.");
+  }
+  if (!cfg.generation) {
+    throw new Error("ragpipe config is missing 'generation' plugin.");
+  }
+  return cfg;
+}
+
+// src/chunker.ts
+var DEFAULT_CHUNK_SIZE = 400;
+var DEFAULT_OVERLAP = 50;
+function defaultChunker(options) {
+  const chunkSize = options?.chunkSize ?? DEFAULT_CHUNK_SIZE;
+  const overlap = options?.overlap ?? DEFAULT_OVERLAP;
+  return {
+    name: "default",
+    chunk(text, source) {
+      const paragraphs = text.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+      const chunks = [];
+      let current = "";
+      for (const paragraph of paragraphs) {
+        if (current && current.length + paragraph.length + 1 > chunkSize) {
+          chunks.push({ source, content: current });
+          const overlapSlice = current.slice(-overlap);
+          current = overlapSlice + paragraph;
+        } else {
+          current = current ? `${current}
+${paragraph}` : paragraph;
+        }
+      }
+      if (current.trim()) {
+        chunks.push({ source, content: current });
+      }
+      if (chunks.length === 0 && text.trim()) {
+        chunks.push({ source, content: text.trim() });
+      }
+      return chunks;
+    }
+  };
+}
+
+// src/rate-limiter.ts
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function createRateLimitedEmbedder(plugin, delayMs) {
+  const delay = delayMs ?? plugin.rateLimit?.delayMs ?? 0;
+  let lastCall = 0;
+  async function throttle() {
+    if (delay <= 0) return;
+    const now = Date.now();
+    const elapsed = now - lastCall;
+    if (elapsed < delay) {
+      await sleep(delay - elapsed);
+    }
+    lastCall = Date.now();
+  }
+  return {
+    name: plugin.name,
+    dimensions: plugin.dimensions,
+    rateLimit: plugin.rateLimit,
+    async embed(text) {
+      await throttle();
+      return plugin.embed(text);
+    },
+    async embedMany(texts) {
+      if (plugin.embedMany) {
+        await throttle();
+        return plugin.embedMany(texts);
+      }
+      const results = [];
+      for (const text of texts) {
+        await throttle();
+        results.push(await plugin.embed(text));
+      }
+      return results;
+    }
+  };
+}
+
+// src/pipeline.ts
+function createPipeline(config) {
+  const chunker = config.chunker ?? defaultChunker();
+  const embedder = config.embedding.rateLimit ? createRateLimitedEmbedder(config.embedding) : config.embedding;
+  return {
+    async ingest(text, source) {
+      const chunks = chunker.chunk(text, source);
+      for (const chunk of chunks) {
+        const vector = await embedder.embed(chunk.content);
+        await config.vectorStore.upsert(chunk.source, chunk.content, vector);
+      }
+      return chunks.length;
+    },
+    async search(query, topK = 5) {
+      const vector = await embedder.embed(query);
+      return config.vectorStore.search(vector, topK);
+    },
+    async ask(query, topK = 5) {
+      const chunks = await this.search(query, topK);
+      const context = chunks.map((c) => `[${c.source}]
+${c.content}`).join("\n\n---\n\n");
+      const answer = await config.generation.generate(query, context, {
+        systemPrompt: config.systemPrompt
+      });
+      return { answer, sources: chunks };
+    }
+  };
+}
+export {
+  createPipeline,
+  createRateLimitedEmbedder,
+  defaultChunker,
+  defineConfig,
+  loadConfig
+};

package/package.json

@@ -0,0 +1,60 @@
+{
+	"name": "ragpipe",
+	"version": "0.0.1",
+	"description": "Pluggable TypeScript RAG toolkit — defineConfig() one file, embed → search → generate.",
+	"type": "module",
+	"license": "MIT",
+	"author": {
+		"name": "yungblud",
+		"url": "https://github.com/yungblud"
+	},
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/yungblud/ragpipe",
+		"directory": "packages/ragpipe"
+	},
+	"bugs": {
+		"url": "https://github.com/yungblud/ragpipe/issues"
+	},
+	"homepage": "https://github.com/yungblud/ragpipe#readme",
+	"publishConfig": {
+		"access": "public"
+	},
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js",
+			"require": "./dist/index.cjs"
+		}
+	},
+	"main": "./dist/index.cjs",
+	"module": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"files": ["dist"],
+	"scripts": {
+		"build": "tsup",
+		"dev": "tsup --watch",
+		"typecheck": "tsc --noEmit",
+		"test": "vitest run",
+		"test:watch": "vitest",
+		"test:coverage": "vitest run --coverage"
+	},
+	"keywords": [
+		"rag",
+		"retrieval-augmented-generation",
+		"embeddings",
+		"vector-search",
+		"llm",
+		"ai",
+		"typescript",
+		"plugin"
+	],
+	"devDependencies": {
+		"tsup": "^8.4.0",
+		"typescript": "^5.8.3",
+		"vitest": "^3.1.1"
+	},
+	"dependencies": {
+		"c12": "^2.0.4"
+	}
+}