retriv 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026-present Harlan Wilton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,282 @@
+<h1>retriv</h1>
+
+[![npm version][npm-version-src]][npm-version-href]
+[![npm downloads][npm-downloads-src]][npm-downloads-href]
+[![License][license-src]][license-href]
+
+Index and retrieve Markdown documents with [up to 30% better recall](https://ragaboutit.com/hybrid-retrieval-for-enterprise-rag-when-to-use-bm25-vectors-or-both/) using hybrid search.
+
+Keyword search (BM25) finds exact matches but misses synonyms. Semantic search understands meaning but struggles with names, codes, and precise terminology. Hybrid search combines both using [Reciprocal Rank Fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf) - [research shows up to 5.8x improvement](https://www.researchgate.net/publication/399428523_Hybrid_Dense-Sparse_Retrieval_for_High-Recall_Information_Retrieval) on standard benchmarks.
+
+<p align="center">
+<table>
+<tbody>
+<td align="center">
+<sub>Made possible by my <a href="https://github.com/sponsors/harlan-zw">Sponsor Program 💖</a><br> Follow me <a href="https://twitter.com/harlan_zw">@harlan_zw</a> 🐦 • Join <a href="https://discord.gg/275MBUBvgP">Discord</a> for help</sub><br>
+</td>
+</tbody>
+</table>
+</p>
+
+## Features
+
+- 🔀 **[Hybrid search](#local-first-sqlite)** - BM25 + vectors with [RRF fusion](https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf) in a single SQLite file
+- 🔌 **[Swappable backends](#drivers)** - SQLite, LibSQL/Turso, pgvector, Upstash, Cloudflare Vectorize
+- 🧠 **[Any embedding provider](#embedding-providers)** - OpenAI, Google, Mistral, Cohere, Ollama, or local [Transformers.js](https://huggingface.co/docs/transformers.js)
+- ✂️ **[Automatic chunking](#with-chunking)** - Split large documents with configurable overlap
+- 📦 **[Unified interface](#api)** - Same `SearchProvider` API across all drivers
+
+## Installation
+
+```bash
+pnpm add retriv
+```
+
+## Usage
+
+### Local-First (SQLite)
+
+Single file with BM25 + vector search. No external services needed.
+
+```bash
+pnpm add @huggingface/transformers sqlite-vec
+```
+
+```ts
+import { createRetriv } from 'retriv'
+import sqlite from 'retriv/db/sqlite'
+import { transformers } from 'retriv/embeddings/transformers'
+
+const search = await createRetriv({
+  driver: sqlite({
+    path: './search.db',
+    embeddings: transformers(), // runs locally, no API key
+  }),
+})
+
+await search.index([
+  {
+    id: '1',
+    content: 'How to mass delete Gmail emails using filters',
+    metadata: { source: 'https://support.google.com/mail', title: 'Gmail Help' },
+  },
+  {
+    id: '2',
+    content: 'Setting up email forwarding rules in Outlook',
+    metadata: { source: 'https://support.microsoft.com', title: 'Outlook Help' },
+  },
+])
+
+const results = await search.search('bulk remove messages', { returnMetadata: true })
+// Finds #1 via semantic similarity even without keyword overlap
+// results[0].metadata.source → 'https://support.google.com/mail'
+```
+
+### Swap to Cloud Embeddings
+
+Same hybrid driver, better embeddings:
+
+```bash
+pnpm add @ai-sdk/openai ai sqlite-vec
+```
+
+```ts
+import { createRetriv } from 'retriv'
+import sqlite from 'retriv/db/sqlite'
+import { openai } from 'retriv/embeddings/openai'
+
+const search = await createRetriv({
+  driver: sqlite({
+    path: './search.db',
+    embeddings: openai(), // uses OPENAI_API_KEY env
+  }),
+})
+```
+
+### Swap to Cloud Vector DB
+
+For serverless or edge deployments:
+
+```bash
+pnpm add @libsql/client better-sqlite3 @ai-sdk/openai ai
+```
+
+```ts
+import { createRetriv } from 'retriv'
+import libsql from 'retriv/db/libsql'
+import sqliteFts from 'retriv/db/sqlite-fts'
+import { openai } from 'retriv/embeddings/openai'
+
+const search = await createRetriv({
+  driver: {
+    // Turso for vectors
+    vector: libsql({
+      url: 'libsql://your-db.turso.io',
+      authToken: process.env.TURSO_AUTH_TOKEN,
+      embeddings: openai(),
+    }),
+    // Local SQLite for BM25
+    keyword: sqliteFts({ path: './search.db' }),
+  },
+})
+```
+
+### With Chunking
+
+Automatically split large documents:
+
+```bash
+pnpm add @huggingface/transformers sqlite-vec
+```
+
+```ts
+import { createRetriv } from 'retriv'
+import sqlite from 'retriv/db/sqlite'
+import { transformers } from 'retriv/embeddings/transformers'
+
+const search = await createRetriv({
+  driver: sqlite({
+    path: './search.db',
+    embeddings: transformers(),
+  }),
+  chunking: {
+    chunkSize: 1000,
+    chunkOverlap: 200,
+  },
+})
+
+await search.index([
+  { id: 'doc-1', content: veryLongArticle },
+])
+
+const results = await search.search('specific topic')
+// Results include _chunk: { parentId, index, range }
+```
+
+## Drivers
+
+### Hybrid (Recommended)
+
+| Driver | Import | Peer Dependencies |
+|--------|--------|-------------------|
+| SQLite | `retriv/db/sqlite` | `sqlite-vec` (Node.js >= 22.5) |
+
+### Vector-Only (for composed hybrid)
+
+| Driver | Import | Peer Dependencies |
+|--------|--------|-------------------|
+| LibSQL | `retriv/db/libsql` | `@libsql/client` |
+| Upstash | `retriv/db/upstash` | `@upstash/vector` |
+| Cloudflare | `retriv/db/cloudflare` | — (uses Cloudflare bindings) |
+| pgvector | `retriv/db/pgvector` | `pg` |
+| sqlite-vec | `retriv/db/sqlite-vec` | `sqlite-vec` (Node.js >= 22.5) |
+
+### Keyword-Only (for composed hybrid)
+
+| Driver | Import | Peer Dependencies |
+|--------|--------|-------------------|
+| SQLite FTS5 | `retriv/db/sqlite-fts` | `better-sqlite3` |
+
+## Embedding Providers
+
+All vector drivers accept an `embeddings` config:
+
+| Provider | Import | Peer Dependencies |
+|----------|--------|-------------------|
+| OpenAI | `retriv/embeddings/openai` | `@ai-sdk/openai ai` |
+| Google | `retriv/embeddings/google` | `@ai-sdk/google ai` |
+| Mistral | `retriv/embeddings/mistral` | `@ai-sdk/mistral ai` |
+| Cohere | `retriv/embeddings/cohere` | `@ai-sdk/cohere ai` |
+| Ollama | `retriv/embeddings/ollama` | `ollama-ai-provider-v2 ai` |
+| Transformers | `retriv/embeddings/transformers` | `@huggingface/transformers` |
+
+```ts
+// Cloud providers (require API keys)
+openai({ model: 'text-embedding-3-small' })
+google({ model: 'text-embedding-004' })
+mistral({ model: 'mistral-embed' })
+cohere({ model: 'embed-english-v3.0' })
+
+// Local (no API key)
+ollama({ model: 'nomic-embed-text' })
+transformers({ model: 'Xenova/all-MiniLM-L6-v2' })
+```
+
+## API
+
+### SearchProvider Interface
+
+All drivers implement the same interface:
+
+```ts
+interface SearchProvider {
+  index: (docs: Document[]) => Promise<{ count: number }>
+  search: (query: string, options?: SearchOptions) => Promise<SearchResult[]>
+  remove?: (ids: string[]) => Promise<{ count: number }>
+  clear?: () => Promise<void>
+  close?: () => Promise<void>
+}
+```
+
+### Search Options
+
+```ts
+interface SearchOptions {
+  limit?: number // Max results (default varies by driver)
+  returnContent?: boolean // Include original content in results
+  returnMetadata?: boolean // Include metadata in results
+  returnMeta?: boolean // Include driver-specific _meta
+}
+```
+
+### Search Result
+
+```ts
+interface SearchResult {
+  id: string // Document ID
+  score: number // 0-1, higher is better
+  content?: string // If returnContent: true
+  metadata?: Record<string, any> // If returnMetadata: true
+  _chunk?: ChunkInfo // When chunking enabled
+  _meta?: SearchMeta // If returnMeta: true (driver-specific extras)
+}
+```
+
+## Benchmarks
+
+Retrieval accuracy on Nuxt documentation (639 docs):
+
+| Test Type | FTS | Vector | Hybrid |
+|-----------|-----|--------|--------|
+| Exact terminology (ports, config names) | 3/3 | 2/3 | 3/3 |
+| Doc retrieval (keyword overlap) | 3/3 | 2/3 | 3/3 |
+| Semantic queries (synonyms, no overlap) | 0/3 | 3/3 | 3/3 |
+| **Total** | **6/9 (67%)** | **7/9 (78%)** | **9/9 (100%)** |
+
+- **FTS** excels at exact terms but fails semantic queries ("reuse logic" → composables)
+- **Vector** understands meaning but misses precise terminology ("port 3000")
+- **Hybrid** combines both - never worse than either method alone
+
+Run locally: `pnpm test:eval`
+
+## Sponsors
+
+<p align="center">
+  <a href="https://raw.githubusercontent.com/harlan-zw/static/main/sponsors.svg">
+    <img src='https://raw.githubusercontent.com/harlan-zw/static/main/sponsors.svg'/>
+  </a>
+</p>
+
+## License
+
+Licensed under the [MIT license](https://github.com/harlan-zw/retriv/blob/main/LICENSE).
+
+<!-- Badges -->
+[npm-version-src]: https://img.shields.io/npm/v/retriv/latest.svg?style=flat&colorA=18181B&colorB=28CF8D
+[npm-version-href]: https://npmjs.com/package/retriv
+
+[npm-downloads-src]: https://img.shields.io/npm/dm/retriv.svg?style=flat&colorA=18181B&colorB=28CF8D
+[npm-downloads-href]: https://npmjs.com/package/retriv
+
+[license-src]: https://img.shields.io/github/license/harlan-zw/retriv.svg?style=flat&colorA=18181B&colorB=28CF8D
+[license-href]: https://github.com/harlan-zw/retriv/blob/main/LICENSE

package/dist/_chunks/split-text.mjs

@@ -0,0 +1,85 @@
+const MARKDOWN_SEPARATORS = [
+	"\n## ",
+	"\n### ",
+	"\n#### ",
+	"\n##### ",
+	"\n###### ",
+	"```\n\n",
+	"\n\n***\n\n",
+	"\n\n---\n\n",
+	"\n\n___\n\n",
+	"\n\n",
+	"\n",
+	" ",
+	""
+];
+function offsetToLine(text, offset) {
+	let line = 1;
+	for (let i = 0; i < offset && i < text.length; i++) if (text[i] === "\n") line++;
+	return line;
+}
+function splitText(text, options = {}) {
+	const { chunkSize = 1e3, chunkOverlap = 200, separators = MARKDOWN_SEPARATORS } = options;
+	if (text.length <= chunkSize) {
+		const endLine = offsetToLine(text, text.length);
+		return [{
+			text,
+			index: 0,
+			range: [0, text.length],
+			lines: [1, endLine]
+		}];
+	}
+	return mergeChunks(splitRecursive(text, chunkSize, separators), chunkSize, chunkOverlap, text);
+}
+function splitRecursive(text, chunkSize, separators) {
+	if (text.length <= chunkSize || separators.length === 0) return [text];
+	const separator = separators.find((sep) => sep === "" || text.includes(sep));
+	if (!separator && separator !== "") return [text];
+	const parts = separator === "" ? [...text] : text.split(separator);
+	const results = [];
+	for (let i = 0; i < parts.length; i++) {
+		const part = parts[i];
+		const withSep = i < parts.length - 1 && separator !== "" ? part + separator : part;
+		if (withSep.length <= chunkSize) results.push(withSep);
+		else {
+			const subParts = splitRecursive(withSep, chunkSize, separators.slice(1));
+			results.push(...subParts);
+		}
+	}
+	return results;
+}
+function mergeChunks(parts, chunkSize, chunkOverlap, originalText) {
+	const chunks = [];
+	let current = "";
+	let currentStart = 0;
+	for (const part of parts) if (current.length + part.length <= chunkSize) current += part;
+	else {
+		if (current) {
+			const start = originalText.indexOf(current, currentStart);
+			const actualStart = start >= 0 ? start : currentStart;
+			const actualEnd = actualStart + current.length;
+			chunks.push({
+				text: current,
+				index: chunks.length,
+				range: [actualStart, actualEnd],
+				lines: [offsetToLine(originalText, actualStart), offsetToLine(originalText, actualEnd)]
+			});
+			currentStart = Math.max(0, actualStart + current.length - chunkOverlap);
+		}
+		if (chunkOverlap > 0 && current.length > chunkOverlap) current = current.slice(-chunkOverlap) + part;
+		else current = part;
+	}
+	if (current) {
+		const start = originalText.indexOf(current, currentStart);
+		const actualStart = start >= 0 ? start : currentStart;
+		const actualEnd = start >= 0 ? start + current.length : originalText.length;
+		chunks.push({
+			text: current,
+			index: chunks.length,
+			range: [actualStart, actualEnd],
+			lines: [offsetToLine(originalText, actualStart), offsetToLine(originalText, actualEnd)]
+		});
+	}
+	return chunks;
+}
+export { splitText as t };

package/dist/db/cloudflare.d.mts

@@ -0,0 +1,36 @@
+import { BaseDriverConfig, EmbeddingConfig, SearchProvider } from "../types.mjs";
+
+//#region src/db/cloudflare.d.ts
+interface VectorizeIndexBinding {
+  query: (vector: number[], options?: any) => Promise<{
+    matches: any[];
+    count?: number;
+  }>;
+  insert: (vectors: any[]) => Promise<void>;
+  upsert: (vectors: any[]) => Promise<void>;
+  deleteByIds: (ids: string[]) => Promise<void>;
+}
+interface CloudflareConfig extends BaseDriverConfig {
+  /** Cloudflare Vectorize binding instance */
+  binding: VectorizeIndexBinding;
+  /** Embedding provider from retriv/embeddings/ */
+  embeddings: EmbeddingConfig;
+}
+/**
+ * Create a Cloudflare Vectorize search provider
+ * For use in Cloudflare Workers at runtime
+ *
+ * @example
+ * ```ts
+ * import { cloudflare } from 'retriv/db/cloudflare'
+ * import { openai } from 'retriv/embeddings/openai'
+ *
+ * const db = await cloudflare({
+ *   binding: env.VECTORIZE,
+ *   embeddings: openai({ model: 'text-embedding-3-small' }),
+ * })
+ * ```
+ */
+declare function cloudflare(config: CloudflareConfig): Promise<SearchProvider>;
+//#endregion
+export { CloudflareConfig, cloudflare, cloudflare as default };

package/dist/db/cloudflare.mjs

@@ -0,0 +1,55 @@
+import { resolveEmbedding } from "../embeddings/resolve.mjs";
+async function cloudflare(config) {
+	const { binding } = config;
+	if (!binding) throw new Error("[cloudflare] binding is required");
+	if (!config.embeddings) throw new Error("[cloudflare] embeddings is required");
+	const { embedder } = await resolveEmbedding(config.embeddings);
+	return {
+		async index(docs) {
+			if (docs.length === 0) return { count: 0 };
+			const embeddings = await embedder(docs.map((d) => d.content));
+			if (embeddings.length !== docs.length) throw new Error(`Embedding count mismatch: expected ${docs.length}, got ${embeddings.length}`);
+			const vectors = docs.map((doc, i) => ({
+				id: doc.id,
+				values: embeddings[i],
+				metadata: {
+					...doc.metadata,
+					_content: doc.content
+				}
+			}));
+			await binding.upsert(vectors);
+			return { count: docs.length };
+		},
+		async search(query, options = {}) {
+			const { limit = 10, returnContent = false, returnMetadata = true } = options;
+			const [embedding] = await embedder([query]);
+			if (!embedding) throw new Error("Failed to generate query embedding");
+			return ((await binding.query(embedding, {
+				topK: limit,
+				returnValues: false,
+				returnMetadata: true
+			})).matches || []).map((m) => {
+				const result = {
+					id: m.id,
+					score: Math.max(0, Math.min(1, m.score))
+				};
+				if (returnContent && m.metadata?._content) result.content = m.metadata._content;
+				if (returnMetadata && m.metadata) {
+					const { _content, ...rest } = m.metadata;
+					if (Object.keys(rest).length > 0) result.metadata = rest;
+				}
+				return result;
+			});
+		},
+		async remove(ids) {
+			await binding.deleteByIds(ids);
+			return { count: ids.length };
+		},
+		async clear() {
+			throw new Error("[cloudflare] clear() is not supported - use wrangler CLI instead");
+		},
+		async close() {}
+	};
+}
+var cloudflare_default = cloudflare;
+export { cloudflare, cloudflare_default as default };

package/dist/db/libsql.d.mts

@@ -0,0 +1,30 @@
+import { BaseDriverConfig, EmbeddingConfig, SearchProvider } from "../types.mjs";
+
+//#region src/db/libsql.d.ts
+interface LibsqlConfig extends BaseDriverConfig {
+  /** Database URL (file:path.db for local, libsql://... for remote) */
+  url?: string;
+  /** Auth token for remote LibSQL/Turso */
+  authToken?: string;
+  /** Embedding provider from retriv/embeddings/ */
+  embeddings: EmbeddingConfig;
+}
+/**
+ * Create a LibSQL/Turso vector search provider
+ * Supports local SQLite files and remote Turso databases
+ *
+ * @example
+ * ```ts
+ * import { libsql } from 'retriv/db/libsql'
+ * import { openai } from 'retriv/embeddings/openai'
+ *
+ * const db = await libsql({
+ *   url: 'libsql://your-db.turso.io',
+ *   authToken: process.env.TURSO_AUTH_TOKEN,
+ *   embeddings: openai({ model: 'text-embedding-3-small' }),
+ * })
+ * ```
+ */
+declare function libsql(config: LibsqlConfig): Promise<SearchProvider>;
+//#endregion
+export { LibsqlConfig, libsql as default, libsql };

package/dist/db/libsql.mjs

@@ -0,0 +1,87 @@
+import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { createClient } from "@libsql/client";
+async function libsql(config) {
+	const url = config.url || config.path || "file:vectors.db";
+	const { authToken } = config;
+	if (!config.embeddings) throw new Error("[libsql] embeddings is required");
+	const { embedder, dimensions } = await resolveEmbedding(config.embeddings);
+	const client = createClient({
+		url,
+		...authToken && { authToken }
+	});
+	await client.execute(`
+    CREATE TABLE IF NOT EXISTS vectors (
+      id TEXT PRIMARY KEY,
+      content TEXT,
+      metadata TEXT,
+      embedding F32_BLOB(${dimensions})
+    )
+  `);
+	return {
+		async index(docs) {
+			if (docs.length === 0) return { count: 0 };
+			const embeddings = await embedder(docs.map((d) => d.content));
+			if (embeddings.length !== docs.length) throw new Error(`Embedding count mismatch: expected ${docs.length}, got ${embeddings.length}`);
+			for (let i = 0; i < docs.length; i++) {
+				const doc = docs[i];
+				const vector = embeddings[i];
+				const vectorStr = JSON.stringify(vector);
+				await client.execute({
+					sql: `
+            INSERT OR REPLACE INTO vectors (id, content, metadata, embedding)
+            VALUES (?, ?, ?, vector(?))
+          `,
+					args: [
+						doc.id,
+						doc.content,
+						doc.metadata ? JSON.stringify(doc.metadata) : null,
+						vectorStr
+					]
+				});
+			}
+			return { count: docs.length };
+		},
+		async search(query, options = {}) {
+			const { limit = 10, returnContent = false, returnMetadata = true } = options;
+			const [embedding] = await embedder([query]);
+			if (!embedding) throw new Error("Failed to generate query embedding");
+			const vectorStr = JSON.stringify(embedding);
+			return ((await client.execute({
+				sql: `
+          SELECT
+            id,
+            content,
+            metadata,
+            vector_distance_cos(embedding, vector32(?)) as distance
+          FROM vectors
+          ORDER BY distance
+          LIMIT ?
+        `,
+				args: [vectorStr, limit]
+			})).rows || []).map((row) => {
+				const result = {
+					id: row.id,
+					score: Math.max(0, 1 - row.distance)
+				};
+				if (returnContent && row.content) result.content = row.content;
+				if (returnMetadata && row.metadata) result.metadata = JSON.parse(row.metadata);
+				return result;
+			});
+		},
+		async remove(ids) {
+			for (const id of ids) await client.execute({
+				sql: "DELETE FROM vectors WHERE id = ?",
+				args: [id]
+			});
+			return { count: ids.length };
+		},
+		async clear() {
+			await client.execute("DELETE FROM vectors");
+		},
+		async close() {
+			client.close();
+		}
+	};
+}
+var libsql_default = libsql;
+export { libsql_default as default, libsql };

package/dist/db/pgvector.d.mts

@@ -0,0 +1,30 @@
+import { BaseDriverConfig, EmbeddingConfig, SearchProvider } from "../types.mjs";
+
+//#region src/db/pgvector.d.ts
+interface PgvectorConfig extends BaseDriverConfig {
+  /** PostgreSQL connection URL */
+  url: string;
+  /** Table name for vectors */
+  table?: string;
+  /** Embedding provider from retriv/embeddings/ */
+  embeddings: EmbeddingConfig;
+  /** Distance metric */
+  metric?: 'cosine' | 'euclidean' | 'inner_product';
+}
+/**
+ * Create a PostgreSQL pgvector search provider
+ *
+ * @example
+ * ```ts
+ * import { pgvector } from 'retriv/db/pgvector'
+ * import { openai } from 'retriv/embeddings/openai'
+ *
+ * const db = await pgvector({
+ *   url: process.env.DATABASE_URL,
+ *   embeddings: openai({ model: 'text-embedding-3-small' }),
+ * })
+ * ```
+ */
+declare function pgvector(config: PgvectorConfig): Promise<SearchProvider>;
+//#endregion
+export { PgvectorConfig, pgvector as default, pgvector };