retriv 0.0.1 → 0.2.0

package/README.md

@@ -244,18 +244,18 @@ interface SearchResult {
 
 ## Benchmarks
 
-Retrieval accuracy on Nuxt documentation (639 docs):
+Retrieval accuracy on Nuxt documentation (2,817 chunks):
 
 | Test Type | FTS | Vector | Hybrid |
 |-----------|-----|--------|--------|
-| Exact terminology (ports, config names) | 3/3 | 2/3 | 3/3 |
-| Doc retrieval (keyword overlap) | 3/3 | 2/3 | 3/3 |
-| Semantic queries (synonyms, no overlap) | 0/3 | 3/3 | 3/3 |
-| **Total** | **6/9 (67%)** | **7/9 (78%)** | **9/9 (100%)** |
+| Exact terminology (ports, config names) | 7/7 | 5/7 | 7/7 |
+| Doc retrieval (keyword overlap) | 0/7 | 5/7 | 5/7 |
+| Semantic queries (synonyms, no overlap) | 1/6 | 5/6 | 5/6 |
+| **Total** | **8/20 (40%)** | **15/20 (75%)** | **17/20 (85%)** |
 
 - **FTS** excels at exact terms but fails semantic queries ("reuse logic" → composables)
-- **Vector** understands meaning but misses precise terminology ("port 3000")
-- **Hybrid** combines both - never worse than either method alone
+- **Vector** understands meaning but misses precise terminology (".global" suffix)
+- **Hybrid** combines both - best overall recall across query types
 
 Run locally: `pnpm test:eval`
 

package/dist/db/sqlite-fts.d.mts

@@ -8,6 +8,7 @@ interface SqliteFtsConfig extends BaseDriverConfig {
 /**
  * Create a SQLite FTS5 full-text search provider
  * Uses the built-in FTS5 extension for fast BM25-based search
+ * Requires Node.js >= 22.5
  *
  * @example
  * ```ts

package/dist/db/sqlite-fts.mjs

@@ -1,10 +1,11 @@
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
-import Database from "better-sqlite3";
 async function sqliteFts(config = {}) {
 	const dbPath = config.path || ":memory:";
+	const nodeSqlite = globalThis.process?.getBuiltinModule?.("node:sqlite");
+	if (!nodeSqlite) throw new Error("node:sqlite not available. Requires Node.js >= 22.5");
 	if (dbPath !== ":memory:") mkdirSync(dirname(dbPath), { recursive: true });
-	const db = new Database(dbPath);
+	const db = new nodeSqlite.DatabaseSync(dbPath);
 	db.exec(`
     CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
       id,
@@ -15,14 +16,18 @@ async function sqliteFts(config = {}) {
   `);
 	return {
 		async index(docs) {
-			const insert = db.prepare(`
-        INSERT OR REPLACE INTO documents_fts (id, content, metadata)
-        VALUES (?, ?, ?)
-      `);
-			db.transaction((documents) => {
-				for (const doc of documents) insert.run(doc.id, doc.content, doc.metadata ? JSON.stringify(doc.metadata) : null);
-			})(docs);
-			return { count: docs.length };
+			db.prepare("BEGIN").run();
+			try {
+				for (const doc of docs) {
+					db.prepare("DELETE FROM documents_fts WHERE id = ?").run(doc.id);
+					db.prepare("INSERT INTO documents_fts (id, content, metadata) VALUES (?, ?, ?)").run(doc.id, doc.content, doc.metadata ? JSON.stringify(doc.metadata) : null);
+				}
+				db.prepare("COMMIT").run();
+				return { count: docs.length };
+			} catch (error) {
+				db.prepare("ROLLBACK").run();
+				throw error;
+			}
 		},
 		async search(query, options = {}) {
 			const { limit = 10, returnContent = false, returnMetadata = true } = options;
@@ -50,17 +55,21 @@ async function sqliteFts(config = {}) {
 			});
 		},
 		async remove(ids) {
-			const del = db.prepare("DELETE FROM documents_fts WHERE id = ?");
-			db.transaction((docIds) => {
-				for (const id of docIds) del.run(id);
-			})(ids);
-			return { count: ids.length };
+			db.prepare("BEGIN").run();
+			try {
+				for (const id of ids) db.prepare("DELETE FROM documents_fts WHERE id = ?").run(id);
+				db.prepare("COMMIT").run();
+				return { count: ids.length };
+			} catch (error) {
+				db.prepare("ROLLBACK").run();
+				throw error;
+			}
 		},
 		async clear() {
 			db.exec("DELETE FROM documents_fts");
 		},
 		async close() {
-			db.close();
+			db.close?.();
 		}
 	};
 }

package/dist/embeddings/cohere.mjs

@@ -1,3 +1,4 @@
+import { getModelDimensions } from "./model-info.mjs";
 import { embed, embedMany } from "ai";
 import { createCohere } from "@ai-sdk/cohere";
 function cohere(options = {}) {
@@ -9,11 +10,14 @@ function cohere(options = {}) {
 			apiKey,
 			baseURL: baseUrl
 		}).textEmbeddingModel(model);
-		const { embedding: testEmbedding } = await embed({
-			model: embeddingModel,
-			value: "test"
-		});
-		const dimensions = testEmbedding.length;
+		let dimensions = getModelDimensions(model);
+		if (!dimensions) {
+			const { embedding } = await embed({
+				model: embeddingModel,
+				value: "test"
+			});
+			dimensions = embedding.length;
+		}
 		const embedder = async (texts) => {
 			if (texts.length === 0) return [];
 			if (texts.length === 1) {

package/dist/embeddings/google.mjs

@@ -1,3 +1,4 @@
+import { getModelDimensions } from "./model-info.mjs";
 import { embed, embedMany } from "ai";
 import { createGoogleGenerativeAI } from "@ai-sdk/google";
 function google(options = {}) {
@@ -9,11 +10,14 @@ function google(options = {}) {
 			apiKey,
 			baseURL: baseUrl
 		}).textEmbeddingModel(model);
-		const { embedding: testEmbedding } = await embed({
-			model: embeddingModel,
-			value: "test"
-		});
-		const dimensions = testEmbedding.length;
+		let dimensions = getModelDimensions(model);
+		if (!dimensions) {
+			const { embedding } = await embed({
+				model: embeddingModel,
+				value: "test"
+			});
+			dimensions = embedding.length;
+		}
 		const embedder = async (texts) => {
 			if (texts.length === 0) return [];
 			if (texts.length === 1) {

package/dist/embeddings/mistral.mjs

@@ -1,3 +1,4 @@
+import { getModelDimensions } from "./model-info.mjs";
 import { embed, embedMany } from "ai";
 import { createMistral } from "@ai-sdk/mistral";
 function mistral(options = {}) {
@@ -9,11 +10,14 @@ function mistral(options = {}) {
 			apiKey,
 			baseURL: baseUrl
 		}).textEmbeddingModel(model);
-		const { embedding: testEmbedding } = await embed({
-			model: embeddingModel,
-			value: "test"
-		});
-		const dimensions = testEmbedding.length;
+		let dimensions = getModelDimensions(model);
+		if (!dimensions) {
+			const { embedding } = await embed({
+				model: embeddingModel,
+				value: "test"
+			});
+			dimensions = embedding.length;
+		}
 		const embedder = async (texts) => {
 			if (texts.length === 0) return [];
 			if (texts.length === 1) {

package/dist/embeddings/model-info.d.mts

@@ -0,0 +1,26 @@
+//#region src/embeddings/model-info.d.ts
+/**
+ * Embedding provider types
+ */
+type EmbeddingPreset = 'openai' | 'google' | 'mistral' | 'cohere' | 'ollama' | 'transformers.js';
+/**
+ * Default models per provider
+ */
+declare const DEFAULT_MODELS: Record<EmbeddingPreset, {
+  model: string;
+  dimensions: number;
+}>;
+/**
+ * Known dimensions for common models
+ */
+declare const MODEL_DIMENSIONS: Record<string, number>;
+/**
+ * Get dimensions for a model (returns undefined if unknown)
+ */
+declare function getModelDimensions(model: string): number | undefined;
+/**
+ * Resolve model name for a specific preset
+ */
+declare function resolveModelForPreset(model: string, preset: string): string;
+//#endregion
+export { DEFAULT_MODELS, EmbeddingPreset, MODEL_DIMENSIONS, getModelDimensions, resolveModelForPreset };

package/dist/embeddings/model-info.mjs

@@ -0,0 +1,65 @@
+const DEFAULT_MODELS = {
+	"openai": {
+		model: "text-embedding-3-small",
+		dimensions: 1536
+	},
+	"google": {
+		model: "text-embedding-004",
+		dimensions: 768
+	},
+	"mistral": {
+		model: "mistral-embed",
+		dimensions: 1024
+	},
+	"cohere": {
+		model: "embed-english-v3.0",
+		dimensions: 1024
+	},
+	"ollama": {
+		model: "nomic-embed-text",
+		dimensions: 768
+	},
+	"transformers.js": {
+		model: "Xenova/bge-small-en-v1.5",
+		dimensions: 384
+	}
+};
+const MODEL_DIMENSIONS = {
+	"text-embedding-3-small": 1536,
+	"text-embedding-3-large": 3072,
+	"text-embedding-ada-002": 1536,
+	"text-embedding-004": 768,
+	"embedding-001": 768,
+	"mistral-embed": 1024,
+	"embed-english-v3.0": 1024,
+	"embed-multilingual-v3.0": 1024,
+	"embed-english-light-v3.0": 384,
+	"embed-multilingual-light-v3.0": 384,
+	"nomic-embed-text": 768,
+	"mxbai-embed-large": 1024,
+	"all-minilm": 384,
+	"snowflake-arctic-embed": 1024,
+	"bge-small-en-v1.5": 384,
+	"bge-base-en-v1.5": 768,
+	"bge-large-en-v1.5": 1024,
+	"bge-m3": 1024,
+	"all-MiniLM-L6-v2": 384,
+	"embeddinggemma-300m": 256,
+	"plamo-embedding-1b": 1024
+};
+function getModelDimensions(model) {
+	if (MODEL_DIMENSIONS[model]) return MODEL_DIMENSIONS[model];
+	return MODEL_DIMENSIONS[model.replace(/^(Xenova\/|onnx-community\/)/, "")];
+}
+const MODEL_MAPPINGS = { "transformers.js": {
+	"bge-base-en-v1.5": "Xenova/bge-base-en-v1.5",
+	"bge-large-en-v1.5": "onnx-community/bge-large-en-v1.5",
+	"bge-small-en-v1.5": "Xenova/bge-small-en-v1.5",
+	"bge-m3": "Xenova/bge-m3",
+	"all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
+	"embeddinggemma-300m": "onnx-community/embeddinggemma-300m-ONNX"
+} };
+function resolveModelForPreset(model, preset) {
+	return MODEL_MAPPINGS[preset]?.[model] ?? model;
+}
+export { DEFAULT_MODELS, MODEL_DIMENSIONS, getModelDimensions, resolveModelForPreset };

package/dist/embeddings/ollama.mjs

@@ -1,3 +1,4 @@
+import { getModelDimensions } from "./model-info.mjs";
 import { embed, embedMany } from "ai";
 import { createOllama } from "ollama-ai-provider-v2";
 function ollama(options = {}) {
@@ -7,11 +8,14 @@ function ollama(options = {}) {
 		if (cached) return cached;
 		const ollamaBaseUrl = baseUrl || process.env.OLLAMA_BASE_URL || "http://localhost:11434";
 		const embeddingModel = createOllama({ baseURL: ollamaBaseUrl.endsWith("/api") ? ollamaBaseUrl : `${ollamaBaseUrl}/api` }).textEmbeddingModel(model);
-		const { embedding: testEmbedding } = await embed({
-			model: embeddingModel,
-			value: "test"
-		});
-		const dimensions = testEmbedding.length;
+		let dimensions = getModelDimensions(model);
+		if (!dimensions) {
+			const { embedding } = await embed({
+				model: embeddingModel,
+				value: "test"
+			});
+			dimensions = embedding.length;
+		}
 		const embedder = async (texts) => {
 			if (texts.length === 0) return [];
 			if (texts.length === 1) {

package/dist/embeddings/openai.mjs

@@ -1,3 +1,4 @@
+import { getModelDimensions } from "./model-info.mjs";
 import { createOpenAI } from "@ai-sdk/openai";
 import { embed, embedMany } from "ai";
 function openai(options = {}) {
@@ -9,11 +10,14 @@ function openai(options = {}) {
 			apiKey,
 			baseURL: baseUrl
 		}).textEmbeddingModel(model);
-		const { embedding: testEmbedding } = await embed({
-			model: embeddingModel,
-			value: "test"
-		});
-		const dimensions = testEmbedding.length;
+		let dimensions = getModelDimensions(model);
+		if (!dimensions) {
+			const { embedding } = await embed({
+				model: embeddingModel,
+				value: "test"
+			});
+			dimensions = embedding.length;
+		}
 		const embedder = async (texts) => {
 			if (texts.length === 0) return [];
 			if (texts.length === 1) {

package/dist/embeddings/transformers-js.d.mts

@@ -0,0 +1,27 @@
+import { EmbeddingConfig } from "../types.mjs";
+
+//#region src/embeddings/transformers-js.d.ts
+interface TransformersEmbeddingOptions {
+  /** Model name (e.g., 'bge-base-en-v1.5' or 'Xenova/bge-base-en-v1.5') */
+  model?: string;
+  /** Embedding dimensions (auto-detected for known models) */
+  dimensions?: number;
+}
+/**
+ * Transformers.js embedding provider (local, in-browser compatible)
+ *
+ * @example
+ * ```ts
+ * import { transformersJs } from 'retriv/embeddings/transformers-js'
+ * import { sqliteVec } from 'retriv/db/sqlite-vec'
+ *
+ * // Auto-resolves model name and dimensions for known models
+ * const db = await sqliteVec({
+ *   path: 'vectors.db',
+ *   embeddings: transformersJs({ model: 'bge-base-en-v1.5' }),
+ * })
+ * ```
+ */
+declare function transformersJs(options?: TransformersEmbeddingOptions): EmbeddingConfig;
+//#endregion
+export { TransformersEmbeddingOptions, transformersJs };

package/dist/embeddings/transformers-js.mjs

@@ -0,0 +1,42 @@
+import { getModelDimensions, resolveModelForPreset } from "./model-info.mjs";
+import { rm } from "node:fs/promises";
+import { env, pipeline } from "@huggingface/transformers";
+async function clearCorruptedCache(error, model) {
+	if (!(error instanceof Error && (error.message?.includes("Protobuf parsing failed") || String(error.cause)?.includes("Protobuf parsing failed"))) || !env.cacheDir) return false;
+	await rm(`${env.cacheDir}/${model}`, {
+		recursive: true,
+		force: true
+	}).catch(() => {});
+	console.warn(`[retriv] Cleared corrupted model cache for ${model}, retrying...`);
+	return true;
+}
+function transformersJs(options = {}) {
+	const model = resolveModelForPreset(options.model ?? "bge-small-en-v1.5", "transformers.js");
+	let cached = null;
+	return { async resolve() {
+		if (cached) return cached;
+		const extractor = await pipeline("feature-extraction", model, { dtype: "fp32" }).catch(async (err) => {
+			if (await clearCorruptedCache(err, model)) return pipeline("feature-extraction", model, { dtype: "fp32" });
+			throw err;
+		});
+		const dimensions = options.dimensions ?? getModelDimensions(model);
+		if (!dimensions) throw new Error(`Unknown dimensions for model ${model}. Please specify dimensions option.`);
+		const embedder = async (texts) => {
+			const results = [];
+			for (const text of texts) {
+				const output = await extractor(text, {
+					pooling: "mean",
+					normalize: true
+				});
+				results.push(Array.from(output.data));
+			}
+			return results;
+		};
+		cached = {
+			embedder,
+			dimensions
+		};
+		return cached;
+	} };
+}
+export { transformersJs };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "retriv",
   "type": "module",
-  "version": "0.0.1",
+  "version": "0.2.0",
   "description": "Unified document retrieval for search - semantic, full-text, and fuzzy.",
   "author": {
     "name": "Harlan Wilton",
@@ -58,9 +58,9 @@
       "types": "./dist/embeddings/ollama.d.mts",
       "import": "./dist/embeddings/ollama.mjs"
     },
-    "./embeddings/transformers": {
-      "types": "./dist/embeddings/transformers.d.mts",
-      "import": "./dist/embeddings/transformers.mjs"
+    "./embeddings/transformers-js": {
+      "types": "./dist/embeddings/transformers-js.d.mts",
+      "import": "./dist/embeddings/transformers-js.mjs"
     },
     "./embeddings/mistral": {
       "types": "./dist/embeddings/mistral.d.mts",
@@ -69,6 +69,10 @@
     "./embeddings/cohere": {
       "types": "./dist/embeddings/cohere.d.mts",
       "import": "./dist/embeddings/cohere.mjs"
+    },
+    "./embeddings/model-info": {
+      "types": "./dist/embeddings/model-info.d.mts",
+      "import": "./dist/embeddings/model-info.mjs"
     }
   },
   "main": "./dist/index.mjs",
@@ -85,7 +89,6 @@
     "@libsql/client": "^0.14.0 || ^0.15.0 || ^0.16.0 || ^0.17.0",
     "@upstash/vector": "^1.0.0",
     "ai": "^4.0.0 || ^5.0.0 || ^6.0.0",
-    "better-sqlite3": "^11.0.0 || ^12.0.0",
     "ollama-ai-provider-v2": "^1.0.0",
     "pg": "^8.0.0",
     "sqlite-vec": "^0.1.0-alpha.0"
@@ -115,9 +118,6 @@
     "ai": {
       "optional": true
     },
-    "better-sqlite3": {
-      "optional": true
-    },
     "ollama-ai-provider-v2": {
       "optional": true
     },
@@ -138,20 +138,19 @@
     "@cloudflare/workers-types": "^4.20260131.0",
     "@huggingface/transformers": "^3.8.1",
     "@libsql/client": "^0.17.0",
-    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^25.2.0",
-    "@types/pg": "^8.11.0",
-    "@upstash/vector": "^1.1.0",
+    "@types/pg": "^8.16.0",
+    "@upstash/vector": "^1.2.2",
     "@vitest/coverage-v8": "^4.0.18",
     "ai": "^6.0.67",
-    "better-sqlite3": "^12.5.0",
-    "bumpp": "^10.3.2",
+    "ai-sdk-provider-gemini-cli": "^2.0.1",
+    "bumpp": "^10.4.0",
     "eslint": "^9.39.2",
-    "obuild": "^0.4.14",
+    "obuild": "^0.4.22",
     "pg": "^8.18.0",
     "sqlite-vec": "^0.1.7-alpha.2",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.16",
+    "vitest": "^4.0.18",
     "zod": "^4.3.6"
   },
   "scripts": {

package/dist/embeddings/transformers.d.mts

@@ -1,24 +0,0 @@
-import { EmbeddingConfig } from "../types.mjs";
-
-//#region src/embeddings/transformers.d.ts
-interface TransformersEmbeddingOptions {
-  /** Model name (default: Xenova/bge-base-en-v1.5) */
-  model?: string;
-}
-/**
- * Transformers.js embedding provider (local, in-browser compatible)
- *
- * @example
- * ```ts
- * import { transformers } from 'retriv/embeddings/transformers'
- * import { sqliteVec } from 'retriv/db/sqlite-vec'
- *
- * const db = await sqliteVec({
- *   path: 'vectors.db',
- *   embeddings: transformers({ model: 'Xenova/bge-base-en-v1.5' }),
- * })
- * ```
- */
-declare function transformers(options?: TransformersEmbeddingOptions): EmbeddingConfig;
-//#endregion
-export { TransformersEmbeddingOptions, transformers };

package/dist/embeddings/transformers.mjs

@@ -1,26 +0,0 @@
-import { pipeline } from "@huggingface/transformers";
-function transformers(options = {}) {
-	const { model = "Xenova/bge-base-en-v1.5" } = options;
-	let cached = null;
-	return { async resolve() {
-		if (cached) return cached;
-		const extractor = await pipeline("feature-extraction", model, { dtype: "fp32" });
-		const embedder = async (texts) => {
-			const results = [];
-			for (const text of texts) {
-				const output = await extractor(text, {
-					pooling: "mean",
-					normalize: true
-				});
-				results.push(Array.from(output.data));
-			}
-			return results;
-		};
-		cached = {
-			embedder,
-			dimensions: (await embedder(["test"]))[0].length
-		};
-		return cached;
-	} };
-}
-export { transformers };