retriv 0.0.1

package/dist/embeddings/openai.d.mts

@@ -0,0 +1,28 @@
+import { EmbeddingConfig } from "../types.mjs";
+
+//#region src/embeddings/openai.d.ts
+interface OpenAIEmbeddingOptions {
+  /** Model name (default: text-embedding-3-small) */
+  model?: string;
+  /** API key (falls back to OPENAI_API_KEY env) */
+  apiKey?: string;
+  /** Base URL override */
+  baseUrl?: string;
+}
+/**
+ * OpenAI embedding provider
+ *
+ * @example
+ * ```ts
+ * import { openai } from 'retriv/embeddings/openai'
+ * import { sqliteVec } from 'retriv/db/sqlite-vec'
+ *
+ * const db = await sqliteVec({
+ *   path: 'vectors.db',
+ *   embeddings: openai({ model: 'text-embedding-3-small' }),
+ * })
+ * ```
+ */
+declare function openai(options?: OpenAIEmbeddingOptions): EmbeddingConfig;
+//#endregion
+export { OpenAIEmbeddingOptions, openai };

package/dist/embeddings/openai.mjs

@@ -0,0 +1,39 @@
+import { createOpenAI } from "@ai-sdk/openai";
+import { embed, embedMany } from "ai";
+function openai(options = {}) {
+	const { model = "text-embedding-3-small", apiKey, baseUrl } = options;
+	let cached = null;
+	return { async resolve() {
+		if (cached) return cached;
+		const embeddingModel = createOpenAI({
+			apiKey,
+			baseURL: baseUrl
+		}).textEmbeddingModel(model);
+		const { embedding: testEmbedding } = await embed({
+			model: embeddingModel,
+			value: "test"
+		});
+		const dimensions = testEmbedding.length;
+		const embedder = async (texts) => {
+			if (texts.length === 0) return [];
+			if (texts.length === 1) {
+				const { embedding } = await embed({
+					model: embeddingModel,
+					value: texts[0]
+				});
+				return [embedding];
+			}
+			const { embeddings } = await embedMany({
+				model: embeddingModel,
+				values: texts
+			});
+			return embeddings;
+		};
+		cached = {
+			embedder,
+			dimensions
+		};
+		return cached;
+	} };
+}
+export { openai };

package/dist/embeddings/resolve.d.mts

@@ -0,0 +1,10 @@
+import { EmbeddingConfig, ResolvedEmbedding } from "../types.mjs";
+
+//#region src/embeddings/resolve.d.ts
+/**
+ * Resolve an EmbeddingConfig to an embedder and dimensions
+ * Simply calls the resolve method on the config
+ */
+declare function resolveEmbedding(config: EmbeddingConfig): Promise<ResolvedEmbedding>;
+//#endregion
+export { resolveEmbedding };

package/dist/embeddings/resolve.mjs

@@ -0,0 +1,4 @@
+async function resolveEmbedding(config) {
+	return config.resolve();
+}
+export { resolveEmbedding };

package/dist/embeddings/transformers.d.mts

@@ -0,0 +1,24 @@
+import { EmbeddingConfig } from "../types.mjs";
+
+//#region src/embeddings/transformers.d.ts
+interface TransformersEmbeddingOptions {
+  /** Model name (default: Xenova/bge-base-en-v1.5) */
+  model?: string;
+}
+/**
+ * Transformers.js embedding provider (local, in-browser compatible)
+ *
+ * @example
+ * ```ts
+ * import { transformers } from 'retriv/embeddings/transformers'
+ * import { sqliteVec } from 'retriv/db/sqlite-vec'
+ *
+ * const db = await sqliteVec({
+ *   path: 'vectors.db',
+ *   embeddings: transformers({ model: 'Xenova/bge-base-en-v1.5' }),
+ * })
+ * ```
+ */
+declare function transformers(options?: TransformersEmbeddingOptions): EmbeddingConfig;
+//#endregion
+export { TransformersEmbeddingOptions, transformers };

package/dist/embeddings/transformers.mjs

@@ -0,0 +1,26 @@
+import { pipeline } from "@huggingface/transformers";
+function transformers(options = {}) {
+	const { model = "Xenova/bge-base-en-v1.5" } = options;
+	let cached = null;
+	return { async resolve() {
+		if (cached) return cached;
+		const extractor = await pipeline("feature-extraction", model, { dtype: "fp32" });
+		const embedder = async (texts) => {
+			const results = [];
+			for (const text of texts) {
+				const output = await extractor(text, {
+					pooling: "mean",
+					normalize: true
+				});
+				results.push(Array.from(output.data));
+			}
+			return results;
+		};
+		cached = {
+			embedder,
+			dimensions: (await embedder(["test"]))[0].length
+		};
+		return cached;
+	} };
+}
+export { transformers };

package/dist/index.d.mts

@@ -0,0 +1,3 @@
+import { ChunkInfo, ChunkingOptions, Document, EmbeddingConfig, EmbeddingProvider, RetrivOptions, SearchOptions, SearchProvider, SearchResult } from "./types.mjs";
+import { createRetriv } from "./retriv.mjs";
+export { type ChunkInfo, type ChunkingOptions, type Document, type EmbeddingConfig, type EmbeddingProvider, type RetrivOptions, type SearchOptions, type SearchProvider, type SearchResult, createRetriv };

package/dist/index.mjs

@@ -0,0 +1,2 @@
+import { createRetriv } from "./retriv.mjs";
+export { createRetriv };

package/dist/retriv.d.mts

@@ -0,0 +1,9 @@
+import { RetrivOptions, SearchProvider } from "./types.mjs";
+
+//#region src/retriv.d.ts
+/**
+ * Create a unified retrieval instance
+ */
+declare function createRetriv(options: RetrivOptions): Promise<SearchProvider>;
+//#endregion
+export { createRetriv };

package/dist/retriv.mjs

@@ -0,0 +1,112 @@
+import { t as splitText } from "./_chunks/split-text.mjs";
+const RRF_K = 60;
+function isComposed(driver) {
+	return typeof driver === "object" && driver !== null && ("vector" in driver || "keyword" in driver);
+}
+function applyRRF(resultSets) {
+	const scores = /* @__PURE__ */ new Map();
+	for (const results of resultSets) for (let rank = 0; rank < results.length; rank++) {
+		const result = results[rank];
+		const rrfScore = 1 / (RRF_K + rank + 1);
+		const existing = scores.get(result.id);
+		if (existing) {
+			existing.score += rrfScore;
+			if (result.content && !existing.result.content) existing.result = {
+				...existing.result,
+				content: result.content
+			};
+			if (result.metadata && !existing.result.metadata) existing.result = {
+				...existing.result,
+				metadata: result.metadata
+			};
+		} else scores.set(result.id, {
+			score: rrfScore,
+			result
+		});
+	}
+	return Array.from(scores.values()).sort((a, b) => b.score - a.score).map(({ score, result }) => ({
+		...result,
+		score
+	}));
+}
+async function createRetriv(options) {
+	const { driver: driverInput, chunking } = options;
+	let drivers;
+	if (isComposed(driverInput)) {
+		drivers = (await Promise.all([driverInput.vector ? Promise.resolve(driverInput.vector) : null, driverInput.keyword ? Promise.resolve(driverInput.keyword) : null])).filter((d) => d !== null);
+		if (drivers.length === 0) throw new Error("At least one driver (vector or keyword) is required");
+	} else drivers = [await Promise.resolve(driverInput)];
+	const isHybrid = drivers.length > 1;
+	const parentDocs = /* @__PURE__ */ new Map();
+	function prepareDocs(docs) {
+		if (!chunking) return docs;
+		const { chunkSize = 1e3, chunkOverlap = 200 } = chunking;
+		const chunkedDocs = [];
+		for (const doc of docs) {
+			const chunks = splitText(doc.content, {
+				chunkSize,
+				chunkOverlap
+			});
+			if (chunks.length === 1) chunkedDocs.push(doc);
+			else {
+				parentDocs.set(doc.id, doc);
+				for (const chunk of chunks) chunkedDocs.push({
+					id: `${doc.id}#chunk-${chunk.index}`,
+					content: chunk.text,
+					metadata: {
+						...doc.metadata,
+						_parentId: doc.id,
+						_chunkIndex: chunk.index,
+						_chunkRange: chunk.range
+					}
+				});
+			}
+		}
+		return chunkedDocs;
+	}
+	function annotateChunks(results) {
+		if (!chunking) return results;
+		return results.map((result) => {
+			const metadata = result.metadata || {};
+			const parentId = metadata._parentId;
+			const chunkIndex = metadata._chunkIndex;
+			const chunkRange = metadata._chunkRange;
+			if (parentId !== void 0 && chunkIndex !== void 0) {
+				const { _parentId, _chunkIndex, _chunkRange, ...cleanMeta } = metadata;
+				return {
+					...result,
+					metadata: Object.keys(cleanMeta).length > 0 ? cleanMeta : void 0,
+					_chunk: {
+						parentId,
+						index: chunkIndex,
+						range: chunkRange
+					}
+				};
+			}
+			return result;
+		});
+	}
+	return {
+		async index(docs) {
+			const prepared = prepareDocs(docs);
+			return { count: (await Promise.all(drivers.map((d) => d.index(prepared))))[0].count };
+		},
+		async search(query, searchOptions = {}) {
+			if (!isHybrid) return annotateChunks(await drivers[0].search(query, searchOptions));
+			let merged = applyRRF(await Promise.all(drivers.map((d) => d.search(query, searchOptions))));
+			if (searchOptions.limit) merged = merged.slice(0, searchOptions.limit);
+			return annotateChunks(merged);
+		},
+		async remove(ids) {
+			return { count: (await Promise.all(drivers.filter((d) => d.remove).map((d) => d.remove(ids))))[0]?.count ?? 0 };
+		},
+		async clear() {
+			await Promise.all(drivers.filter((d) => d.clear).map((d) => d.clear()));
+			parentDocs.clear();
+		},
+		async close() {
+			await Promise.all(drivers.filter((d) => d.close).map((d) => d.close()));
+		}
+	};
+}
+export { createRetriv };

package/dist/types.d.mts

@@ -0,0 +1,203 @@
+//#region src/types.d.ts
+/**
+ * A document to be indexed for search
+ */
+interface Document {
+  /** Unique identifier */
+  id: string;
+  /** Text content to search */
+  content: string;
+  /** Optional metadata to store alongside */
+  metadata?: Record<string, any>;
+}
+/**
+ * Chunk info when chunking is enabled
+ */
+interface ChunkInfo {
+  parentId: string;
+  index: number;
+  range?: [number, number];
+}
+/**
+ * Driver-specific metadata
+ */
+interface SearchMeta {
+  bm25Score?: number;
+  highlights?: string[];
+  distance?: number;
+  vector?: number[];
+  matches?: Array<{
+    indices: [number, number][];
+    value: string;
+  }>;
+  [key: string]: any;
+}
+/**
+ * A search result
+ */
+interface SearchResult {
+  /** Document ID */
+  id: string;
+  /** Relevance score (0-1, higher is better) */
+  score: number;
+  /** Original content (if returnContent: true) */
+  content?: string;
+  /** Document metadata (if returnMetadata: true) */
+  metadata?: Record<string, any>;
+  /** Chunk info (when chunking enabled) */
+  _chunk?: ChunkInfo;
+  /** Driver-specific extras (if returnMeta: true) */
+  _meta?: SearchMeta;
+}
+/**
+ * Search options
+ */
+interface SearchOptions {
+  /** Maximum results to return */
+  limit?: number;
+  /** Return original content */
+  returnContent?: boolean;
+  /** Return metadata */
+  returnMetadata?: boolean;
+  /** Return driver-specific _meta */
+  returnMeta?: boolean;
+}
+/**
+ * Search provider interface - unified across all driver types
+ */
+interface SearchProvider {
+  /**
+   * Index documents for search
+   */
+  index: (docs: Document[]) => Promise<{
+    count: number;
+  }>;
+  /**
+   * Search for documents
+   */
+  search: (query: string, options?: SearchOptions) => Promise<SearchResult[]>;
+  /**
+   * Remove documents by ID
+   */
+  remove?: (ids: string[]) => Promise<{
+    count: number;
+  }>;
+  /**
+   * Clear all indexed documents
+   */
+  clear?: () => Promise<void>;
+  /**
+   * Close the provider and release resources
+   */
+  close?: () => Promise<void>;
+}
+/**
+ * Base config shared by all drivers
+ */
+interface BaseDriverConfig {
+  /** Database/index path or URL */
+  path?: string;
+  url?: string;
+}
+/**
+ * Embedding provider function
+ * Takes text(s) and returns embedding vectors
+ */
+type EmbeddingProvider = (texts: string[]) => Promise<number[][]>;
+/**
+ * Resolved embedding result
+ */
+interface ResolvedEmbedding {
+  embedder: EmbeddingProvider;
+  dimensions: number;
+}
+/**
+ * Embedding config (returned by embedding modules)
+ */
+interface EmbeddingConfig {
+  /** Resolve the embedding provider */
+  resolve: () => Promise<ResolvedEmbedding>;
+}
+type VectorFloatArray = Float32Array | Float64Array;
+type VectorizeVectorMetadata = string | number | boolean | string[];
+interface VectorizeVector {
+  id: string;
+  values: VectorFloatArray | number[];
+  namespace?: string;
+  metadata?: Record<string, VectorizeVectorMetadata>;
+}
+interface VectorizeMatch {
+  id: string;
+  score: number;
+  values?: number[];
+  namespace?: string;
+  metadata?: Record<string, VectorizeVectorMetadata>;
+}
+interface VectorizeMatches {
+  matches: VectorizeMatch[];
+  count: number;
+}
+interface VectorizeQueryOptions {
+  topK?: number;
+  namespace?: string;
+  returnValues?: boolean;
+  returnMetadata?: boolean;
+  filter?: Record<string, any>;
+}
+/**
+ * Semantic driver config (vector-based search)
+ */
+interface SemanticDriverConfig extends BaseDriverConfig {
+  /** Embedding provider from retriv/embeddings/ */
+  embeddings: EmbeddingConfig;
+  /** Auth token for remote databases */
+  authToken?: string;
+  /** Namespace for vector isolation */
+  namespace?: string;
+  /** Distance metric */
+  metric?: 'cosine' | 'euclidean' | 'dot-product';
+}
+/**
+ * Text-native driver config (e.g., Upstash - handles embeddings server-side)
+ */
+interface TextNativeDriverConfig extends BaseDriverConfig {
+  /** API token */
+  token: string;
+  /** Namespace for isolation */
+  namespace?: string;
+}
+/**
+ * Chunking configuration
+ */
+interface ChunkingOptions {
+  chunkSize?: number;
+  chunkOverlap?: number;
+}
+/**
+ * Resolvable driver (can be promise)
+ */
+type Resolvable<T> = T | Promise<T>;
+/**
+ * Any search provider (for composed drivers - loosened for driver compatibility)
+ */
+type AnyDriver = Resolvable<SearchProvider>;
+/**
+ * Composed driver config for hybrid search
+ */
+interface ComposedDriver {
+  vector?: AnyDriver;
+  keyword?: AnyDriver;
+}
+/**
+ * Driver input - single driver or composed
+ */
+type DriverInput = AnyDriver | ComposedDriver;
+/**
+ * Options for createRetriv factory
+ */
+interface RetrivOptions {
+  driver: DriverInput;
+  chunking?: ChunkingOptions;
+}
+//#endregion
+export { BaseDriverConfig, ChunkInfo, ChunkingOptions, ComposedDriver, Document, DriverInput, EmbeddingConfig, EmbeddingProvider, ResolvedEmbedding, RetrivOptions, SearchMeta, SearchOptions, SearchProvider, SearchResult, SemanticDriverConfig, TextNativeDriverConfig, VectorFloatArray, VectorizeMatch, VectorizeMatches, VectorizeQueryOptions, VectorizeVector, VectorizeVectorMetadata };

package/dist/types.mjs

@@ -0,0 +1 @@
+export {};

package/dist/utils/split-text.d.mts

@@ -0,0 +1,23 @@
+//#region src/utils/split-text.d.ts
+/**
+ * Recursive markdown text splitter (LangChain-style)
+ */
+interface SplitTextOptions {
+  chunkSize?: number;
+  chunkOverlap?: number;
+  separators?: string[];
+}
+interface TextChunk {
+  text: string;
+  index: number;
+  /** Character range [start, end] in original text */
+  range: [number, number];
+  /** Line range [startLine, endLine] (1-indexed) */
+  lines: [number, number];
+}
+/**
+ * Split text recursively using markdown-aware separators
+ */
+declare function splitText(text: string, options?: SplitTextOptions): TextChunk[];
+//#endregion
+export { SplitTextOptions, TextChunk, splitText };

package/dist/utils/split-text.mjs

@@ -0,0 +1,2 @@
+import { t as splitText } from "../_chunks/split-text.mjs";
+export { splitText };

package/package.json

@@ -0,0 +1,167 @@
+{
+  "name": "retriv",
+  "type": "module",
+  "version": "0.0.1",
+  "description": "Unified document retrieval for search - semantic, full-text, and fuzzy.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/harlan-zw/retriv"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    },
+    "./db/sqlite": {
+      "types": "./dist/db/sqlite.d.mts",
+      "import": "./dist/db/sqlite.mjs"
+    },
+    "./db/sqlite-vec": {
+      "types": "./dist/db/sqlite-vec.d.mts",
+      "import": "./dist/db/sqlite-vec.mjs"
+    },
+    "./db/sqlite-fts": {
+      "types": "./dist/db/sqlite-fts.d.mts",
+      "import": "./dist/db/sqlite-fts.mjs"
+    },
+    "./db/libsql": {
+      "types": "./dist/db/libsql.d.mts",
+      "import": "./dist/db/libsql.mjs"
+    },
+    "./db/upstash": {
+      "types": "./dist/db/upstash.d.mts",
+      "import": "./dist/db/upstash.mjs"
+    },
+    "./db/pgvector": {
+      "types": "./dist/db/pgvector.d.mts",
+      "import": "./dist/db/pgvector.mjs"
+    },
+    "./db/cloudflare": {
+      "types": "./dist/db/cloudflare.d.mts",
+      "import": "./dist/db/cloudflare.mjs"
+    },
+    "./embeddings/openai": {
+      "types": "./dist/embeddings/openai.d.mts",
+      "import": "./dist/embeddings/openai.mjs"
+    },
+    "./embeddings/google": {
+      "types": "./dist/embeddings/google.d.mts",
+      "import": "./dist/embeddings/google.mjs"
+    },
+    "./embeddings/ollama": {
+      "types": "./dist/embeddings/ollama.d.mts",
+      "import": "./dist/embeddings/ollama.mjs"
+    },
+    "./embeddings/transformers": {
+      "types": "./dist/embeddings/transformers.d.mts",
+      "import": "./dist/embeddings/transformers.mjs"
+    },
+    "./embeddings/mistral": {
+      "types": "./dist/embeddings/mistral.d.mts",
+      "import": "./dist/embeddings/mistral.mjs"
+    },
+    "./embeddings/cohere": {
+      "types": "./dist/embeddings/cohere.d.mts",
+      "import": "./dist/embeddings/cohere.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "files": [
+    "dist"
+  ],
+  "peerDependencies": {
+    "@ai-sdk/cohere": "^3.0.0",
+    "@ai-sdk/google": "^3.0.0",
+    "@ai-sdk/mistral": "^3.0.0",
+    "@ai-sdk/openai": "^3.0.0",
+    "@huggingface/transformers": "^3.0.0",
+    "@libsql/client": "^0.14.0 || ^0.15.0 || ^0.16.0 || ^0.17.0",
+    "@upstash/vector": "^1.0.0",
+    "ai": "^4.0.0 || ^5.0.0 || ^6.0.0",
+    "better-sqlite3": "^11.0.0 || ^12.0.0",
+    "ollama-ai-provider-v2": "^1.0.0",
+    "pg": "^8.0.0",
+    "sqlite-vec": "^0.1.0-alpha.0"
+  },
+  "peerDependenciesMeta": {
+    "@ai-sdk/cohere": {
+      "optional": true
+    },
+    "@ai-sdk/google": {
+      "optional": true
+    },
+    "@ai-sdk/mistral": {
+      "optional": true
+    },
+    "@ai-sdk/openai": {
+      "optional": true
+    },
+    "@huggingface/transformers": {
+      "optional": true
+    },
+    "@libsql/client": {
+      "optional": true
+    },
+    "@upstash/vector": {
+      "optional": true
+    },
+    "ai": {
+      "optional": true
+    },
+    "better-sqlite3": {
+      "optional": true
+    },
+    "ollama-ai-provider-v2": {
+      "optional": true
+    },
+    "pg": {
+      "optional": true
+    },
+    "sqlite-vec": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@ai-sdk/cohere": "^3.0.18",
+    "@ai-sdk/google": "^3.0.20",
+    "@ai-sdk/mistral": "^3.0.18",
+    "@ai-sdk/openai": "^3.0.25",
+    "@antfu/eslint-config": "^7.2.0",
+    "@arethetypeswrong/cli": "^0.18.2",
+    "@cloudflare/workers-types": "^4.20260131.0",
+    "@huggingface/transformers": "^3.8.1",
+    "@libsql/client": "^0.17.0",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.2.0",
+    "@types/pg": "^8.11.0",
+    "@upstash/vector": "^1.1.0",
+    "@vitest/coverage-v8": "^4.0.18",
+    "ai": "^6.0.67",
+    "better-sqlite3": "^12.5.0",
+    "bumpp": "^10.3.2",
+    "eslint": "^9.39.2",
+    "obuild": "^0.4.14",
+    "pg": "^8.18.0",
+    "sqlite-vec": "^0.1.7-alpha.2",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16",
+    "zod": "^4.3.6"
+  },
+  "scripts": {
+    "build": "obuild",
+    "dev:prepare": "obuild --stub",
+    "lint": "eslint .",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest --project unit",
+    "test:eval": "vitest run --project eval",
+    "test:attw": "attw --pack",
+    "release": "pnpm build && bumpp --output=CHANGELOG.md"
+  }
+}