retriv 0.1.0 → 0.3.0

package/README.md

@@ -244,18 +244,18 @@ interface SearchResult {
 
 ## Benchmarks
 
-Retrieval accuracy on Nuxt documentation (639 docs):
+Retrieval accuracy on Nuxt documentation (2,817 chunks):
 
 | Test Type | FTS | Vector | Hybrid |
 |-----------|-----|--------|--------|
-| Exact terminology (ports, config names) | 3/3 | 2/3 | 3/3 |
-| Doc retrieval (keyword overlap) | 3/3 | 2/3 | 3/3 |
-| Semantic queries (synonyms, no overlap) | 0/3 | 3/3 | 3/3 |
-| **Total** | **6/9 (67%)** | **7/9 (78%)** | **9/9 (100%)** |
+| Exact terminology (ports, config names) | 7/7 | 5/7 | 7/7 |
+| Doc retrieval (keyword overlap) | 0/7 | 5/7 | 5/7 |
+| Semantic queries (synonyms, no overlap) | 1/6 | 5/6 | 5/6 |
+| **Total** | **8/20 (40%)** | **15/20 (75%)** | **17/20 (85%)** |
 
 - **FTS** excels at exact terms but fails semantic queries ("reuse logic" → composables)
-- **Vector** understands meaning but misses precise terminology ("port 3000")
-- **Hybrid** combines both - never worse than either method alone
+- **Vector** understands meaning but misses precise terminology (".global" suffix)
+- **Hybrid** combines both - best overall recall across query types
 
 Run locally: `pnpm test:eval`
 

package/dist/_chunks/extract-snippet.mjs

@@ -0,0 +1,122 @@
+const STOPWORDS = new Set([
+	"the",
+	"a",
+	"an",
+	"and",
+	"or",
+	"but",
+	"in",
+	"on",
+	"at",
+	"to",
+	"for",
+	"of",
+	"with",
+	"by",
+	"from",
+	"as",
+	"is",
+	"was",
+	"are",
+	"were",
+	"been",
+	"be",
+	"have",
+	"has",
+	"had",
+	"do",
+	"does",
+	"did",
+	"will",
+	"would",
+	"could",
+	"should",
+	"may",
+	"might",
+	"must",
+	"can",
+	"this",
+	"that",
+	"these",
+	"those",
+	"it",
+	"its",
+	"they",
+	"them",
+	"their",
+	"we",
+	"our",
+	"you",
+	"your",
+	"what",
+	"which",
+	"who",
+	"how",
+	"when",
+	"where",
+	"why",
+	"all",
+	"each",
+	"every",
+	"both",
+	"few",
+	"more",
+	"most",
+	"other",
+	"some",
+	"such",
+	"no",
+	"not",
+	"only"
+]);
+function scoreTerms(terms, content) {
+	const contentLower = content.toLowerCase();
+	const contentLen = content.length;
+	return terms.filter((t) => contentLower.includes(t)).map((term) => {
+		const regex = new RegExp(term, "gi");
+		const matches = contentLower.match(regex);
+		const tf = matches ? matches.length : 0;
+		const k1 = 1.2;
+		const tfNorm = tf * (k1 + 1) / (tf + k1 * (.25 + .75 * (contentLen / 500)));
+		const stopwordPenalty = STOPWORDS.has(term) ? .1 : 1;
+		const lengthBoost = Math.min(term.length / 5, 1.5);
+		return {
+			term,
+			score: tfNorm * stopwordPenalty * lengthBoost
+		};
+	}).sort((a, b) => b.score - a.score);
+}
+function extractSnippet(content, query, contextLines = 2) {
+	const lines = content.split("\n");
+	const totalContext = contextLines * 2 + 1;
+	const queryWords = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
+	const scoredTerms = scoreTerms(queryWords, content);
+	const highlights = scoredTerms.slice(0, 5).map((t) => t.term);
+	if (lines.length <= totalContext) return {
+		snippet: content,
+		highlights
+	};
+	const termScores = new Map(scoredTerms.map((t) => [t.term, t.score]));
+	let bestIdx = 0;
+	let bestScore = 0;
+	for (let i = 0; i < lines.length; i++) {
+		const lineLower = lines[i].toLowerCase();
+		let score = 0;
+		for (const word of queryWords) if (lineLower.includes(word)) score += termScores.get(word) || 1;
+		if (score > bestScore) {
+			bestScore = score;
+			bestIdx = i;
+		}
+	}
+	if (bestScore === 0) return {
+		snippet: lines.slice(0, totalContext).join("\n"),
+		highlights
+	};
+	const start = Math.max(0, bestIdx - contextLines);
+	const end = Math.min(lines.length, bestIdx + contextLines + 1);
+	return {
+		snippet: lines.slice(start, end).join("\n"),
+		highlights
+	};
+}
+export { extractSnippet as t };

package/dist/db/cloudflare.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 async function cloudflare(config) {
 	const { binding } = config;
 	if (!binding) throw new Error("[cloudflare] binding is required");
@@ -33,7 +34,14 @@ async function cloudflare(config) {
 					id: m.id,
 					score: Math.max(0, Math.min(1, m.score))
 				};
-				if (returnContent && m.metadata?._content) result.content = m.metadata._content;
+				if (returnContent && m.metadata?._content) {
+					const { snippet, highlights } = extractSnippet(m.metadata._content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && m.metadata) {
 					const { _content, ...rest } = m.metadata;
 					if (Object.keys(rest).length > 0) result.metadata = rest;

package/dist/db/libsql.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { createClient } from "@libsql/client";
 async function libsql(config) {
 	const url = config.url || config.path || "file:vectors.db";
@@ -63,7 +64,14 @@ async function libsql(config) {
 					id: row.id,
 					score: Math.max(0, 1 - row.distance)
 				};
-				if (returnContent && row.content) result.content = row.content;
+				if (returnContent && row.content) {
+					const { snippet, highlights } = extractSnippet(row.content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && row.metadata) result.metadata = JSON.parse(row.metadata);
 				return result;
 			});

package/dist/db/pgvector.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import pg from "pg";
 async function pgvector(config) {
 	const { url, table = "vectors", metric = "cosine" } = config;
@@ -59,7 +60,14 @@ async function pgvector(config) {
 					id: row.id,
 					score
 				};
-				if (returnContent && row.content) searchResult.content = row.content;
+				if (returnContent && row.content) {
+					const { snippet, highlights } = extractSnippet(row.content, query);
+					searchResult.content = snippet;
+					if (highlights.length) searchResult._meta = {
+						...searchResult._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && row.metadata) searchResult.metadata = row.metadata;
 				return searchResult;
 			});

package/dist/db/sqlite-fts.mjs

@@ -1,3 +1,4 @@
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 async function sqliteFts(config = {}) {
@@ -49,7 +50,14 @@ async function sqliteFts(config = {}) {
 					id: row.id,
 					score: normalizedScore
 				};
-				if (returnContent && row.content) result.content = row.content;
+				if (returnContent && row.content) {
+					const { snippet, highlights } = extractSnippet(row.content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && row.metadata) result.metadata = JSON.parse(row.metadata);
 				return result;
 			});

package/dist/db/sqlite-vec.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import * as sqliteVecExt from "sqlite-vec";
@@ -73,7 +74,14 @@ async function sqliteVec(config) {
 					id: meta.id,
 					score: 1 / (1 + row.distance)
 				};
-				if (returnContent && meta.content) result.content = meta.content;
+				if (returnContent && meta.content) {
+					const { snippet, highlights } = extractSnippet(meta.content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && meta.metadata) result.metadata = JSON.parse(meta.metadata);
 				return result;
 			}).filter(Boolean);

package/dist/db/upstash.mjs

@@ -1,3 +1,4 @@
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { Index } from "@upstash/vector";
 async function upstash(config) {
 	const { url, token, namespace } = config;
@@ -34,7 +35,14 @@ async function upstash(config) {
 					id: m.id,
 					score: Math.max(0, Math.min(1, m.score))
 				};
-				if (returnContent && m.metadata?._content) result.content = m.metadata._content;
+				if (returnContent && m.metadata?._content) {
+					const { snippet, highlights } = extractSnippet(m.metadata._content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && m.metadata) {
 					const { _content, ...rest } = m.metadata;
 					if (Object.keys(rest).length > 0) result.metadata = rest;

package/dist/embeddings/model-info.mjs

@@ -20,8 +20,8 @@ const DEFAULT_MODELS = {
 		dimensions: 768
 	},
 	"transformers.js": {
-		model: "Xenova/bge-base-en-v1.5",
-		dimensions: 768
+		model: "Xenova/bge-small-en-v1.5",
+		dimensions: 384
 	}
 };
 const MODEL_DIMENSIONS = {

package/dist/embeddings/transformers-js.mjs

@@ -11,7 +11,7 @@ async function clearCorruptedCache(error, model) {
 	return true;
 }
 function transformersJs(options = {}) {
-	const model = resolveModelForPreset(options.model ?? "bge-base-en-v1.5", "transformers.js");
+	const model = resolveModelForPreset(options.model ?? "bge-small-en-v1.5", "transformers.js");
 	let cached = null;
 	return { async resolve() {
 		if (cached) return cached;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "retriv",
   "type": "module",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Unified document retrieval for search - semantic, full-text, and fuzzy.",
   "author": {
     "name": "Harlan Wilton",
@@ -139,17 +139,18 @@
     "@huggingface/transformers": "^3.8.1",
     "@libsql/client": "^0.17.0",
     "@types/node": "^25.2.0",
-    "@types/pg": "^8.11.0",
-    "@upstash/vector": "^1.1.0",
+    "@types/pg": "^8.16.0",
+    "@upstash/vector": "^1.2.2",
     "@vitest/coverage-v8": "^4.0.18",
     "ai": "^6.0.67",
-    "bumpp": "^10.3.2",
+    "ai-sdk-provider-gemini-cli": "^2.0.1",
+    "bumpp": "^10.4.0",
     "eslint": "^9.39.2",
-    "obuild": "^0.4.14",
+    "obuild": "^0.4.22",
     "pg": "^8.18.0",
     "sqlite-vec": "^0.1.7-alpha.2",
     "typescript": "^5.9.3",
-    "vitest": "^4.0.16",
+    "vitest": "^4.0.18",
     "zod": "^4.3.6"
   },
   "scripts": {
@@ -158,6 +159,7 @@
     "lint": "eslint .",
     "typecheck": "tsc --noEmit",
     "test": "vitest --project unit",
+    "test:e2e": "vitest run --project e2e",
     "test:eval": "vitest run --project eval",
     "test:attw": "attw --pack",
     "release": "pnpm build && bumpp --output=CHANGELOG.md"