retriv 0.2.0 → 0.3.0

package/dist/_chunks/extract-snippet.mjs

@@ -0,0 +1,122 @@
+const STOPWORDS = new Set([
+	"the",
+	"a",
+	"an",
+	"and",
+	"or",
+	"but",
+	"in",
+	"on",
+	"at",
+	"to",
+	"for",
+	"of",
+	"with",
+	"by",
+	"from",
+	"as",
+	"is",
+	"was",
+	"are",
+	"were",
+	"been",
+	"be",
+	"have",
+	"has",
+	"had",
+	"do",
+	"does",
+	"did",
+	"will",
+	"would",
+	"could",
+	"should",
+	"may",
+	"might",
+	"must",
+	"can",
+	"this",
+	"that",
+	"these",
+	"those",
+	"it",
+	"its",
+	"they",
+	"them",
+	"their",
+	"we",
+	"our",
+	"you",
+	"your",
+	"what",
+	"which",
+	"who",
+	"how",
+	"when",
+	"where",
+	"why",
+	"all",
+	"each",
+	"every",
+	"both",
+	"few",
+	"more",
+	"most",
+	"other",
+	"some",
+	"such",
+	"no",
+	"not",
+	"only"
+]);
+function scoreTerms(terms, content) {
+	const contentLower = content.toLowerCase();
+	const contentLen = content.length;
+	return terms.filter((t) => contentLower.includes(t)).map((term) => {
+		const regex = new RegExp(term, "gi");
+		const matches = contentLower.match(regex);
+		const tf = matches ? matches.length : 0;
+		const k1 = 1.2;
+		const tfNorm = tf * (k1 + 1) / (tf + k1 * (.25 + .75 * (contentLen / 500)));
+		const stopwordPenalty = STOPWORDS.has(term) ? .1 : 1;
+		const lengthBoost = Math.min(term.length / 5, 1.5);
+		return {
+			term,
+			score: tfNorm * stopwordPenalty * lengthBoost
+		};
+	}).sort((a, b) => b.score - a.score);
+}
+function extractSnippet(content, query, contextLines = 2) {
+	const lines = content.split("\n");
+	const totalContext = contextLines * 2 + 1;
+	const queryWords = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
+	const scoredTerms = scoreTerms(queryWords, content);
+	const highlights = scoredTerms.slice(0, 5).map((t) => t.term);
+	if (lines.length <= totalContext) return {
+		snippet: content,
+		highlights
+	};
+	const termScores = new Map(scoredTerms.map((t) => [t.term, t.score]));
+	let bestIdx = 0;
+	let bestScore = 0;
+	for (let i = 0; i < lines.length; i++) {
+		const lineLower = lines[i].toLowerCase();
+		let score = 0;
+		for (const word of queryWords) if (lineLower.includes(word)) score += termScores.get(word) || 1;
+		if (score > bestScore) {
+			bestScore = score;
+			bestIdx = i;
+		}
+	}
+	if (bestScore === 0) return {
+		snippet: lines.slice(0, totalContext).join("\n"),
+		highlights
+	};
+	const start = Math.max(0, bestIdx - contextLines);
+	const end = Math.min(lines.length, bestIdx + contextLines + 1);
+	return {
+		snippet: lines.slice(start, end).join("\n"),
+		highlights
+	};
+}
+export { extractSnippet as t };

package/dist/db/cloudflare.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 async function cloudflare(config) {
 	const { binding } = config;
 	if (!binding) throw new Error("[cloudflare] binding is required");
@@ -33,7 +34,14 @@ async function cloudflare(config) {
 					id: m.id,
 					score: Math.max(0, Math.min(1, m.score))
 				};
-				if (returnContent && m.metadata?._content) result.content = m.metadata._content;
+				if (returnContent && m.metadata?._content) {
+					const { snippet, highlights } = extractSnippet(m.metadata._content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && m.metadata) {
 					const { _content, ...rest } = m.metadata;
 					if (Object.keys(rest).length > 0) result.metadata = rest;

package/dist/db/libsql.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { createClient } from "@libsql/client";
 async function libsql(config) {
 	const url = config.url || config.path || "file:vectors.db";
@@ -63,7 +64,14 @@ async function libsql(config) {
 					id: row.id,
 					score: Math.max(0, 1 - row.distance)
 				};
-				if (returnContent && row.content) result.content = row.content;
+				if (returnContent && row.content) {
+					const { snippet, highlights } = extractSnippet(row.content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && row.metadata) result.metadata = JSON.parse(row.metadata);
 				return result;
 			});

package/dist/db/pgvector.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import pg from "pg";
 async function pgvector(config) {
 	const { url, table = "vectors", metric = "cosine" } = config;
@@ -59,7 +60,14 @@ async function pgvector(config) {
 					id: row.id,
 					score
 				};
-				if (returnContent && row.content) searchResult.content = row.content;
+				if (returnContent && row.content) {
+					const { snippet, highlights } = extractSnippet(row.content, query);
+					searchResult.content = snippet;
+					if (highlights.length) searchResult._meta = {
+						...searchResult._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && row.metadata) searchResult.metadata = row.metadata;
 				return searchResult;
 			});

package/dist/db/sqlite-fts.mjs

@@ -1,3 +1,4 @@
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 async function sqliteFts(config = {}) {
@@ -49,7 +50,14 @@ async function sqliteFts(config = {}) {
 					id: row.id,
 					score: normalizedScore
 				};
-				if (returnContent && row.content) result.content = row.content;
+				if (returnContent && row.content) {
+					const { snippet, highlights } = extractSnippet(row.content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && row.metadata) result.metadata = JSON.parse(row.metadata);
 				return result;
 			});

package/dist/db/sqlite-vec.mjs

@@ -1,4 +1,5 @@
 import { resolveEmbedding } from "../embeddings/resolve.mjs";
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { mkdirSync } from "node:fs";
 import { dirname } from "node:path";
 import * as sqliteVecExt from "sqlite-vec";
@@ -73,7 +74,14 @@ async function sqliteVec(config) {
 					id: meta.id,
 					score: 1 / (1 + row.distance)
 				};
-				if (returnContent && meta.content) result.content = meta.content;
+				if (returnContent && meta.content) {
+					const { snippet, highlights } = extractSnippet(meta.content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && meta.metadata) result.metadata = JSON.parse(meta.metadata);
 				return result;
 			}).filter(Boolean);

package/dist/db/upstash.mjs

@@ -1,3 +1,4 @@
+import { t as extractSnippet } from "../_chunks/extract-snippet.mjs";
 import { Index } from "@upstash/vector";
 async function upstash(config) {
 	const { url, token, namespace } = config;
@@ -34,7 +35,14 @@ async function upstash(config) {
 					id: m.id,
 					score: Math.max(0, Math.min(1, m.score))
 				};
-				if (returnContent && m.metadata?._content) result.content = m.metadata._content;
+				if (returnContent && m.metadata?._content) {
+					const { snippet, highlights } = extractSnippet(m.metadata._content, query);
+					result.content = snippet;
+					if (highlights.length) result._meta = {
+						...result._meta,
+						highlights
+					};
+				}
 				if (returnMetadata && m.metadata) {
 					const { _content, ...rest } = m.metadata;
 					if (Object.keys(rest).length > 0) result.metadata = rest;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "retriv",
   "type": "module",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Unified document retrieval for search - semantic, full-text, and fuzzy.",
   "author": {
     "name": "Harlan Wilton",
@@ -159,6 +159,7 @@
     "lint": "eslint .",
     "typecheck": "tsc --noEmit",
     "test": "vitest --project unit",
+    "test:e2e": "vitest run --project e2e",
     "test:eval": "vitest run --project eval",
     "test:attw": "attw --pack",
     "release": "pnpm build && bumpp --output=CHANGELOG.md"