vecito 0.1.0

package/lib/bm25.js

@@ -0,0 +1,265 @@
+const DEFAULT_K1 = 1.2;
+const DEFAULT_B = 0.75;
+
+/**
+ * Tokenize text into lowercase terms for BM25. Splits on non-alphanumeric
+ * characters (keeping `-`, `_`, `.`) and drops single-character tokens.
+ * @param {string} text
+ * @returns {string[]} Lowercased tokens.
+ */
+function tokenize(text) {
+	return text
+		.toLowerCase()
+		.replace(/[^a-z0-9\-_\.]+/g, ' ')
+		.split(/\s+/)
+		.filter(t => t.length > 1);
+}
+
+/**
+ * BM25 sparse lexical model.
+ *
+ * Fit over a corpus to learn vocabulary and document frequencies, then produce
+ * sparse vectors (term-id → weight) for documents ({@link BM25#score}) and
+ * queries ({@link BM25#querySparse}). Serializable via {@link BM25#toJSON} /
+ * {@link BM25.fromJSON}.
+ */
+export class BM25 {
+	#vocab;       // term → id
+	#vocabSize;
+	#df;          // term id → document frequency
+	#docCount;
+	#avgDocLen;
+	#docLens;     // per-document token count
+	#k1;
+	#b;
+	#reverseVocab; // id → term, built lazily
+
+	/**
+	 * @param {object} [opts]
+	 * @param {number} [opts.k1=1.2] Term-frequency saturation parameter.
+	 * @param {number} [opts.b=0.75] Document-length normalization (0..1).
+	 */
+	constructor({ k1, b } = {}) {
+		this.#vocab = new Map();
+		this.#vocabSize = 0;
+		this.#df = [];
+		this.#docCount = 0;
+		this.#avgDocLen = 0;
+		this.#docLens = [];
+		this.#k1 = k1 ?? DEFAULT_K1;
+		this.#b = b ?? DEFAULT_B;
+		this.#reverseVocab = null;
+	}
+
+	/**
+	 * Build the vocabulary and corpus statistics (document frequencies, average
+	 * length) from a set of documents. Must be called before scoring.
+	 * @param {string[]} texts The full corpus.
+	 * @returns {void}
+	 */
+	fit(texts) {
+		const docs = texts.map(t => tokenize(t));
+
+		// Build vocabulary
+		for (const tokens of docs) {
+			for (const t of tokens) {
+				if (!this.#vocab.has(t)) {
+					this.#vocab.set(t, this.#vocabSize++);
+				}
+			}
+		}
+
+		// Compute document frequencies
+		this.#df = new Uint32Array(this.#vocabSize);
+		this.#docCount = docs.length;
+		let totalLen = 0;
+
+		for (const tokens of docs) {
+			const seen = new Set();
+			this.#docLens.push(tokens.length);
+			totalLen += tokens.length;
+			for (const t of tokens) {
+				const id = this.#vocab.get(t);
+				if (!seen.has(id)) {
+					this.#df[id]++;
+					seen.add(id);
+				}
+			}
+		}
+
+		this.#avgDocLen = totalLen / this.#docCount;
+	}
+
+	/**
+	 * Compute the BM25 sparse vector for a document. Out-of-vocabulary terms are
+	 * ignored.
+	 * @param {string} text Document text.
+	 * @returns {{indices: Uint32Array, values: Float32Array, dim: number}} Sorted
+	 *   sparse vector over the vocabulary.
+	 */
+	score(text) {
+		const tokens = tokenize(text);
+		// term frequency in this document
+		const tf = new Map();
+		for (const t of tokens) {
+			const id = this.#vocab.get(t);
+			if (id !== undefined) {
+				tf.set(id, (tf.get(id) || 0) + 1);
+			}
+		}
+		return this.#scoreFromTf(tf, tokens.length);
+	}
+
+	/**
+	 * Convenience wrapper that scores many documents.
+	 * @param {string[]} texts
+	 * @returns {Array<{indices: Uint32Array, values: Float32Array, dim: number}>}
+	 */
+	scoreAll(texts) {
+		return texts.map(t => this.score(t));
+	}
+
+	/**
+	 * Map a query string to the list of in-vocabulary term ids it contains.
+	 * @param {string} queryText
+	 * @returns {{indices: number[], vocabSize: number}} Matched term ids.
+	 */
+	scoreQuery(queryText) {
+		const queryTokens = tokenize(queryText);
+		const queryIds = [];
+		for (const t of queryTokens) {
+			const id = this.#vocab.get(t);
+			if (id !== undefined) queryIds.push(id);
+		}
+		return { indices: queryIds, vocabSize: this.#vocabSize };
+	}
+
+	/**
+	 * Build an IDF-weighted sparse vector for a query (assumes term frequency 1
+	 * per query term). Use this to drive sparse/hybrid search.
+	 * @param {string} queryText
+	 * @returns {{indices: Uint32Array, values: Float32Array, dim: number}} Sorted
+	 *   sparse vector; empty `indices` means no query term is in the vocabulary.
+	 */
+	querySparse(queryText) {
+		const tokens = tokenize(queryText);
+		const termScores = {};
+		const seen = new Set();
+
+		for (const t of tokens) {
+			const id = this.#vocab.get(t);
+			if (id === undefined || seen.has(id)) continue;
+			seen.add(id);
+			// IDF for this term
+			const df = this.#df[id];
+			const idf = Math.log((this.#docCount - df + 0.5) / (df + 0.5) + 1);
+			// Query term gets weight = IDF (tf=1 in query)
+			if (idf > 0) termScores[id] = idf;
+		}
+
+		return toSparse(termScores, this.#vocabSize);
+	}
+
+	/**
+	 * Apply the BM25 weighting formula to a term-frequency map.
+	 * @param {Map<number, number>} tf term id → frequency in the document.
+	 * @param {number} docLen Document length in tokens.
+	 * @returns {{indices: Uint32Array, values: Float32Array, dim: number}}
+	 */
+	#scoreFromTf(tf, docLen) {
+		const termScores = {};
+		const N = this.#docCount;
+		const avgDl = this.#avgDocLen;
+		const k1 = this.#k1;
+		const b = this.#b;
+
+		for (const [id, freq] of tf) {
+			const df = this.#df[id];
+			const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+			const tfNorm = (freq * (k1 + 1)) / (freq + k1 * (1 - b + b * docLen / avgDl));
+			const s = idf * tfNorm;
+			if (s > 0) termScores[id] = s;
+		}
+
+		return toSparse(termScores, this.#vocabSize);
+	}
+
+	/**
+	 * Serialize the fitted model to a plain JSON-safe object.
+	 * @returns {object} Pass to {@link BM25.fromJSON} to restore.
+	 */
+	toJSON() {
+		return {
+			vocab: Object.fromEntries(this.#vocab),
+			df: Array.from(this.#df),
+			docCount: this.#docCount,
+			avgDocLen: this.#avgDocLen,
+			docLens: this.#docLens,
+			k1: this.#k1,
+			b: this.#b,
+		};
+	}
+
+	/**
+	 * Reconstruct a fitted model from {@link BM25#toJSON} output.
+	 * @param {object} data
+	 * @returns {BM25}
+	 */
+	static fromJSON(data) {
+		const bm25 = new BM25({ k1: data.k1, b: data.b });
+		bm25.#vocab = new Map(Object.entries(data.vocab).map(([k, v]) => [k, Number(v)]));
+		bm25.#vocabSize = bm25.#vocab.size;
+		bm25.#df = new Uint32Array(data.df);
+		bm25.#docCount = data.docCount;
+		bm25.#avgDocLen = data.avgDocLen;
+		bm25.#docLens = data.docLens;
+		return bm25;
+	}
+
+	/**
+	 * Map a list of vocabulary indices back to their original term strings.
+	 * The reverse map is built lazily on first call and cached.
+	 * @param {Uint32Array|number[]} indices Vocabulary term ids.
+	 * @returns {string[]} The corresponding terms (unknown ids silently omitted).
+	 */
+	termsForIndices(indices) {
+		if (!this.#reverseVocab) {
+			this.#reverseVocab = new Map();
+			for (const [term, id] of this.#vocab) this.#reverseVocab.set(id, term);
+		}
+		const out = [];
+		for (const id of indices) {
+			const term = this.#reverseVocab.get(id);
+			if (term !== undefined) out.push(term);
+		}
+		return out;
+	}
+
+	/**
+	 * Number of distinct terms in the fitted vocabulary (the sparse dimension).
+	 * @returns {number}
+	 */
+	get vocabSize() {
+		return this.#vocabSize;
+	}
+}
+
+/**
+ * Convert a term-id → score map into a sorted sparse vector, dropping
+ * non-positive weights.
+ * @param {Record<string, number>} termScores
+ * @param {number} dim Vocabulary size.
+ * @returns {{indices: Uint32Array, values: Float32Array, dim: number}}
+ */
+function toSparse(termScores, dim) {
+	const entries = Object.entries(termScores)
+		.map(([id, score]) => [Number(id), score])
+		.filter(([, s]) => s > 0)
+		.sort((a, b) => a[0] - b[0]);
+
+	return {
+		indices: new Uint32Array(entries.map(e => e[0])),
+		values: new Float32Array(entries.map(e => e[1])),
+		dim,
+	};
+}

package/lib/embedder.js

@@ -0,0 +1,114 @@
+import { pipeline } from '@huggingface/transformers';
+
+/**
+ * Dense text embedder backed by transformers.js.
+ *
+ * Produces L2-normalized, mean-pooled sentence embeddings. The model is loaded
+ * lazily on first use and cached for the lifetime of the instance.
+ */
+export class Embedder {
+	#model;
+	#dtype;
+	#pipe;
+	#dimensions;
+
+	/**
+	 * @param {object} [opts]
+	 * @param {string} [opts.model='Xenova/all-MiniLM-L6-v2'] Hugging Face model id
+	 *   for a feature-extraction pipeline. Any output width is supported — the
+	 *   actual dimension is detected at {@link Embedder#init} (see
+	 *   {@link Embedder#dimensions}).
+	 * @param {string} [opts.dtype='q8'] Weight precision to load: `'q8'`
+	 *   (quantized, ~4× smaller download, the default) or `'fp32'` (full
+	 *   precision), plus other transformers.js dtypes (`'fp16'`, `'q4'`, …) when
+	 *   the model provides them.
+	 */
+	constructor({ model, dtype } = {}) {
+		this.#model = model || 'Xenova/all-MiniLM-L6-v2';
+		this.#dtype = dtype || 'q8';
+		this.#pipe = null;
+		this.#dimensions = null;
+	}
+
+	/**
+	 * Load the underlying pipeline if it hasn't been loaded yet, and detect the
+	 * model's embedding width with a one-token probe. Safe to call repeatedly;
+	 * subsequent calls are no-ops. Called automatically by {@link Embedder#embed}
+	 * / {@link Embedder#embedBatch}.
+	 * @returns {Promise<void>}
+	 */
+	async init() {
+		if (!this.#pipe) {
+			this.#pipe = await pipeline('feature-extraction', this.#model, {
+				dtype: this.#dtype,
+			});
+			// Detect the real embedding width (384, 768, …) rather than assuming.
+			const probe = await this.#pipe('x', { pooling: 'mean', normalize: true });
+			this.#dimensions = probe.data.length;
+		}
+	}
+
+	/**
+	 * Embed a single string into a normalized dense vector.
+	 * @param {string} text Input text.
+	 * @returns {Promise<Float32Array>} A {@link Embedder#dimensions}-length vector.
+	 */
+	async embed(text) {
+		await this.init();
+		const output = await this.#pipe(text, { pooling: 'mean', normalize: true });
+		return new Float32Array(output.data);
+	}
+
+	/**
+	 * Embed many strings, processing them in batches for throughput.
+	 * @param {string[]} texts Input texts.
+	 * @param {object} [opts]
+	 * @param {number} [opts.batchSize=32] Number of texts per forward pass.
+	 * @returns {Promise<Float32Array[]>} One vector per input, in input order.
+	 *   Each is a view into a shared buffer — copy it if you need to retain it
+	 *   independently.
+	 */
+	async embedBatch(texts, { batchSize = 32 } = {}) {
+		await this.init();
+		const dims = this.dimensions;
+		const results = [];
+		for (let i = 0; i < texts.length; i += batchSize) {
+			const batch = texts.slice(i, i + batchSize);
+			const output = await this.#pipe(batch, { pooling: 'mean', normalize: true });
+			for (let j = 0; j < batch.length; j++) {
+				results.push(new Float32Array(output.data.buffer, output.data.byteOffset + j * dims * 4, dims));
+			}
+		}
+		return results;
+	}
+
+	/**
+	 * Dimensionality of the vectors this model produces. Available only after
+	 * {@link Embedder#init} (or a first {@link Embedder#embed}) has run, since it
+	 * is detected from the loaded model.
+	 * @returns {number} e.g. 384 for MiniLM/BGE-small, 768 for MPNet/GTE-base.
+	 * @throws {Error} If called before the model has been initialized.
+	 */
+	get dimensions() {
+		if (this.#dimensions == null) {
+			throw new Error('Embedder.dimensions is only known after init() or embed()');
+		}
+		return this.#dimensions;
+	}
+
+	/**
+	 * The Hugging Face model id this embedder uses.
+	 * @returns {string}
+	 */
+	get model() {
+		return this.#model;
+	}
+
+	/**
+	 * Weight precision this embedder loads (e.g. `'q8'`, `'fp32'`).
+	 * @returns {string}
+	 */
+	get dtype() {
+		return this.#dtype;
+	}
+}

package/lib/extract.js

@@ -0,0 +1,43 @@
+/**
+ * Universal (Node + browser) helpers for turning arbitrary data into the text
+ * Vecito indexes and the metadata it returns with hits. No filesystem deps.
+ */
+
+/**
+ * Recursively collect string leaves from any value and join them. Lets raw JSON
+ * objects/arrays be indexed without the caller pre-extracting their text.
+ * @param {*} value Any value — object, array, string, etc.
+ * @returns {string} All string leaves, '. '-joined (empty string if none).
+ */
+export function flattenStrings(value) {
+	const parts = [];
+	const visit = v => {
+		if (typeof v === 'string') parts.push(v);
+		else if (Array.isArray(v)) v.forEach(visit);
+		else if (v && typeof v === 'object') Object.values(v).forEach(visit);
+	};
+	visit(value);
+	return parts.join('. ');
+}
+
+/**
+ * Default text extractor: strings pass through; objects/arrays are flattened to
+ * their string leaves; everything else is stringified.
+ * @param {*} item
+ * @returns {string}
+ */
+export function defaultText(item) {
+	if (typeof item === 'string') return item;
+	if (item && typeof item === 'object') return flattenStrings(item);
+	return item == null ? '' : String(item);
+}
+
+/**
+ * Default metadata extractor: objects are returned as-is (so search hits carry
+ * the original data back); non-objects carry no metadata.
+ * @param {*} item
+ * @returns {Record<string, any>}
+ */
+export function defaultMetadata(item) {
+	return item && typeof item === 'object' && !Array.isArray(item) ? item : {};
+}

package/lib/file-index.js

@@ -0,0 +1,118 @@
+import { readdirSync, readFileSync, statSync } from 'fs';
+import { join, basename, relative } from 'path';
+import { Vecito } from './vecito.js';
+import { defaultText } from './extract.js';
+
+/**
+ * Filesystem layer on top of the core {@link Vecito} library. Turns files and
+ * directories into data items and feeds them to the generic indexer. Node-only
+ * (imports `fs`/`path`); use it via the `vecito/file` subpath export.
+ */
+
+/** Broad default set of text-ish extensions. Override with the `ext` option. */
+export const DEFAULT_EXTENSIONS = [
+	// docs / text
+	'.md', '.markdown', '.mdx', '.txt', '.text', '.rst', '.org', '.tex', '.log',
+	// data / config
+	'.json', '.jsonl', '.ndjson', '.csv', '.tsv', '.yaml', '.yml', '.toml',
+	'.ini', '.cfg', '.conf', '.xml', '.html', '.htm',
+	// code
+	'.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.py', '.rb', '.php', '.go',
+	'.rs', '.java', '.c', '.h', '.cpp', '.hpp', '.cs', '.sh', '.sql', '.css',
+	'.scss', '.move', '.sol',
+];
+
+/**
+ * Normalize an extension list to lowercase, dot-prefixed entries.
+ * @param {string[]} exts
+ * @returns {string[]}
+ */
+function normalizeExts(exts) {
+	return exts.map(e => (e.startsWith('.') ? e : '.' + e).toLowerCase());
+}
+
+/**
+ * Recursively collect files under `dir` whose extension is allowed. Skips
+ * dotfiles and dot-directories unless `hidden` is true.
+ * @param {string} dir Directory to walk.
+ * @param {object} [opts]
+ * @param {string[]} [opts.ext=DEFAULT_EXTENSIONS] Extensions to include.
+ * @param {boolean} [opts.hidden=false] Include entries whose name starts with '.'.
+ * @param {number} [opts.limit=Infinity] Stop after this many files.
+ * @returns {string[]} Matching file paths.
+ */
+export function walk(dir, { ext = DEFAULT_EXTENSIONS, hidden = false, limit = Infinity } = {}) {
+	const exts = normalizeExts(ext);
+	const out = [];
+	const recurse = current => {
+		for (const entry of readdirSync(current)) {
+			if (out.length >= limit) return;
+			// Skip dotfiles / dot-directories by default (.git, .env, .DS_Store, ...).
+			if (!hidden && entry.startsWith('.')) continue;
+			const full = join(current, entry);
+			const st = statSync(full);
+			if (st.isDirectory()) {
+				recurse(full);
+			} else if (st.isFile()) {
+				const lower = entry.toLowerCase();
+				if (exts.some(e => lower.endsWith(e))) out.push(full);
+			}
+		}
+	};
+	recurse(dir);
+	return out.slice(0, limit);
+}
+
+/**
+ * Read a file into a data item. `.json`/`.jsonl`/`.ndjson` are parsed to objects
+ * (falling back to raw text on parse failure); everything else stays a string.
+ * @param {string} file Absolute or relative file path.
+ * @param {string} [base] Base dir for the item's relative `path` metadata.
+ * @returns {{data: any, path: string, name: string}}
+ */
+function readItem(file, base) {
+	const raw = readFileSync(file, 'utf-8');
+	const lower = file.toLowerCase();
+	let data = raw;
+	if (lower.endsWith('.json') || lower.endsWith('.jsonl') || lower.endsWith('.ndjson')) {
+		try { data = JSON.parse(raw); } catch { data = raw; }
+	}
+	return { data, path: base ? relative(base, file) : file, name: basename(file) };
+}
+
+/**
+ * Index an explicit list of files into a fresh {@link Vecito}.
+ * @param {string[]} paths File paths to index (one document each).
+ * @param {object} [opts]
+ * @param {string} [opts.model] Embedding model id passed to Vecito.
+ * @param {string} [opts.dtype] Weight precision passed to Vecito.
+ * @param {'hybrid'|'dense'} [opts.mode='hybrid'] Index mode passed to Vecito.
+ * @param {string} [opts.base] Base dir for relative `path` metadata.
+ * @returns {Promise<Vecito>}
+ */
+export async function indexFiles(paths, { model, dtype, mode, base } = {}) {
+	const items = paths.map(p => readItem(p, base));
+	const vecito = new Vecito({ model, dtype, mode });
+	await vecito.addDocuments(items, {
+		text: it => defaultText(it.data).trim(),
+		metadata: it => ({ path: it.path, name: it.name }),
+	});
+	return vecito;
+}
+
+/**
+ * Walk a directory and index every matching file into a fresh {@link Vecito}.
+ * @param {string} dir Directory to index.
+ * @param {object} [opts]
+ * @param {string[]} [opts.ext=DEFAULT_EXTENSIONS] Extensions to include.
+ * @param {boolean} [opts.hidden=false] Include dotfiles/dot-directories.
+ * @param {number} [opts.limit=Infinity] Index at most this many files.
+ * @param {string} [opts.model] Embedding model id passed to Vecito.
+ * @param {string} [opts.dtype] Weight precision passed to Vecito.
+ * @param {'hybrid'|'dense'} [opts.mode='hybrid'] Index mode passed to Vecito.
+ * @returns {Promise<Vecito>}
+ */
+export async function indexDirectory(dir, { ext, hidden, limit, model, dtype, mode } = {}) {
+	const files = walk(dir, { ext, hidden, limit });
+	return indexFiles(files, { model, dtype, mode, base: dir });
+}