sparse-encode 0.1.0

package/README.md

@@ -0,0 +1,102 @@
+# sparse-encode
+
+Generate BM25 and TF-IDF sparse vectors in JavaScript/TypeScript. Designed for use with sparse vector search engines (e.g., Pinecone sparse indexes, Qdrant sparse vectors).
+
+## Install
+
+```bash
+npm install sparse-encode
+```
+
+## Quick Start
+
+```typescript
+import { createBM25, createTFIDF } from 'sparse-encode'
+
+const docs = [
+  'the quick brown fox jumps over the lazy dog',
+  'the dog barked loudly at the fox',
+  'cats and dogs are common pets',
+]
+
+// BM25
+const bm25 = createBM25({ k1: 1.5, b: 0.75 })
+bm25.fit(docs)
+const vec = bm25.encode('quick fox')
+// { indices: [2, 5, ...], values: [0.82, 1.23, ...] }
+
+const queryVec = bm25.encodeQuery('fox')
+
+// TF-IDF
+const tfidf = createTFIDF({ sublinearTf: false })
+tfidf.fit(docs)
+const tVec = tfidf.encode('quick fox')
+// L2-normalized: sum of squares ≈ 1.0
+```
+
+## BM25 Encoder
+
+```typescript
+const enc = createBM25(options?)
+enc.fit(documents: string[])           // build vocabulary + IDF statistics
+enc.encode(text: string): SparseVector // encode a document
+enc.encodeQuery(text: string): SparseVector // encode a query (no length norm)
+enc.encodeBatch(texts: string[]): SparseVector[]
+enc.getStats(): FitStats               // { N, avgdl, vocabSize, totalTokens }
+enc.serialize(): string                // JSON snapshot (vocab + df + options)
+```
+
+### Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `k1` | `1.5` | Term frequency saturation parameter |
+| `b` | `0.75` | Length normalization parameter |
+| `stem` | `true` | Apply Porter stemmer |
+| `stopwords` | `[]` | Additional stopwords to remove |
+| `tokenizer` | built-in | Custom tokenizer function |
+
+## TF-IDF Encoder
+
+```typescript
+const enc = createTFIDF(options?)
+enc.fit(documents: string[])
+enc.encode(text: string): SparseVector  // L2-normalized TF-IDF vector
+enc.encodeQuery(text: string): SparseVector
+enc.encodeBatch(texts: string[]): SparseVector[]
+enc.getStats(): FitStats
+enc.serialize(): string
+```
+
+### Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `stem` | `true` | Apply Porter stemmer |
+| `stopwords` | `[]` | Additional stopwords to remove |
+| `sublinearTf` | `false` | Use `1 + log(tf)` instead of `tf / doc_length` |
+| `tokenizer` | built-in | Custom tokenizer function |
+
+## SparseVector Format
+
+```typescript
+interface SparseVector {
+  indices: number[]  // term IDs (sorted ascending)
+  values: number[]   // corresponding scores
+}
+```
+
+Each position `i` maps `indices[i]` → `values[i]`. Zero values are omitted.
+
+## Tokenizer Pipeline
+
+The built-in tokenizer:
+1. Lowercases the input
+2. Splits on non-word characters
+3. Removes pure numbers
+4. Removes common English stopwords
+5. Applies the Porter stemmer (can be disabled with `stem: false`)
+
+## License
+
+MIT

package/dist/__tests__/bm25.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=bm25.test.d.ts.map

package/dist/__tests__/bm25.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/bm25.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/bm25.test.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const bm25_1 = require("../bm25");
+(0, vitest_1.describe)('createBM25', () => {
+    const docs = [
+        'the quick brown fox jumps over the lazy dog',
+        'the dog barked loudly at the fox',
+        'cats and dogs are common pets',
+    ];
+    (0, vitest_1.it)('throws if encode called before fit', () => {
+        const enc = (0, bm25_1.createBM25)();
+        (0, vitest_1.expect)(() => enc.encode('test')).toThrow();
+    });
+    (0, vitest_1.it)('returns a SparseVector with indices and values', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const vec = enc.encode('fox');
+        (0, vitest_1.expect)(Array.isArray(vec.indices)).toBe(true);
+        (0, vitest_1.expect)(Array.isArray(vec.values)).toBe(true);
+        (0, vitest_1.expect)(vec.indices.length).toBe(vec.values.length);
+    });
+    (0, vitest_1.it)('encodes to non-empty vector for known term', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const vec = enc.encode('fox');
+        (0, vitest_1.expect)(vec.indices.length).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)('indices are sorted ascending', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const vec = enc.encode('quick brown fox');
+        for (let i = 1; i < vec.indices.length; i++) {
+            (0, vitest_1.expect)(vec.indices[i]).toBeGreaterThan(vec.indices[i - 1]);
+        }
+    });
+    (0, vitest_1.it)('term in doc scores higher than term not in corpus', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const vec = enc.encode('fox');
+        const foxIdx = vec.indices.findIndex(i => i >= 0);
+        // vector should have at least one positive value
+        (0, vitest_1.expect)(vec.values[foxIdx]).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)('getStats returns correct N and vocabSize', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const stats = enc.getStats();
+        (0, vitest_1.expect)(stats.N).toBe(3);
+        (0, vitest_1.expect)(stats.vocabSize).toBeGreaterThan(0);
+        (0, vitest_1.expect)(stats.avgdl).toBeGreaterThan(0);
+        (0, vitest_1.expect)(stats.totalTokens).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)('encodeBatch returns one vector per text', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const vecs = enc.encodeBatch(['fox', 'dog', 'cat']);
+        (0, vitest_1.expect)(vecs).toHaveLength(3);
+    });
+    (0, vitest_1.it)('encodeQuery returns a SparseVector', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const vec = enc.encodeQuery('brown fox');
+        (0, vitest_1.expect)(Array.isArray(vec.indices)).toBe(true);
+        (0, vitest_1.expect)(Array.isArray(vec.values)).toBe(true);
+    });
+    (0, vitest_1.it)('serialize returns valid JSON', () => {
+        const enc = (0, bm25_1.createBM25)();
+        enc.fit(docs);
+        const json = enc.serialize();
+        (0, vitest_1.expect)(() => JSON.parse(json)).not.toThrow();
+        const parsed = JSON.parse(json);
+        (0, vitest_1.expect)(parsed.N).toBe(3);
+        (0, vitest_1.expect)(parsed.vocab).toBeDefined();
+    });
+    (0, vitest_1.it)('custom k1/b options affect scores', () => {
+        const enc1 = (0, bm25_1.createBM25)({ k1: 1.0, b: 0.5 });
+        const enc2 = (0, bm25_1.createBM25)({ k1: 2.0, b: 0.9 });
+        enc1.fit(docs);
+        enc2.fit(docs);
+        const v1 = enc1.encode('fox');
+        const v2 = enc2.encode('fox');
+        // Scores differ with different parameters
+        const sum1 = v1.values.reduce((a, b) => a + b, 0);
+        const sum2 = v2.values.reduce((a, b) => a + b, 0);
+        (0, vitest_1.expect)(sum1).not.toBeCloseTo(sum2, 5);
+    });
+});
+//# sourceMappingURL=bm25.test.js.map

package/dist/__tests__/bm25.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.test.js","sourceRoot":"","sources":["../../src/__tests__/bm25.test.ts"],"names":[],"mappings":";;AAAA,mCAA6C;AAC7C,kCAAoC;AAEpC,IAAA,iBAAQ,EAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,MAAM,IAAI,GAAG;QACX,6CAA6C;QAC7C,kCAAkC;QAClC,+BAA+B;KAChC,CAAA;IAED,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,IAAA,eAAM,EAAC,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAA,eAAM,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7C,IAAA,eAAM,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAC5D,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QACjD,iDAAiD;QACjD,IAAA,eAAM,EAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAA;QAC5B,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,IAAA,eAAM,EAAC,KAAK,CAAC,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAC1C,IAAA,eAAM,EAAC,KAAK,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACtC,IAAA,eAAM,EAAC,KAAK,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAA;QACnD,IAAA,eAAM,EAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAA;QACxC,IAAA,eAAM,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7C,IAAA,eAAM,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,GAAG,GAAG,IAAA,iBAAU,GAAE,CAAA;QACxB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,EAAE,CAAA;QAC5B,IAAA,eAAM,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAA;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxB,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IACpC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,IAAI,GAAG,IAAA,iBAAU,EAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;QAC5C,MAAM,IAAI,GAAG,IAAA,iBAAU,EAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;QAC5C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,0CAA0C;QAC1C,MAAM,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QACjD,MAAM,IAAI,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QACjD,IAAA,eAAM,EAAC,IAAI,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IACvC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/dist/__tests__/tfidf.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tfidf.test.d.ts.map

package/dist/__tests__/tfidf.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/tfidf.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/tfidf.test.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const tfidf_1 = require("../tfidf");
+(0, vitest_1.describe)('createTFIDF', () => {
+    const docs = [
+        'the quick brown fox jumps over the lazy dog',
+        'the dog barked loudly at the fox',
+        'cats and dogs are common pets',
+    ];
+    (0, vitest_1.it)('throws if encode called before fit', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        (0, vitest_1.expect)(() => enc.encode('test')).toThrow();
+    });
+    (0, vitest_1.it)('returns a SparseVector with indices and values', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encode('fox');
+        (0, vitest_1.expect)(Array.isArray(vec.indices)).toBe(true);
+        (0, vitest_1.expect)(Array.isArray(vec.values)).toBe(true);
+        (0, vitest_1.expect)(vec.indices.length).toBe(vec.values.length);
+    });
+    (0, vitest_1.it)('encodes to non-empty vector for known term', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encode('fox');
+        (0, vitest_1.expect)(vec.indices.length).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)('indices are sorted ascending', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encode('quick brown fox');
+        for (let i = 1; i < vec.indices.length; i++) {
+            (0, vitest_1.expect)(vec.indices[i]).toBeGreaterThan(vec.indices[i - 1]);
+        }
+    });
+    (0, vitest_1.it)('output vector is unit-length (L2 norm ≈ 1)', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encode('quick brown fox');
+        const sumSq = vec.values.reduce((acc, v) => acc + v * v, 0);
+        (0, vitest_1.expect)(sumSq).toBeCloseTo(1.0, 5);
+    });
+    (0, vitest_1.it)('unit-length holds for single-term query', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encode('fox');
+        if (vec.values.length > 0) {
+            const sumSq = vec.values.reduce((acc, v) => acc + v * v, 0);
+            (0, vitest_1.expect)(sumSq).toBeCloseTo(1.0, 5);
+        }
+    });
+    (0, vitest_1.it)('getStats returns correct N and vocabSize', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const stats = enc.getStats();
+        (0, vitest_1.expect)(stats.N).toBe(3);
+        (0, vitest_1.expect)(stats.vocabSize).toBeGreaterThan(0);
+        (0, vitest_1.expect)(stats.totalTokens).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)('encodeBatch returns one vector per text', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vecs = enc.encodeBatch(['fox', 'dog', 'cat']);
+        (0, vitest_1.expect)(vecs).toHaveLength(3);
+    });
+    (0, vitest_1.it)('encodeQuery returns unit-length SparseVector', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encodeQuery('brown fox');
+        if (vec.values.length > 0) {
+            const sumSq = vec.values.reduce((acc, v) => acc + v * v, 0);
+            (0, vitest_1.expect)(sumSq).toBeCloseTo(1.0, 5);
+        }
+    });
+    (0, vitest_1.it)('serialize returns valid JSON', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const json = enc.serialize();
+        (0, vitest_1.expect)(() => JSON.parse(json)).not.toThrow();
+        const parsed = JSON.parse(json);
+        (0, vitest_1.expect)(parsed.N).toBe(3);
+        (0, vitest_1.expect)(parsed.vocab).toBeDefined();
+    });
+    (0, vitest_1.it)('sublinearTf option changes scores', () => {
+        const enc1 = (0, tfidf_1.createTFIDF)({ sublinearTf: false });
+        const enc2 = (0, tfidf_1.createTFIDF)({ sublinearTf: true });
+        enc1.fit(docs);
+        enc2.fit(docs);
+        // Use a text with repeated terms to make sublinear difference visible
+        const text = 'fox fox fox quick';
+        const v1 = enc1.encode(text);
+        const v2 = enc2.encode(text);
+        // Both should be unit vectors but with different distributions
+        if (v1.values.length > 0 && v2.values.length > 0) {
+            const s1 = v1.values.reduce((a, b) => a + b * b, 0);
+            const s2 = v2.values.reduce((a, b) => a + b * b, 0);
+            (0, vitest_1.expect)(s1).toBeCloseTo(1.0, 5);
+            (0, vitest_1.expect)(s2).toBeCloseTo(1.0, 5);
+        }
+    });
+    (0, vitest_1.it)('empty text returns empty vector', () => {
+        const enc = (0, tfidf_1.createTFIDF)();
+        enc.fit(docs);
+        const vec = enc.encode('');
+        (0, vitest_1.expect)(vec.indices).toHaveLength(0);
+        (0, vitest_1.expect)(vec.values).toHaveLength(0);
+    });
+});
+//# sourceMappingURL=tfidf.test.js.map

package/dist/__tests__/tfidf.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.test.js","sourceRoot":"","sources":["../../src/__tests__/tfidf.test.ts"],"names":[],"mappings":";;AAAA,mCAA6C;AAC7C,oCAAsC;AAEtC,IAAA,iBAAQ,EAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,IAAI,GAAG;QACX,6CAA6C;QAC7C,kCAAkC;QAClC,+BAA+B;KAChC,CAAA;IAED,IAAA,WAAE,EAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,IAAA,eAAM,EAAC,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAA,eAAM,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7C,IAAA,eAAM,EAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;QAC5D,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAC3D,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YAC3D,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QACnC,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAA;QAC5B,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACvB,IAAA,eAAM,EAAC,KAAK,CAAC,SAAS,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAC1C,IAAA,eAAM,EAAC,KAAK,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAA;QACnD,IAAA,eAAM,EAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YAC3D,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QACnC,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,8BAA8B,EAAE,GAAG,EAAE;QACtC,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,EAAE,CAAA;QAC5B,IAAA,eAAM,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAA;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC/B,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxB,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IACpC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,IAAI,GAAG,IAAA,mBAAW,EAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAA;QAChD,MAAM,IAAI,GAAG,IAAA,mBAAW,EAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAC/C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,sEAAsE;QACtE,MAAM,IAAI,GAAG,mBAAmB,CAAA;QAChC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC5B,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC5B,+DAA+D;QAC/D,IAAI,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YACnD,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;YACnD,IAAA,eAAM,EAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YAC9B,IAAA,eAAM,EAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QAChC,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,GAAG,GAAG,IAAA,mBAAW,GAAE,CAAA;QACzB,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACb,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC1B,IAAA,eAAM,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACnC,IAAA,eAAM,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/dist/__tests__/tokenizer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tokenizer.test.d.ts.map

package/dist/__tests__/tokenizer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/tokenizer.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/tokenizer.test.js

@@ -0,0 +1,51 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const tokenizer_1 = require("../tokenizer");
+(0, vitest_1.describe)('defaultTokenizer', () => {
+    (0, vitest_1.it)('lowercases input', () => {
+        const tokens = (0, tokenizer_1.defaultTokenizer)('Hello World');
+        (0, vitest_1.expect)(tokens.every(t => t === t.toLowerCase())).toBe(true);
+    });
+    (0, vitest_1.it)('removes stopwords', () => {
+        const tokens = (0, tokenizer_1.defaultTokenizer)('the cat sat on the mat');
+        (0, vitest_1.expect)(tokens).not.toContain('the');
+        (0, vitest_1.expect)(tokens).not.toContain('on');
+    });
+    (0, vitest_1.it)('removes pure numbers', () => {
+        const tokens = (0, tokenizer_1.defaultTokenizer)('I have 42 cats');
+        (0, vitest_1.expect)(tokens).not.toContain('42');
+    });
+    (0, vitest_1.it)('splits on non-word characters', () => {
+        const tokens = (0, tokenizer_1.defaultTokenizer)('hello-world foo.bar');
+        (0, vitest_1.expect)(tokens.length).toBeGreaterThanOrEqual(2);
+    });
+    (0, vitest_1.it)('applies Porter stemmer (running → run)', () => {
+        const tokens = (0, tokenizer_1.defaultTokenizer)('running');
+        // porter stemmer maps running → run
+        (0, vitest_1.expect)(tokens).toContain('run');
+    });
+    (0, vitest_1.it)('applies Porter stemmer (cats → cat)', () => {
+        const tokens = (0, tokenizer_1.defaultTokenizer)('cats');
+        (0, vitest_1.expect)(tokens).toContain('cat');
+    });
+});
+(0, vitest_1.describe)('tokenize', () => {
+    (0, vitest_1.it)('uses custom tokenizer when provided', () => {
+        const custom = (t) => t.split(',');
+        const tokens = (0, tokenizer_1.tokenize)('a,b,c', { tokenizer: custom });
+        (0, vitest_1.expect)(tokens).toEqual(['a', 'b', 'c']);
+    });
+    (0, vitest_1.it)('applies extra stopwords', () => {
+        const tokens = (0, tokenizer_1.tokenize)('cat dog bird', { stem: false, stopwords: ['dog'] });
+        (0, vitest_1.expect)(tokens).not.toContain('dog');
+        (0, vitest_1.expect)(tokens).toContain('cat');
+        (0, vitest_1.expect)(tokens).toContain('bird');
+    });
+    (0, vitest_1.it)('skips stemming when stem=false', () => {
+        const tokens = (0, tokenizer_1.tokenize)('running cats', { stem: false });
+        (0, vitest_1.expect)(tokens).toContain('running');
+        (0, vitest_1.expect)(tokens).toContain('cats');
+    });
+});
+//# sourceMappingURL=tokenizer.test.js.map

package/dist/__tests__/tokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.js","sourceRoot":"","sources":["../../src/__tests__/tokenizer.test.ts"],"names":[],"mappings":";;AAAA,mCAA6C;AAC7C,4CAAyD;AAEzD,IAAA,iBAAQ,EAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,IAAA,WAAE,EAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,MAAM,GAAG,IAAA,4BAAgB,EAAC,aAAa,CAAC,CAAA;QAC9C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,mBAAmB,EAAE,GAAG,EAAE;QAC3B,MAAM,MAAM,GAAG,IAAA,4BAAgB,EAAC,wBAAwB,CAAC,CAAA;QACzD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACnC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,MAAM,GAAG,IAAA,4BAAgB,EAAC,gBAAgB,CAAC,CAAA;QACjD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,+BAA+B,EAAE,GAAG,EAAE;QACvC,MAAM,MAAM,GAAG,IAAA,4BAAgB,EAAC,qBAAqB,CAAC,CAAA;QACtD,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,MAAM,GAAG,IAAA,4BAAgB,EAAC,SAAS,CAAC,CAAA;QAC1C,oCAAoC;QACpC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG,IAAA,4BAAgB,EAAC,MAAM,CAAC,CAAA;QACvC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,IAAA,iBAAQ,EAAC,UAAU,EAAE,GAAG,EAAE;IACxB,IAAA,WAAE,EAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC1C,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;QACvD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,cAAc,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC5E,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QACnC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAC/B,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;IAEF,IAAA,WAAE,EAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,MAAM,GAAG,IAAA,oBAAQ,EAAC,cAAc,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;QACxD,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAA;QACnC,IAAA,eAAM,EAAC,MAAM,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/dist/bm25.d.ts

@@ -0,0 +1,3 @@
+import type { BM25Options, BM25Encoder } from './types';
+export declare function createBM25(options?: BM25Options): BM25Encoder;
+//# sourceMappingURL=bm25.d.ts.map

package/dist/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAA0B,MAAM,SAAS,CAAA;AAE/E,wBAAgB,UAAU,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,WAAW,CA0G7D"}

package/dist/bm25.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createBM25 = createBM25;
+const tokenizer_1 = require("./tokenizer");
+const vocab_1 = require("./vocab");
+function createBM25(options) {
+    const k1 = options?.k1 ?? 1.5;
+    const b = options?.b ?? 0.75;
+    const stemOpt = options?.stem ?? true;
+    const tokenizerFn = options?.tokenizer;
+    const extraStopwords = options?.stopwords ?? [];
+    const vocab = new vocab_1.Vocabulary();
+    // df[termId] = number of documents containing that term
+    const df = new Map();
+    let N = 0;
+    let avgdl = 0;
+    let totalTokens = 0;
+    let fitted = false;
+    function tokenizeText(text) {
+        return (0, tokenizer_1.tokenize)(text, { stem: stemOpt, tokenizer: tokenizerFn, stopwords: extraStopwords });
+    }
+    function fit(documents) {
+        N = documents.length;
+        let totalLen = 0;
+        const tokenizedDocs = [];
+        for (const doc of documents) {
+            const tokens = tokenizeText(doc);
+            tokenizedDocs.push(tokens);
+            totalLen += tokens.length;
+            totalTokens += tokens.length;
+            // Register all terms in vocab first
+            const seen = new Set();
+            for (const token of tokens) {
+                const id = vocab.getOrAdd(token);
+                if (!seen.has(id)) {
+                    seen.add(id);
+                    df.set(id, (df.get(id) ?? 0) + 1);
+                }
+            }
+        }
+        avgdl = N > 0 ? totalLen / N : 0;
+        fitted = true;
+    }
+    function computeBM25(tokens, dl) {
+        if (!fitted)
+            throw new Error('BM25Encoder must be fit() before encode()');
+        // Count term frequencies
+        const tf = new Map();
+        for (const token of tokens) {
+            const id = vocab.getId(token);
+            if (id !== undefined) {
+                tf.set(id, (tf.get(id) ?? 0) + 1);
+            }
+        }
+        const entries = [];
+        for (const [termId, termTf] of tf) {
+            const termDf = df.get(termId) ?? 0;
+            const idf = Math.log((N - termDf + 0.5) / (termDf + 0.5) + 1);
+            const score = idf * (termTf * (k1 + 1)) / (termTf + k1 * (1 - b + b * dl / avgdl));
+            if (score > 0) {
+                entries.push({ idx: termId, val: score });
+            }
+        }
+        entries.sort((a, c) => a.idx - c.idx);
+        return {
+            indices: entries.map(e => e.idx),
+            values: entries.map(e => e.val),
+        };
+    }
+    function encode(text) {
+        const tokens = tokenizeText(text);
+        return computeBM25(tokens, tokens.length);
+    }
+    function encodeBatch(texts) {
+        return texts.map(t => encode(t));
+    }
+    function encodeQuery(text) {
+        // Query encoding: no length normalization (treat avgdl as doc length)
+        if (!fitted)
+            throw new Error('BM25Encoder must be fit() before encodeQuery()');
+        const tokens = tokenizeText(text);
+        return computeBM25(tokens, avgdl);
+    }
+    function serialize() {
+        return JSON.stringify({
+            N,
+            avgdl,
+            totalTokens,
+            df: Object.fromEntries(df),
+            vocab: vocab.serialize(),
+            options: { k1, b, stem: stemOpt, stopwords: extraStopwords },
+        });
+    }
+    function getStats() {
+        return { N, avgdl, vocabSize: vocab.size, totalTokens };
+    }
+    return { fit, encode, encodeBatch, encodeQuery, serialize, getStats };
+}
+//# sourceMappingURL=bm25.js.map

package/dist/bm25.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":";;AAIA,gCA0GC;AA9GD,2CAAsC;AACtC,mCAAoC;AAGpC,SAAgB,UAAU,CAAC,OAAqB;IAC9C,MAAM,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,GAAG,CAAA;IAC7B,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC,IAAI,IAAI,CAAA;IAC5B,MAAM,OAAO,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAA;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,SAAS,CAAA;IACtC,MAAM,cAAc,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAA;IAE/C,MAAM,KAAK,GAAG,IAAI,kBAAU,EAAE,CAAA;IAC9B,wDAAwD;IACxD,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAA;IACpC,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,MAAM,GAAG,KAAK,CAAA;IAElB,SAAS,YAAY,CAAC,IAAY;QAChC,OAAO,IAAA,oBAAQ,EAAC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;IAC7F,CAAC;IAED,SAAS,GAAG,CAAC,SAAmB;QAC9B,CAAC,GAAG,SAAS,CAAC,MAAM,CAAA;QACpB,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,MAAM,aAAa,GAAe,EAAE,CAAA;QAEpC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAChC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC1B,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAA;YACzB,WAAW,IAAI,MAAM,CAAC,MAAM,CAAA;YAE5B,oCAAoC;YACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;YAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;gBAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;oBACZ,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,GAAG,IAAI,CAAA;IACf,CAAC;IAED,SAAS,WAAW,CAAC,MAAgB,EAAE,EAAU;QAC/C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;QAEzE,yBAAyB;QACzB,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAA;QACpC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACnC,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAmC,EAAE,CAAA;QAClD,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;YAC7D,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,CAAA;YAClF,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAChC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SAChC,CAAA;IACH,CAAC;IAED,SAAS,MAAM,CAAC,IAAY;QAC1B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QACjC,OAAO,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IAC3C,CAAC;IAED,SAAS,WAAW,CAAC,KAAe;QAClC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAClC,CAAC;IAED,SAAS,WAAW,CAAC,IAAY;QAC/B,sEAAsE;QACtE,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAC9E,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QACjC,OAAO,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IACnC,CAAC;IAED,SAAS,SAAS;QAChB,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,CAAC;YACD,KAAK;YACL,WAAW;YACX,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YAC1B,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE;YACxB,OAAO,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE;SAC7D,CAAC,CAAA;IACJ,CAAC;IAED,SAAS,QAAQ;QACf,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,CAAA;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAA;AACvE,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { createBM25 } from './bm25';
+export { createTFIDF } from './tfidf';
+export type { SparseVector, FitStats, BM25Options, TFIDFOptions, TokenizerFn, BM25Encoder, TFIDFEncoder, } from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,YAAY,EACV,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,YAAY,EACZ,WAAW,EACX,WAAW,EACX,YAAY,GACb,MAAM,SAAS,CAAA"}

package/dist/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTFIDF = exports.createBM25 = void 0;
+// sparse-encode - Generate BM25 and TF-IDF sparse vectors in JavaScript
+var bm25_1 = require("./bm25");
+Object.defineProperty(exports, "createBM25", { enumerable: true, get: function () { return bm25_1.createBM25; } });
+var tfidf_1 = require("./tfidf");
+Object.defineProperty(exports, "createTFIDF", { enumerable: true, get: function () { return tfidf_1.createTFIDF; } });
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,wEAAwE;AACxE,+BAAmC;AAA1B,kGAAA,UAAU,OAAA;AACnB,iCAAqC;AAA5B,oGAAA,WAAW,OAAA"}

package/dist/porter-stemmer.d.ts

@@ -0,0 +1,2 @@
+export declare function stem(word: string): string;
+//# sourceMappingURL=porter-stemmer.d.ts.map

package/dist/porter-stemmer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"porter-stemmer.d.ts","sourceRoot":"","sources":["../src/porter-stemmer.ts"],"names":[],"mappings":"AA+CA,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA+IzC"}

package/dist/porter-stemmer.js

@@ -0,0 +1,195 @@
+"use strict";
+// Porter Stemmer — classic algorithm (Porter 1980)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stem = stem;
+function hasCVC(word) {
+    // ends in consonant-vowel-consonant where final consonant is not w, x, y
+    if (word.length < 3)
+        return false;
+    const last = word[word.length - 1];
+    if ('wxy'.includes(last))
+        return false;
+    const vowels = new Set(['a', 'e', 'i', 'o', 'u']);
+    const c = word.length - 1;
+    return !vowels.has(word[c]) && vowels.has(word[c - 1]) && !vowels.has(word[c - 2]);
+}
+function containsVowel(word) {
+    for (const ch of word) {
+        if ('aeiou'.includes(ch))
+            return true;
+        if (ch === 'y' && word.indexOf(ch) > 0)
+            return true;
+    }
+    return false;
+}
+function measure(word) {
+    // count VC sequences
+    const vowels = new Set(['a', 'e', 'i', 'o', 'u']);
+    let m = 0;
+    let inVowel = false;
+    for (let i = 0; i < word.length; i++) {
+        const ch = word[i];
+        const isVowel = vowels.has(ch) || (ch === 'y' && i > 0);
+        if (isVowel) {
+            inVowel = true;
+        }
+        else {
+            if (inVowel) {
+                m++;
+                inVowel = false;
+            }
+        }
+    }
+    return m;
+}
+function endsDoubleConsonant(word) {
+    if (word.length < 2)
+        return false;
+    const last = word[word.length - 1];
+    const prev = word[word.length - 2];
+    return last === prev && !'aeiou'.includes(last);
+}
+function stem(word) {
+    if (word.length <= 2)
+        return word;
+    let w = word.toLowerCase();
+    // Step 1a
+    if (w.endsWith('sses')) {
+        w = w.slice(0, -2);
+    }
+    else if (w.endsWith('ies')) {
+        w = w.slice(0, -2); // ies → i
+    }
+    else if (w.endsWith('ss')) {
+        // keep
+    }
+    else if (w.endsWith('s')) {
+        w = w.slice(0, -1);
+    }
+    // Step 1b
+    let step1bTriggered = false;
+    if (w.endsWith('eed')) {
+        const stem1b = w.slice(0, -3);
+        if (measure(stem1b) > 0) {
+            w = w.slice(0, -1); // eed → ee
+        }
+    }
+    else if (w.endsWith('ed')) {
+        const stem1b = w.slice(0, -2);
+        if (containsVowel(stem1b)) {
+            w = stem1b;
+            step1bTriggered = true;
+        }
+    }
+    else if (w.endsWith('ing')) {
+        const stem1b = w.slice(0, -3);
+        if (containsVowel(stem1b)) {
+            w = stem1b;
+            step1bTriggered = true;
+        }
+    }
+    if (step1bTriggered) {
+        if (w.endsWith('at') || w.endsWith('bl') || w.endsWith('iz')) {
+            w = w + 'e';
+        }
+        else if (endsDoubleConsonant(w) && !w.endsWith('l') && !w.endsWith('s') && !w.endsWith('z')) {
+            w = w.slice(0, -1);
+        }
+        else if (measure(w) === 1 && hasCVC(w)) {
+            w = w + 'e';
+        }
+    }
+    // Step 1c
+    if (w.endsWith('y') && w.length > 2) {
+        const before = w.slice(0, -1);
+        if (containsVowel(before)) {
+            w = before + 'i';
+        }
+    }
+    // Step 2
+    const step2Map = [
+        ['ational', 'ate'],
+        ['tional', 'tion'],
+        ['enci', 'ence'],
+        ['anci', 'ance'],
+        ['izer', 'ize'],
+        ['abli', 'able'],
+        ['alli', 'al'],
+        ['entli', 'ent'],
+        ['eli', 'e'],
+        ['ousli', 'ous'],
+        ['ization', 'ize'],
+        ['ation', 'ate'],
+        ['ator', 'ate'],
+        ['alism', 'al'],
+        ['iveness', 'ive'],
+        ['fulness', 'ful'],
+        ['ousness', 'ous'],
+        ['aliti', 'al'],
+        ['iviti', 'ive'],
+        ['biliti', 'ble'],
+    ];
+    for (const [suffix, replacement] of step2Map) {
+        if (w.endsWith(suffix)) {
+            const base = w.slice(0, -suffix.length);
+            if (measure(base) > 0) {
+                w = base + replacement;
+            }
+            break;
+        }
+    }
+    // Step 3
+    const step3Map = [
+        ['icate', 'ic'],
+        ['ative', ''],
+        ['alize', 'al'],
+        ['iciti', 'ic'],
+        ['ical', 'ic'],
+        ['ful', ''],
+        ['ness', ''],
+    ];
+    for (const [suffix, replacement] of step3Map) {
+        if (w.endsWith(suffix)) {
+            const base = w.slice(0, -suffix.length);
+            if (measure(base) > 0) {
+                w = base + replacement;
+            }
+            break;
+        }
+    }
+    // Step 4
+    const step4Suffixes = [
+        'ement', 'ment', 'ance', 'ence', 'able', 'ible', 'ant', 'ent',
+        'ion', 'ism', 'ate', 'iti', 'ous', 'ive', 'ize', 'al', 'er', 'ic',
+    ];
+    for (const suffix of step4Suffixes) {
+        if (w.endsWith(suffix)) {
+            const base = w.slice(0, -suffix.length);
+            if (suffix === 'ion') {
+                if (measure(base) > 1 && (base.endsWith('s') || base.endsWith('t'))) {
+                    w = base;
+                }
+            }
+            else if (measure(base) > 1) {
+                w = base;
+            }
+            break;
+        }
+    }
+    // Step 5a
+    if (w.endsWith('e')) {
+        const base = w.slice(0, -1);
+        if (measure(base) > 1) {
+            w = base;
+        }
+        else if (measure(base) === 1 && !hasCVC(base)) {
+            w = base;
+        }
+    }
+    // Step 5b
+    if (w.endsWith('ll') && measure(w.slice(0, -1)) > 1) {
+        w = w.slice(0, -1);
+    }
+    return w;
+}
+//# sourceMappingURL=porter-stemmer.js.map

package/dist/porter-stemmer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"porter-stemmer.js","sourceRoot":"","sources":["../src/porter-stemmer.ts"],"names":[],"mappings":";AAAA,mDAAmD;;AA+CnD,oBA+IC;AA5LD,SAAS,MAAM,CAAC,IAAY;IAC1B,yEAAyE;IACzE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAClC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACjD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IACzB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;AACpF,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;QACtB,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,OAAO,IAAI,CAAA;QACrC,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAA;IACrD,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,qBAAqB;IACrB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;IACjD,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAClB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QACvD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,GAAG,IAAI,CAAA;QAChB,CAAC;aAAM,CAAC;YACN,IAAI,OAAO,EAAE,CAAC;gBACZ,CAAC,EAAE,CAAA;gBACH,OAAO,GAAG,KAAK,CAAA;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAClC,OAAO,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AACjD,CAAC;AAED,SAAgB,IAAI,CAAC,IAAY;IAC/B,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAEjC,IAAI,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAE1B,UAAU;IACV,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACvB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA,CAAC,UAAU;IAC/B,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,UAAU;IACV,IAAI,eAAe,GAAG,KAAK,CAAA;IAC3B,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA,CAAC,WAAW;QAChC,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,CAAC,GAAG,MAAM,CAAA;YACV,eAAe,GAAG,IAAI,CAAA;QACxB,CAAC;IACH,CAAC;SAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,CAAC,GAAG,MAAM,CAAA;YACV,eAAe,GAAG,IAAI,CAAA;QACxB,CAAC;IACH,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7D,CAAC,GAAG,CAAC,GAAG,GAAG,CAAA;QACb,CAAC;aAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9F,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACpB,CAAC;aAAM,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAA;QACb,CAAC;IACH,CAAC;IAED,UAAU;IACV,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC7B,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,CAAC,GAAG,MAAM,GAAG,GAAG,CAAA;QAClB,CAAC;IACH,CAAC;IAED,SAAS;IACT,MAAM,QAAQ,GAAuB;QACnC,CAAC,SAAS,EAAE,KAAK,CAAC;QAClB,CAAC,QAAQ,EAAE,MAAM,CAAC;QAClB,CAAC,MAAM,EAAE,MAAM,CAAC;QAChB,CAAC,MAAM,EAAE,MAAM,CAAC;QAChB,CAAC,MAAM,EAAE,KAAK,CAAC;QACf,CAAC,MAAM,EAAE,MAAM,CAAC;QAChB,CAAC,MAAM,EAAE,IAAI,CAAC;QACd,CAAC,OAAO,EAAE,KAAK,CAAC;QAChB,CAAC,KAAK,EAAE,GAAG,CAAC;QACZ,CAAC,OAAO,EAAE,KAAK,CAAC;QAChB,CAAC,SAAS,EAAE,KAAK,CAAC;QAClB,CAAC,OAAO,EAAE,KAAK,CAAC;QAChB,CAAC,MAAM,EAAE,KAAK,CAAC;QACf,CAAC,OAAO,EAAE,IAAI,CAAC;QACf,CAAC,SAAS,EAAE,KAAK,CAAC;QAClB,CAAC,SAAS,EAAE,KAAK,CAAC;QAClB,CAAC,SAAS,EAAE,KAAK,CAAC;QAClB,CAAC,OAAO,EAAE,IAAI,CAAC;QACf,CAAC,OAAO,EAAE,KAAK,CAAC;QAChB,CAAC,QAAQ,EAAE,KAAK,CAAC;KAClB,CAAA;IACD,KAAK,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YACvC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,CAAC,GAAG,IAAI,GAAG,WAAW,CAAA;YACxB,CAAC;YACD,MAAK;QACP,CAAC;IACH,CAAC;IAED,SAAS;IACT,MAAM,QAAQ,GAAuB;QACnC,CAAC,OAAO,EAAE,IAAI,CAAC;QACf,CAAC,OAAO,EAAE,EAAE,CAAC;QACb,CAAC,OAAO,EAAE,IAAI,CAAC;QACf,CAAC,OAAO,EAAE,IAAI,CAAC;QACf,CAAC,MAAM,EAAE,IAAI,CAAC;QACd,CAAC,KAAK,EAAE,EAAE,CAAC;QACX,CAAC,MAAM,EAAE,EAAE,CAAC;KACb,CAAA;IACD,KAAK,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YACvC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,CAAC,GAAG,IAAI,GAAG,WAAW,CAAA;YACxB,CAAC;YACD,MAAK;QACP,CAAC;IACH,CAAC;IAED,SAAS;IACT,MAAM,aAAa,GAAG;QACpB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;QAC7D,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;KAClE,CAAA;IACD,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YACvC,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACpE,CAAC,GAAG,IAAI,CAAA;gBACV,CAAC;YACH,CAAC;iBAAM,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,CAAC,GAAG,IAAI,CAAA;YACV,CAAC;YACD,MAAK;QACP,CAAC;IACH,CAAC;IAED,UAAU;IACV,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC3B,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,CAAC,GAAG,IAAI,CAAA;QACV,CAAC;aAAM,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,CAAC,GAAG,IAAI,CAAA;QACV,CAAC;IACH,CAAC;IAED,UAAU;IACV,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACpD,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,OAAO,CAAC,CAAA;AACV,CAAC"}

package/dist/tfidf.d.ts

@@ -0,0 +1,3 @@
+import type { TFIDFOptions, TFIDFEncoder } from './types';
+export declare function createTFIDF(options?: TFIDFOptions): TFIDFEncoder;
+//# sourceMappingURL=tfidf.d.ts.map

package/dist/tfidf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.d.ts","sourceRoot":"","sources":["../src/tfidf.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAA0B,MAAM,SAAS,CAAA;AAEjF,wBAAgB,WAAW,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,CAyGhE"}

package/dist/tfidf.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTFIDF = createTFIDF;
+const tokenizer_1 = require("./tokenizer");
+const vocab_1 = require("./vocab");
+function createTFIDF(options) {
+    const stemOpt = options?.stem ?? true;
+    const tokenizerFn = options?.tokenizer;
+    const extraStopwords = options?.stopwords ?? [];
+    const sublinearTf = options?.sublinearTf ?? false;
+    const vocab = new vocab_1.Vocabulary();
+    const df = new Map();
+    let N = 0;
+    let totalTokens = 0;
+    let avgdl = 0;
+    let fitted = false;
+    function tokenizeText(text) {
+        return (0, tokenizer_1.tokenize)(text, { stem: stemOpt, tokenizer: tokenizerFn, stopwords: extraStopwords });
+    }
+    function fit(documents) {
+        N = documents.length;
+        let totalLen = 0;
+        for (const doc of documents) {
+            const tokens = tokenizeText(doc);
+            totalLen += tokens.length;
+            totalTokens += tokens.length;
+            const seen = new Set();
+            for (const token of tokens) {
+                const id = vocab.getOrAdd(token);
+                if (!seen.has(id)) {
+                    seen.add(id);
+                    df.set(id, (df.get(id) ?? 0) + 1);
+                }
+            }
+        }
+        avgdl = N > 0 ? totalLen / N : 0;
+        fitted = true;
+    }
+    function encode(text) {
+        if (!fitted)
+            throw new Error('TFIDFEncoder must be fit() before encode()');
+        const tokens = tokenizeText(text);
+        const dl = tokens.length;
+        if (dl === 0)
+            return { indices: [], values: [] };
+        // Count raw term frequencies
+        const rawTf = new Map();
+        for (const token of tokens) {
+            const id = vocab.getId(token);
+            if (id !== undefined) {
+                rawTf.set(id, (rawTf.get(id) ?? 0) + 1);
+            }
+        }
+        const entries = [];
+        for (const [termId, count] of rawTf) {
+            const tf = sublinearTf ? 1 + Math.log(count) : count / dl;
+            const termDf = df.get(termId) ?? 0;
+            const idf = Math.log((N + 1) / (termDf + 1)) + 1;
+            const score = tf * idf;
+            if (score > 0) {
+                entries.push({ idx: termId, val: score });
+            }
+        }
+        // L2 normalize
+        const norm = Math.sqrt(entries.reduce((acc, e) => acc + e.val * e.val, 0));
+        if (norm > 0) {
+            for (const e of entries)
+                e.val /= norm;
+        }
+        entries.sort((a, c) => a.idx - c.idx);
+        return {
+            indices: entries.map(e => e.idx),
+            values: entries.map(e => e.val),
+        };
+    }
+    function encodeBatch(texts) {
+        return texts.map(t => encode(t));
+    }
+    function encodeQuery(text) {
+        // For queries, same as encode
+        return encode(text);
+    }
+    function serialize() {
+        return JSON.stringify({
+            N,
+            avgdl,
+            totalTokens,
+            df: Object.fromEntries(df),
+            vocab: vocab.serialize(),
+            options: { stem: stemOpt, sublinearTf, stopwords: extraStopwords },
+        });
+    }
+    function getStats() {
+        return { N, avgdl, vocabSize: vocab.size, totalTokens };
+    }
+    return { fit, encode, encodeBatch, encodeQuery, serialize, getStats };
+}
+//# sourceMappingURL=tfidf.js.map

package/dist/tfidf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.js","sourceRoot":"","sources":["../src/tfidf.ts"],"names":[],"mappings":";;AAIA,kCAyGC;AA7GD,2CAAsC;AACtC,mCAAoC;AAGpC,SAAgB,WAAW,CAAC,OAAsB;IAChD,MAAM,OAAO,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAA;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,SAAS,CAAA;IACtC,MAAM,cAAc,GAAG,OAAO,EAAE,SAAS,IAAI,EAAE,CAAA;IAC/C,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,KAAK,CAAA;IAEjD,MAAM,KAAK,GAAG,IAAI,kBAAU,EAAE,CAAA;IAC9B,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAA;IACpC,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,MAAM,GAAG,KAAK,CAAA;IAElB,SAAS,YAAY,CAAC,IAAY;QAChC,OAAO,IAAA,oBAAQ,EAAC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;IAC7F,CAAC;IAED,SAAS,GAAG,CAAC,SAAmB;QAC9B,CAAC,GAAG,SAAS,CAAC,MAAM,CAAA;QACpB,IAAI,QAAQ,GAAG,CAAC,CAAA;QAEhB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAChC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAA;YACzB,WAAW,IAAI,MAAM,CAAC,MAAM,CAAA;YAE5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;YAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;gBAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;oBACZ,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAChC,MAAM,GAAG,IAAI,CAAA;IACf,CAAC;IAED,SAAS,MAAM,CAAC,IAAY;QAC1B,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;QAE1E,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAA;QACxB,IAAI,EAAE,KAAK,CAAC;YAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAA;QAEhD,6BAA6B;QAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;QACvC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;gBACrB,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzC,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAmC,EAAE,CAAA;QAClD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YACpC,MAAM,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAA;YACzD,MAAM,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YAChD,MAAM,KAAK,GAAG,EAAE,GAAG,GAAG,CAAA;YACtB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;YAC3C,CAAC;QACH,CAAC;QAED,eAAe;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;QAC1E,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,KAAK,MAAM,CAAC,IAAI,OAAO;gBAAE,CAAC,CAAC,GAAG,IAAI,IAAI,CAAA;QACxC,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAChC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;SAChC,CAAA;IACH,CAAC;IAED,SAAS,WAAW,CAAC,KAAe;QAClC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAClC,CAAC;IAED,SAAS,WAAW,CAAC,IAAY;QAC/B,8BAA8B;QAC9B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAA;IACrB,CAAC;IAED,SAAS,SAAS;QAChB,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,CAAC;YACD,KAAK;YACL,WAAW;YACX,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YAC1B,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE;YACxB,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE;SACnE,CAAC,CAAA;IACJ,CAAC;IAED,SAAS,QAAQ;QACf,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,EAAE,CAAA;IACzD,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAA;AACvE,CAAC"}

package/dist/tokenizer.d.ts

@@ -0,0 +1,8 @@
+import type { TokenizerFn } from './types';
+export declare function defaultTokenizer(text: string): string[];
+export declare function tokenize(text: string, options?: {
+    stopwords?: string[];
+    stem?: boolean;
+    tokenizer?: TokenizerFn;
+}): string[];
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAS1C,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOvD;AAED,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;IACR,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,SAAS,CAAC,EAAE,WAAW,CAAA;CACxB,GACA,MAAM,EAAE,CAyBV"}

package/dist/tokenizer.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultTokenizer = defaultTokenizer;
+exports.tokenize = tokenize;
+const porter_stemmer_1 = require("./porter-stemmer");
+const DEFAULT_STOPWORDS = new Set([
+    'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+    'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+    'should', 'may', 'might', 'shall', 'can',
+]);
+function defaultTokenizer(text) {
+    return text
+        .toLowerCase()
+        .split(/[^\w]+/)
+        .filter(t => t.length > 0 && !/^\d+$/.test(t))
+        .filter(t => !DEFAULT_STOPWORDS.has(t))
+        .map(t => (0, porter_stemmer_1.stem)(t));
+}
+function tokenize(text, options) {
+    const { tokenizer, stopwords, stem: doStem = true } = options ?? {};
+    let tokens;
+    if (tokenizer) {
+        tokens = tokenizer(text);
+    }
+    else {
+        // Manual pipeline so we can honour stem=false
+        tokens = text
+            .toLowerCase()
+            .split(/[^\w]+/)
+            .filter(t => t.length > 0 && !/^\d+$/.test(t))
+            .filter(t => !DEFAULT_STOPWORDS.has(t));
+        if (doStem) {
+            tokens = tokens.map(t => (0, porter_stemmer_1.stem)(t));
+        }
+    }
+    if (stopwords && stopwords.length > 0) {
+        const extra = new Set(stopwords.map(s => s.toLowerCase()));
+        tokens = tokens.filter(t => !extra.has(t));
+    }
+    return tokens;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":";;AAUA,4CAOC;AAED,4BAgCC;AAnDD,qDAAuC;AAGvC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IACnE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACrE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK;CACzC,CAAC,CAAA;AAEF,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAC7C,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAA,qBAAI,EAAC,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED,SAAgB,QAAQ,CACtB,IAAY,EACZ,OAIC;IAED,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,GAAG,OAAO,IAAI,EAAE,CAAA;IAEnE,IAAI,MAAgB,CAAA;IACpB,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC;SAAM,CAAC;QACN,8CAA8C;QAC9C,MAAM,GAAG,IAAI;aACV,WAAW,EAAE;aACb,KAAK,CAAC,QAAQ,CAAC;aACf,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aAC7C,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAEzC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAA,qBAAI,EAAC,CAAC,CAAC,CAAC,CAAA;QACnC,CAAC;IACH,CAAC;IAED,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QAC1D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,41 @@
+export interface SparseVector {
+    indices: number[];
+    values: number[];
+}
+export interface FitStats {
+    N: number;
+    avgdl: number;
+    vocabSize: number;
+    totalTokens: number;
+}
+export type TokenizerFn = (text: string) => string[];
+export interface BM25Options {
+    k1?: number;
+    b?: number;
+    tokenizer?: TokenizerFn;
+    stopwords?: string[];
+    stem?: boolean;
+}
+export interface TFIDFOptions {
+    tokenizer?: TokenizerFn;
+    stopwords?: string[];
+    stem?: boolean;
+    sublinearTf?: boolean;
+}
+export interface BM25Encoder {
+    fit(documents: string[]): void;
+    encode(text: string): SparseVector;
+    encodeBatch(texts: string[]): SparseVector[];
+    encodeQuery(text: string): SparseVector;
+    serialize(): string;
+    getStats(): FitStats;
+}
+export interface TFIDFEncoder {
+    fit(documents: string[]): void;
+    encode(text: string): SparseVector;
+    encodeBatch(texts: string[]): SparseVector[];
+    encodeQuery(text: string): SparseVector;
+    serialize(): string;
+    getStats(): FitStats;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,CAAC,EAAE,MAAM,CAAA;IACT,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAA;AAEpD,MAAM,WAAW,WAAW;IAC1B,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,CAAC,CAAC,EAAE,MAAM,CAAA;IACV,SAAS,CAAC,EAAE,WAAW,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB,IAAI,CAAC,EAAE,OAAO,CAAA;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,CAAC,EAAE,WAAW,CAAA;IACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAA;IACpB,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,WAAW,CAAC,EAAE,OAAO,CAAA;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IAC9B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAA;IAClC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;IAC5C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAA;IACvC,SAAS,IAAI,MAAM,CAAA;IACnB,QAAQ,IAAI,QAAQ,CAAA;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IAC9B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAA;IAClC,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;IAC5C,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAAA;IACvC,SAAS,IAAI,MAAM,CAAA;IACnB,QAAQ,IAAI,QAAQ,CAAA;CACrB"}

package/dist/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/vocab.d.ts

@@ -0,0 +1,11 @@
+export declare class Vocabulary {
+    private termToId;
+    private idToTerm;
+    getOrAdd(term: string): number;
+    getId(term: string): number | undefined;
+    get size(): number;
+    terms(): string[];
+    serialize(): object;
+    static deserialize(data: object): Vocabulary;
+}
+//# sourceMappingURL=vocab.d.ts.map

package/dist/vocab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab.d.ts","sourceRoot":"","sources":["../src/vocab.ts"],"names":[],"mappings":"AAAA,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAA4B;IAC5C,OAAO,CAAC,QAAQ,CAAe;IAE/B,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAU9B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIvC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,KAAK,IAAI,MAAM,EAAE;IAIjB,SAAS,IAAI,MAAM;IAInB,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU;CAQ7C"}

package/dist/vocab.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Vocabulary = void 0;
+class Vocabulary {
+    termToId = new Map();
+    idToTerm = [];
+    getOrAdd(term) {
+        let id = this.termToId.get(term);
+        if (id === undefined) {
+            id = this.idToTerm.length;
+            this.termToId.set(term, id);
+            this.idToTerm.push(term);
+        }
+        return id;
+    }
+    getId(term) {
+        return this.termToId.get(term);
+    }
+    get size() {
+        return this.idToTerm.length;
+    }
+    terms() {
+        return [...this.idToTerm];
+    }
+    serialize() {
+        return { terms: this.idToTerm };
+    }
+    static deserialize(data) {
+        const v = new Vocabulary();
+        const d = data;
+        for (const term of d.terms) {
+            v.getOrAdd(term);
+        }
+        return v;
+    }
+}
+exports.Vocabulary = Vocabulary;
+//# sourceMappingURL=vocab.js.map

package/dist/vocab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab.js","sourceRoot":"","sources":["../src/vocab.ts"],"names":[],"mappings":";;;AAAA,MAAa,UAAU;IACb,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;IACpC,QAAQ,GAAa,EAAE,CAAA;IAE/B,QAAQ,CAAC,IAAY;QACnB,IAAI,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACrB,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAA;YACzB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;QACD,OAAO,EAAE,CAAA;IACX,CAAC;IAED,KAAK,CAAC,IAAY;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAA;IAC7B,CAAC;IAED,KAAK;QACH,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC3B,CAAC;IAED,SAAS;QACP,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,MAAM,CAAC,WAAW,CAAC,IAAY;QAC7B,MAAM,CAAC,GAAG,IAAI,UAAU,EAAE,CAAA;QAC1B,MAAM,CAAC,GAAG,IAA2B,CAAA;QACrC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YAC3B,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QACD,OAAO,CAAC,CAAA;IACV,CAAC;CACF;AAtCD,gCAsCC"}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "sparse-encode",
+  "version": "0.1.0",
+  "description": "Generate BM25 and TF-IDF sparse vectors in JavaScript",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "lint": "eslint src/",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@typescript-eslint/eslint-plugin": "^8.57.1",
+    "@typescript-eslint/parser": "^8.57.1",
+    "eslint": "^10.1.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.0"
+  }
+}