vectra 0.12.2 → 0.14.0

package/lib/TransformersEmbeddings.js

@@ -0,0 +1,176 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformersEmbeddings = void 0;
+const TransformersTokenizer_1 = require("./TransformersTokenizer");
+const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
+/**
+ * An embeddings model using Transformers.js for local, offline inference.
+ * @remarks
+ * Requires @huggingface/transformers as a peer dependency.
+ * Use the static `create()` method to instantiate.
+ *
+ * @example
+ * ```typescript
+ * const embeddings = await TransformersEmbeddings.create({
+ *     model: 'Xenova/all-MiniLM-L6-v2'
+ * });
+ *
+ * const index = new LocalDocumentIndex({
+ *     folderPath: 'my-index',
+ *     embeddings: embeddings,
+ *     tokenizer: embeddings.getTokenizer()
+ * });
+ * ```
+ */
+class TransformersEmbeddings {
+    /**
+     * Private constructor - use TransformersEmbeddings.create() instead.
+     */
+    constructor(extractor, tokenizer, options) {
+        this._extractor = extractor;
+        this._tokenizer = tokenizer;
+        this._options = options;
+        this.maxTokens = options.maxTokens;
+    }
+    /**
+     * Creates a new TransformersEmbeddings instance.
+     * @param options Configuration options.
+     * @returns Promise resolving to initialized TransformersEmbeddings instance.
+     * @throws Error if @huggingface/transformers is not installed.
+     */
+    static create(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c, _d, _e, _f;
+            // Dynamically import to allow optional dependency
+            let transformers;
+            try {
+                transformers = yield Promise.resolve().then(() => __importStar(require('@huggingface/transformers')));
+            }
+            catch (e) {
+                throw new Error('TransformersEmbeddings requires @huggingface/transformers. ' +
+                    'Install it with: npm install @huggingface/transformers');
+            }
+            const { pipeline } = transformers;
+            // Apply defaults
+            const opts = {
+                model: (_a = options === null || options === void 0 ? void 0 : options.model) !== null && _a !== void 0 ? _a : DEFAULT_MODEL,
+                maxTokens: (_b = options === null || options === void 0 ? void 0 : options.maxTokens) !== null && _b !== void 0 ? _b : 512,
+                device: (_c = options === null || options === void 0 ? void 0 : options.device) !== null && _c !== void 0 ? _c : 'auto',
+                dtype: (_d = options === null || options === void 0 ? void 0 : options.dtype) !== null && _d !== void 0 ? _d : 'fp32',
+                normalize: (_e = options === null || options === void 0 ? void 0 : options.normalize) !== null && _e !== void 0 ? _e : true,
+                pooling: (_f = options === null || options === void 0 ? void 0 : options.pooling) !== null && _f !== void 0 ? _f : 'mean',
+                progressCallback: options === null || options === void 0 ? void 0 : options.progressCallback
+            };
+            // Build pipeline options
+            const pipelineOptions = {
+                device: opts.device,
+                dtype: opts.dtype
+            };
+            if (opts.progressCallback) {
+                pipelineOptions.progress_callback = opts.progressCallback;
+            }
+            // Load the feature extraction pipeline
+            const extractor = yield pipeline('feature-extraction', opts.model, pipelineOptions);
+            // Load the tokenizer separately for use with TextSplitter
+            const tokenizer = extractor.tokenizer;
+            return new TransformersEmbeddings(extractor, tokenizer, opts);
+        });
+    }
+    /**
+     * Returns a tokenizer that uses the same tokenization as this embedding model.
+     * @remarks
+     * Use this tokenizer with LocalDocumentIndex to ensure text chunking
+     * aligns with the embedding model's token boundaries.
+     * @returns TransformersTokenizer instance.
+     */
+    getTokenizer() {
+        return new TransformersTokenizer_1.TransformersTokenizer(this._tokenizer);
+    }
+    /**
+     * Creates embeddings for the given inputs.
+     * @param inputs Text inputs to create embeddings for.
+     * @returns EmbeddingsResponse with status and generated embeddings.
+     */
+    createEmbeddings(inputs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const inputArray = Array.isArray(inputs) ? inputs : [inputs];
+                // Process all inputs in a single batch
+                const output = yield this._extractor(inputArray, {
+                    pooling: this._options.pooling,
+                    normalize: this._options.normalize
+                });
+                const [batchSize, embeddingDim] = output.dims;
+                const data = output.data;
+                // Slice the flat array into individual embeddings
+                const embeddings = [];
+                for (let i = 0; i < batchSize; i++) {
+                    const start = i * embeddingDim;
+                    const end = start + embeddingDim;
+                    embeddings.push(Array.from(data.slice(start, end)));
+                }
+                return {
+                    status: 'success',
+                    output: embeddings,
+                    model: this._options.model
+                };
+            }
+            catch (error) {
+                return {
+                    status: 'error',
+                    message: `Error generating embeddings: ${error.message}`
+                };
+            }
+        });
+    }
+    /**
+     * Returns the model name being used.
+     */
+    get model() {
+        return this._options.model;
+    }
+}
+exports.TransformersEmbeddings = TransformersEmbeddings;
+//# sourceMappingURL=TransformersEmbeddings.js.map

package/lib/TransformersEmbeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersEmbeddings.js","sourceRoot":"","sources":["../src/TransformersEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,mEAAgE;AAIhE,MAAM,aAAa,GAAG,yBAAyB,CAAC;AA2EhD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAa,sBAAsB;IAO/B;;OAEG;IACH,YACI,SAAoC,EACpC,SAA8B,EAC9B,OAAoI;QAEpI,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAO,MAAM,CAAC,OAAuC;;;YAC9D,kDAAkD;YAClD,IAAI,YAAiC,CAAC;YAEtC,IAAI,CAAC;gBACD,YAAY,GAAG,wDAAa,2BAA2B,GAAC,CAAC;YAC7D,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CACX,6DAA6D;oBAC7D,wDAAwD,CAC3D,CAAC;YACN,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,GAAG,YAAY,CAAC;YAElC,iBAAiB;YACjB,MAAM,IAAI,GAAG;gBACT,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,aAAa;gBACtC,SAAS,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,mCAAI,GAAG;gBACpC,MAAM,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,mCAAI,MAAM;gBACjC,KAAK,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,MAAM;gBAC/B,SAAS,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,SAAS,mCAAI,IAAI;gBACrC,OAAO,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,OAAO,mCAAI,MAAM;gBACnC,gBAAgB,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,gBAAgB;aAC9C,CAAC;YAEF,yBAAyB;YACzB,MAAM,eAAe,GAAQ;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;aACpB,CAAC;YAEF,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,eAAe,CAAC,iBAAiB,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAC9D,CAAC;YAED,uCAAuC;YACvC,MAAM,SAAS,GAAG,MAAM,QAAQ,CAC5B,oBAAoB,EACpB,IAAI,CAAC,KAAK,EACV,eAAe,CAClB,CAAC;YAEF,0DAA0D;YAC1D,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YAEtC,OAAO,IAAI,sBAAsB,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAClE,CAAC;KAAA;IAED;;;;;;OAMG;IACI,YAAY;QACf,OAAO,IAAI,6CAAqB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACU,gBAAgB,CAAC,MAAyB;;YACnD,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBAE7D,uCAAuC;gBACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE;oBAC7C,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;oBAC9B,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;iBACrC,CAAC,CAAC;gBAEH,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC;gBAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAoB,CAAC;gBAEzC,kDAAkD;gBAClD,MAAM,UAAU,GAAe,EAAE,CAAC;gBAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;oBACjC,MAAM,KAAK,GAAG,CAAC,GAAG,YAAY,CAAC;oBAC/B,MAAM,GAAG,GAAG,KAAK,GAAG,YAAY,CAAC;oBACjC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;gBACxD,CAAC;gBAED,OAAO;oBACH,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,UAAU;oBAClB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;iBAC7B,CAAC;YACN,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACtB,OAAO;oBACH,MAAM,EAAE,OAAO;oBACf,OAAO,EAAE,gCAAiC,KAAe,CAAC,OAAO,EAAE;iBACtE,CAAC;YACN,CAAC;QACL,CAAC;KAAA;IAED;;OAEG;IACH,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;CACJ;AApID,wDAoIC"}

package/lib/TransformersEmbeddings.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=TransformersEmbeddings.spec.d.ts.map

package/lib/TransformersEmbeddings.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersEmbeddings.spec.d.ts","sourceRoot":"","sources":["../src/TransformersEmbeddings.spec.ts"],"names":[],"mappings":""}

package/lib/TransformersEmbeddings.spec.js

@@ -0,0 +1,198 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = require("node:assert");
+const mocha_1 = require("mocha");
+const sinon_1 = __importDefault(require("sinon"));
+const transformersModule = __importStar(require("@huggingface/transformers"));
+(0, mocha_1.describe)('TransformersEmbeddings', () => {
+    let TransformersEmbeddings;
+    let mockExtractor;
+    let mockTokenizer;
+    let sandbox;
+    let pipelineStub;
+    (0, mocha_1.beforeEach)(() => __awaiter(void 0, void 0, void 0, function* () {
+        sandbox = sinon_1.default.createSandbox();
+        // Create mock tokenizer
+        mockTokenizer = {
+            __call__: sandbox.stub().returns({
+                input_ids: { data: BigInt64Array.from([BigInt(1), BigInt(2), BigInt(3)]) }
+            }),
+            decode: sandbox.stub().returns('decoded text')
+        };
+        // Make it callable
+        const callableTokenizer = Object.assign((...args) => mockTokenizer.__call__(...args), mockTokenizer);
+        // Create mock extractor (feature extraction pipeline)
+        mockExtractor = sandbox.stub().callsFake((inputs) => __awaiter(void 0, void 0, void 0, function* () {
+            const inputArray = Array.isArray(inputs) ? inputs : [inputs];
+            const batchSize = inputArray.length;
+            const embeddingDim = 4;
+            const data = new Float32Array(batchSize * embeddingDim);
+            for (let i = 0; i < batchSize; i++) {
+                data[i * embeddingDim] = 0.1;
+                data[i * embeddingDim + 1] = 0.2;
+                data[i * embeddingDim + 2] = 0.3;
+                data[i * embeddingDim + 3] = 0.4;
+            }
+            return {
+                data: data,
+                dims: [batchSize, embeddingDim]
+            };
+        }));
+        // Attach tokenizer to the mock extractor so pipeline result has .tokenizer
+        mockExtractor.tokenizer = callableTokenizer;
+        // Stub the pipeline function from @huggingface/transformers
+        pipelineStub = sandbox.stub(transformersModule, 'pipeline').resolves(mockExtractor);
+        // Import TransformersEmbeddings fresh (uses the stubbed pipeline via dynamic import)
+        const mod = yield Promise.resolve().then(() => __importStar(require('./TransformersEmbeddings')));
+        TransformersEmbeddings = mod.TransformersEmbeddings;
+    }));
+    (0, mocha_1.afterEach)(() => {
+        sandbox.restore();
+    });
+    (0, mocha_1.describe)('create()', () => {
+        (0, mocha_1.it)('creates instance with default options', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            node_assert_1.strict.equal(embeddings.maxTokens, 512, 'default maxTokens should be 512');
+            node_assert_1.strict.equal(embeddings.model, 'Xenova/all-MiniLM-L6-v2', 'default model should be all-MiniLM-L6-v2');
+            // Verify pipeline was called with correct arguments
+            node_assert_1.strict.ok(pipelineStub.calledOnce, 'pipeline should be called once');
+            node_assert_1.strict.equal(pipelineStub.firstCall.args[0], 'feature-extraction');
+            node_assert_1.strict.equal(pipelineStub.firstCall.args[1], 'Xenova/all-MiniLM-L6-v2');
+        }));
+        (0, mocha_1.it)('creates instance with custom options', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create({
+                model: 'Xenova/bge-small-en-v1.5',
+                maxTokens: 256,
+                device: 'cpu',
+                normalize: false,
+                pooling: 'cls'
+            });
+            node_assert_1.strict.equal(embeddings.maxTokens, 256);
+            node_assert_1.strict.equal(embeddings.model, 'Xenova/bge-small-en-v1.5');
+        }));
+        (0, mocha_1.it)('implements EmbeddingsModel interface', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            node_assert_1.strict.equal(typeof embeddings.maxTokens, 'number');
+            node_assert_1.strict.equal(typeof embeddings.createEmbeddings, 'function');
+        }));
+    });
+    (0, mocha_1.describe)('createEmbeddings()', () => {
+        (0, mocha_1.it)('generates embeddings for single string', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            const result = yield embeddings.createEmbeddings('hello world');
+            node_assert_1.strict.equal(result.status, 'success');
+            node_assert_1.strict.ok(result.output, 'output should be defined');
+            node_assert_1.strict.equal(result.output.length, 1, 'should have one embedding');
+            node_assert_1.strict.equal(result.output[0].length, 4, 'embedding should have 4 dimensions');
+            const expected = [0.1, 0.2, 0.3, 0.4];
+            result.output[0].forEach((val, i) => {
+                node_assert_1.strict.ok(Math.abs(val - expected[i]) < 0.001, `value ${val} should be close to ${expected[i]}`);
+            });
+            node_assert_1.strict.equal(result.model, 'Xenova/all-MiniLM-L6-v2');
+        }));
+        (0, mocha_1.it)('generates embeddings for string array', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            const result = yield embeddings.createEmbeddings(['hello', 'world']);
+            node_assert_1.strict.equal(result.status, 'success');
+            node_assert_1.strict.ok(result.output, 'output should be defined');
+            node_assert_1.strict.equal(result.output.length, 2, 'should have two embeddings');
+            node_assert_1.strict.equal(mockExtractor.callCount, 1);
+            node_assert_1.strict.deepEqual(mockExtractor.firstCall.args[0], ['hello', 'world']);
+        }));
+        (0, mocha_1.it)('passes pooling and normalize options to extractor', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create({
+                pooling: 'cls',
+                normalize: false
+            });
+            yield embeddings.createEmbeddings('test');
+            node_assert_1.strict.ok(mockExtractor.calledOnce);
+            const options = mockExtractor.firstCall.args[1];
+            node_assert_1.strict.equal(options.pooling, 'cls');
+            node_assert_1.strict.equal(options.normalize, false);
+        }));
+        (0, mocha_1.it)('returns error status on failure', () => __awaiter(void 0, void 0, void 0, function* () {
+            var _a;
+            mockExtractor.rejects(new Error('Model inference failed'));
+            const embeddings = yield TransformersEmbeddings.create();
+            const result = yield embeddings.createEmbeddings('test');
+            node_assert_1.strict.equal(result.status, 'error');
+            node_assert_1.strict.ok((_a = result.message) === null || _a === void 0 ? void 0 : _a.includes('Model inference failed'));
+        }));
+        (0, mocha_1.it)('handles empty string input', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            const result = yield embeddings.createEmbeddings('');
+            node_assert_1.strict.equal(result.status, 'success');
+            node_assert_1.strict.ok(result.output);
+            node_assert_1.strict.equal(result.output.length, 1);
+        }));
+        (0, mocha_1.it)('handles empty array input', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            const result = yield embeddings.createEmbeddings([]);
+            node_assert_1.strict.equal(result.status, 'success');
+            node_assert_1.strict.ok(result.output);
+            node_assert_1.strict.equal(result.output.length, 0);
+        }));
+    });
+    (0, mocha_1.describe)('getTokenizer()', () => {
+        (0, mocha_1.it)('returns a TransformersTokenizer instance', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            const tokenizer = embeddings.getTokenizer();
+            node_assert_1.strict.ok(tokenizer, 'tokenizer should be defined');
+            node_assert_1.strict.equal(typeof tokenizer.encode, 'function');
+            node_assert_1.strict.equal(typeof tokenizer.decode, 'function');
+        }));
+        (0, mocha_1.it)('returns consistent tokenizer across calls', () => __awaiter(void 0, void 0, void 0, function* () {
+            const embeddings = yield TransformersEmbeddings.create();
+            const tokenizer1 = embeddings.getTokenizer();
+            const tokenizer2 = embeddings.getTokenizer();
+            node_assert_1.strict.ok(tokenizer1);
+            node_assert_1.strict.ok(tokenizer2);
+        }));
+    });
+});
+//# sourceMappingURL=TransformersEmbeddings.spec.js.map

package/lib/TransformersEmbeddings.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersEmbeddings.spec.js","sourceRoot":"","sources":["../src/TransformersEmbeddings.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA+C;AAC/C,iCAA4D;AAC5D,kDAA0B;AAE1B,8EAAgE;AAEhE,IAAA,gBAAQ,EAAC,wBAAwB,EAAE,GAAG,EAAE;IACpC,IAAI,sBAA2B,CAAC;IAChC,IAAI,aAA8B,CAAC;IACnC,IAAI,aAAkB,CAAC;IACvB,IAAI,OAA2B,CAAC;IAChC,IAAI,YAA6B,CAAC;IAElC,IAAA,kBAAU,EAAC,GAAS,EAAE;QAClB,OAAO,GAAG,eAAK,CAAC,aAAa,EAAE,CAAC;QAEhC,wBAAwB;QACxB,aAAa,GAAG;YACZ,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;aAC7E,CAAC;YACF,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC;SACjD,CAAC;QACF,mBAAmB;QACnB,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACnC,CAAC,GAAG,IAAW,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,EACnD,aAAa,CAChB,CAAC;QAEF,sDAAsD;QACtD,aAAa,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,CAAO,MAAyB,EAAE,EAAE;YACzE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC7D,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;YACpC,MAAM,YAAY,GAAG,CAAC,CAAC;YAEvB,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,SAAS,GAAG,YAAY,CAAC,CAAC;YACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC;gBAC7B,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;gBACjC,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;gBACjC,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;YACrC,CAAC;YAED,OAAO;gBACH,IAAI,EAAE,IAAI;gBACV,IAAI,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC;aAClC,CAAC;QACN,CAAC,CAAA,CAAC,CAAC;QAEH,2EAA2E;QAC1E,aAAqB,CAAC,SAAS,GAAG,iBAAiB,CAAC;QAErD,4DAA4D;QAC5D,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,UAAiB,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAE3F,qFAAqF;QACrF,MAAM,GAAG,GAAG,wDAAa,0BAA0B,GAAC,CAAC;QACrD,sBAAsB,GAAG,GAAG,CAAC,sBAAsB,CAAC;IACxD,CAAC,CAAA,CAAC,CAAC;IAEH,IAAA,iBAAS,EAAC,GAAG,EAAE;QACX,OAAO,CAAC,OAAO,EAAE,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,UAAU,EAAE,GAAG,EAAE;QACtB,IAAA,UAAE,EAAC,uCAAuC,EAAE,GAAS,EAAE;YACnD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YAEzD,oBAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,EAAE,iCAAiC,CAAC,CAAC;YAC3E,oBAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,yBAAyB,EAAE,0CAA0C,CAAC,CAAC;YAEtG,oDAAoD;YACpD,oBAAM,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAC;YACrE,oBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC;YACnE,oBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAC5E,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,sCAAsC,EAAE,GAAS,EAAE;YAClD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,0BAA0B;gBACjC,SAAS,EAAE,GAAG;gBACd,MAAM,EAAE,KAAK;gBACb,SAAS,EAAE,KAAK;gBAChB,OAAO,EAAE,KAAK;aACjB,CAAC,CAAC;YAEH,oBAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACxC,oBAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;QAC/D,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,sCAAsC,EAAE,GAAS,EAAE;YAClD,MAAM,UAAU,GAAoB,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YAE1E,oBAAM,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACpD,oBAAM,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;QACjE,CAAC,CAAA,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,oBAAoB,EAAE,GAAG,EAAE;QAChC,IAAA,UAAE,EAAC,wCAAwC,EAAE,GAAS,EAAE;YACpD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;YAEhE,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACvC,oBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAC;YACrD,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAO,CAAC,MAAM,EAAE,CAAC,EAAE,2BAA2B,CAAC,CAAC;YACpE,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,oCAAoC,CAAC,CAAC;YAChF,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACtC,MAAM,CAAC,MAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAW,EAAE,CAAS,EAAE,EAAE;gBACjD,oBAAM,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,SAAS,GAAG,uBAAuB,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrG,CAAC,CAAC,CAAC;YACH,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAC1D,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,uCAAuC,EAAE,GAAS,EAAE;YACnD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAErE,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACvC,oBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAC;YACrD,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAO,CAAC,MAAM,EAAE,CAAC,EAAE,4BAA4B,CAAC,CAAC;YAErE,oBAAM,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YACzC,oBAAM,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,mDAAmD,EAAE,GAAS,EAAE;YAC/D,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,CAAC;gBACnD,OAAO,EAAE,KAAK;gBACd,SAAS,EAAE,KAAK;aACnB,CAAC,CAAC;YACH,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAE1C,oBAAM,CAAC,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YACpC,MAAM,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChD,oBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACrC,oBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC3C,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,iCAAiC,EAAE,GAAS,EAAE;;YAC7C,aAAa,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC;YAE3D,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAEzD,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACrC,oBAAM,CAAC,EAAE,CAAC,MAAA,MAAM,CAAC,OAAO,0CAAE,QAAQ,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAClE,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,4BAA4B,EAAE,GAAS,EAAE;YACxC,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YAErD,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACvC,oBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,2BAA2B,EAAE,GAAS,EAAE;YACvC,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC;YAErD,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACvC,oBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzB,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAA,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,IAAA,gBAAQ,EAAC,gBAAgB,EAAE,GAAG,EAAE;QAC5B,IAAA,UAAE,EAAC,0CAA0C,EAAE,GAAS,EAAE;YACtD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,EAAE,CAAC;YAE5C,oBAAM,CAAC,EAAE,CAAC,SAAS,EAAE,6BAA6B,CAAC,CAAC;YACpD,oBAAM,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;YAClD,oBAAM,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACtD,CAAC,CAAA,CAAC,CAAC;QAEH,IAAA,UAAE,EAAC,2CAA2C,EAAE,GAAS,EAAE;YACvD,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,UAAU,GAAG,UAAU,CAAC,YAAY,EAAE,CAAC;YAC7C,MAAM,UAAU,GAAG,UAAU,CAAC,YAAY,EAAE,CAAC;YAE7C,oBAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;YACtB,oBAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QAC1B,CAAC,CAAA,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/TransformersTokenizer.d.ts

@@ -0,0 +1,33 @@
+import { PreTrainedTokenizer } from "@huggingface/transformers";
+import { Tokenizer } from "./types";
+/**
+ * A tokenizer wrapper for Transformers.js models.
+ * @remarks
+ * This tokenizer uses the same tokenizer as the embedding model,
+ * ensuring consistency between text splitting and embedding generation.
+ *
+ * Obtain an instance via TransformersEmbeddings.getTokenizer().
+ */
+export declare class TransformersTokenizer implements Tokenizer {
+    private readonly _tokenizer;
+    /**
+     * Creates a new TransformersTokenizer.
+     * @param tokenizer The underlying Transformers.js tokenizer.
+     * @remarks
+     * Typically created via TransformersEmbeddings.getTokenizer().
+     */
+    constructor(tokenizer: PreTrainedTokenizer);
+    /**
+     * Encodes text into token IDs.
+     * @param text The text to encode.
+     * @returns Array of token IDs.
+     */
+    encode(text: string): number[];
+    /**
+     * Decodes token IDs back into text.
+     * @param tokens Array of token IDs.
+     * @returns Decoded text string.
+     */
+    decode(tokens: number[]): string;
+}
+//# sourceMappingURL=TransformersTokenizer.d.ts.map

package/lib/TransformersTokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersTokenizer.d.ts","sourceRoot":"","sources":["../src/TransformersTokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEpC;;;;;;;GAOG;AACH,qBAAa,qBAAsB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IAEjD;;;;;OAKG;gBACgB,SAAS,EAAE,mBAAmB;IAIjD;;;;OAIG;IACI,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAOrC;;;;OAIG;IACI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM;CAG1C"}

package/lib/TransformersTokenizer.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformersTokenizer = void 0;
+/**
+ * A tokenizer wrapper for Transformers.js models.
+ * @remarks
+ * This tokenizer uses the same tokenizer as the embedding model,
+ * ensuring consistency between text splitting and embedding generation.
+ *
+ * Obtain an instance via TransformersEmbeddings.getTokenizer().
+ */
+class TransformersTokenizer {
+    /**
+     * Creates a new TransformersTokenizer.
+     * @param tokenizer The underlying Transformers.js tokenizer.
+     * @remarks
+     * Typically created via TransformersEmbeddings.getTokenizer().
+     */
+    constructor(tokenizer) {
+        this._tokenizer = tokenizer;
+    }
+    /**
+     * Encodes text into token IDs.
+     * @param text The text to encode.
+     * @returns Array of token IDs.
+     */
+    encode(text) {
+        var _a, _b, _c;
+        const encoded = this._tokenizer(text);
+        // Transformers.js returns an object with input_ids as BigInt64Array or similar
+        const inputIds = (_c = (_b = (_a = encoded.input_ids) === null || _a === void 0 ? void 0 : _a.data) !== null && _b !== void 0 ? _b : encoded.input_ids) !== null && _c !== void 0 ? _c : encoded;
+        return Array.from(inputIds).map((id) => Number(id));
+    }
+    /**
+     * Decodes token IDs back into text.
+     * @param tokens Array of token IDs.
+     * @returns Decoded text string.
+     */
+    decode(tokens) {
+        return this._tokenizer.decode(tokens, { skip_special_tokens: true });
+    }
+}
+exports.TransformersTokenizer = TransformersTokenizer;
+//# sourceMappingURL=TransformersTokenizer.js.map

package/lib/TransformersTokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersTokenizer.js","sourceRoot":"","sources":["../src/TransformersTokenizer.ts"],"names":[],"mappings":";;;AAGA;;;;;;;GAOG;AACH,MAAa,qBAAqB;IAG9B;;;;;OAKG;IACH,YAAmB,SAA8B;QAC7C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAChC,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,IAAY;;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,MAAA,MAAA,MAAA,OAAO,CAAC,SAAS,0CAAE,IAAI,mCAAI,OAAO,CAAC,SAAS,mCAAI,OAAO,CAAC;QACzE,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED;;;;OAIG;IACI,MAAM,CAAC,MAAgB;QAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACzE,CAAC;CACJ;AAjCD,sDAiCC"}

package/lib/TransformersTokenizer.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=TransformersTokenizer.spec.d.ts.map

package/lib/TransformersTokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersTokenizer.spec.d.ts","sourceRoot":"","sources":["../src/TransformersTokenizer.spec.ts"],"names":[],"mappings":""}

package/lib/TransformersTokenizer.spec.js

@@ -0,0 +1,112 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = require("node:assert");
+const mocha_1 = require("mocha");
+const TransformersTokenizer_1 = require("./TransformersTokenizer");
+(0, mocha_1.describe)('TransformersTokenizer', () => {
+    // Create a mock tokenizer that mimics Transformers.js behavior
+    function createMockTokenizer() {
+        const vocab = new Map([
+            ['hello', 101],
+            ['world', 102],
+            ['test', 103],
+            ['[CLS]', 1],
+            ['[SEP]', 2]
+        ]);
+        const reverseVocab = new Map();
+        vocab.forEach((v, k) => reverseVocab.set(v, k));
+        return {
+            // Mimics the callable tokenizer behavior
+            __call__: (text) => {
+                const words = text.toLowerCase().split(/\s+/).filter(w => w);
+                const ids = words.map(w => { var _a; return (_a = vocab.get(w)) !== null && _a !== void 0 ? _a : 100; });
+                return {
+                    input_ids: {
+                        data: BigInt64Array.from(ids.map(id => BigInt(id)))
+                    }
+                };
+            },
+            decode: (tokens, options) => {
+                const words = tokens
+                    .filter(t => !(options === null || options === void 0 ? void 0 : options.skip_special_tokens) || (t !== 1 && t !== 2))
+                    .map(t => { var _a; return (_a = reverseVocab.get(t)) !== null && _a !== void 0 ? _a : '[UNK]'; });
+                return words.join(' ');
+            }
+        };
+    }
+    (0, mocha_1.it)('encodes text to token array using callable tokenizer', () => {
+        const mockTokenizer = createMockTokenizer();
+        // Make it callable
+        const callableTokenizer = Object.assign((text) => mockTokenizer.__call__(text), { decode: mockTokenizer.decode });
+        const tokenizer = new TransformersTokenizer_1.TransformersTokenizer(callableTokenizer);
+        const tokens = tokenizer.encode('hello world');
+        node_assert_1.strict.ok(Array.isArray(tokens), 'encode should return an array');
+        node_assert_1.strict.equal(tokens.length, 2, 'should have 2 tokens');
+        node_assert_1.strict.deepEqual(tokens, [101, 102], 'tokens should match expected values');
+    });
+    (0, mocha_1.it)('handles BigInt64Array conversion correctly', () => {
+        const mockTokenizer = {
+            __call__: () => ({
+                input_ids: {
+                    data: BigInt64Array.from([BigInt(1), BigInt(2), BigInt(3)])
+                }
+            }),
+            decode: () => 'decoded'
+        };
+        const callableTokenizer = Object.assign(() => mockTokenizer.__call__(), { decode: mockTokenizer.decode });
+        const tokenizer = new TransformersTokenizer_1.TransformersTokenizer(callableTokenizer);
+        const tokens = tokenizer.encode('any text');
+        node_assert_1.strict.deepEqual(tokens, [1, 2, 3], 'should convert BigInt to number');
+        tokens.forEach(t => {
+            node_assert_1.strict.equal(typeof t, 'number', 'each token should be a number');
+        });
+    });
+    (0, mocha_1.it)('decodes tokens back to text', () => {
+        const mockTokenizer = {
+            __call__: () => ({ input_ids: { data: BigInt64Array.from([]) } }),
+            decode: (tokens, opts) => {
+                if (opts === null || opts === void 0 ? void 0 : opts.skip_special_tokens) {
+                    return 'hello world';
+                }
+                return '[CLS] hello world [SEP]';
+            }
+        };
+        const callableTokenizer = Object.assign(() => mockTokenizer.__call__(), { decode: mockTokenizer.decode });
+        const tokenizer = new TransformersTokenizer_1.TransformersTokenizer(callableTokenizer);
+        const text = tokenizer.decode([1, 101, 102, 2]);
+        node_assert_1.strict.equal(text, 'hello world', 'should decode with skip_special_tokens=true');
+    });
+    (0, mocha_1.it)('handles empty input', () => {
+        const mockTokenizer = {
+            __call__: () => ({
+                input_ids: { data: BigInt64Array.from([]) }
+            }),
+            decode: () => ''
+        };
+        const callableTokenizer = Object.assign(() => mockTokenizer.__call__(), { decode: mockTokenizer.decode });
+        const tokenizer = new TransformersTokenizer_1.TransformersTokenizer(callableTokenizer);
+        const tokens = tokenizer.encode('');
+        node_assert_1.strict.deepEqual(tokens, [], 'empty input should return empty array');
+        const text = tokenizer.decode([]);
+        node_assert_1.strict.equal(text, '', 'empty tokens should return empty string');
+    });
+    (0, mocha_1.it)('returns consistent results for same input', () => {
+        let callCount = 0;
+        const mockTokenizer = {
+            __call__: () => {
+                callCount++;
+                return {
+                    input_ids: { data: BigInt64Array.from([BigInt(101), BigInt(102)]) }
+                };
+            },
+            decode: () => 'hello world'
+        };
+        const callableTokenizer = Object.assign(() => mockTokenizer.__call__(), { decode: mockTokenizer.decode });
+        const tokenizer = new TransformersTokenizer_1.TransformersTokenizer(callableTokenizer);
+        const tokens1 = tokenizer.encode('hello world');
+        const tokens2 = tokenizer.encode('hello world');
+        node_assert_1.strict.deepEqual(tokens1, tokens2, 'encode should be deterministic');
+        node_assert_1.strict.equal(callCount, 2, 'should call underlying tokenizer each time');
+    });
+});
+//# sourceMappingURL=TransformersTokenizer.spec.js.map

package/lib/TransformersTokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformersTokenizer.spec.js","sourceRoot":"","sources":["../src/TransformersTokenizer.spec.ts"],"names":[],"mappings":";;AAAA,6CAA+C;AAC/C,iCAAqC;AACrC,mEAAgE;AAEhE,IAAA,gBAAQ,EAAC,uBAAuB,EAAE,GAAG,EAAE;IACnC,+DAA+D;IAC/D,SAAS,mBAAmB;QACxB,MAAM,KAAK,GAAwB,IAAI,GAAG,CAAC;YACvC,CAAC,OAAO,EAAE,GAAG,CAAC;YACd,CAAC,OAAO,EAAE,GAAG,CAAC;YACd,CAAC,MAAM,EAAE,GAAG,CAAC;YACb,CAAC,OAAO,EAAE,CAAC,CAAC;YACZ,CAAC,OAAO,EAAE,CAAC,CAAC;SACf,CAAC,CAAC;QACH,MAAM,YAAY,GAAwB,IAAI,GAAG,EAAE,CAAC;QACpD,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhD,OAAO;YACH,yCAAyC;YACzC,QAAQ,EAAE,CAAC,IAAY,EAAE,EAAE;gBACvB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAC7D,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAC,OAAA,MAAA,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,mCAAI,GAAG,CAAA,EAAA,CAAC,CAAC;gBAChD,OAAO;oBACH,SAAS,EAAE;wBACP,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;qBACtD;iBACJ,CAAC;YACN,CAAC;YACD,MAAM,EAAE,CAAC,MAAgB,EAAE,OAA2C,EAAE,EAAE;gBACtE,MAAM,KAAK,GAAG,MAAM;qBACf,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,mBAAmB,CAAA,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;qBAClE,GAAG,CAAC,CAAC,CAAC,EAAE,WAAC,OAAA,MAAA,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,mCAAI,OAAO,CAAA,EAAA,CAAC,CAAC;gBAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;SACJ,CAAC;IACN,CAAC;IAED,IAAA,UAAE,EAAC,sDAAsD,EAAE,GAAG,EAAE;QAC5D,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAC;QAC5C,mBAAmB;QACnB,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACnC,CAAC,IAAY,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC9C,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAC5B,CAAC;QAET,MAAM,SAAS,GAAG,IAAI,6CAAqB,CAAC,iBAAiB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAE/C,oBAAM,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,+BAA+B,CAAC,CAAC;QAClE,oBAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,sBAAsB,CAAC,CAAC;QACvD,oBAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qCAAqC,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,4CAA4C,EAAE,GAAG,EAAE;QAClD,MAAM,aAAa,GAAG;YAClB,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;gBACb,SAAS,EAAE;oBACP,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC9D;aACJ,CAAC;YACF,MAAM,EAAE,GAAG,EAAE,CAAC,SAAS;SAC1B,CAAC;QACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACnC,GAAG,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,EAC9B,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAC5B,CAAC;QAET,MAAM,SAAS,GAAG,IAAI,6CAAqB,CAAC,iBAAiB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAE5C,oBAAM,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,iCAAiC,CAAC,CAAC;QACvE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACf,oBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,6BAA6B,EAAE,GAAG,EAAE;QACnC,MAAM,aAAa,GAAG;YAClB,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACjE,MAAM,EAAE,CAAC,MAAgB,EAAE,IAAwC,EAAE,EAAE;gBACnE,IAAI,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,mBAAmB,EAAE,CAAC;oBAC5B,OAAO,aAAa,CAAC;gBACzB,CAAC;gBACD,OAAO,yBAAyB,CAAC;YACrC,CAAC;SACJ,CAAC;QACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACnC,GAAG,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,EAC9B,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAC5B,CAAC;QAET,MAAM,SAAS,GAAG,IAAI,6CAAqB,CAAC,iBAAiB,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhD,oBAAM,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,6CAA6C,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,qBAAqB,EAAE,GAAG,EAAE;QAC3B,MAAM,aAAa,GAAG;YAClB,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;gBACb,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;aAC9C,CAAC;YACF,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE;SACnB,CAAC;QACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACnC,GAAG,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,EAC9B,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAC5B,CAAC;QAET,MAAM,SAAS,GAAG,IAAI,6CAAqB,CAAC,iBAAiB,CAAC,CAAC;QAE/D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACpC,oBAAM,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,EAAE,uCAAuC,CAAC,CAAC;QAEtE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClC,oBAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,yCAAyC,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,IAAA,UAAE,EAAC,2CAA2C,EAAE,GAAG,EAAE;QACjD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,aAAa,GAAG;YAClB,QAAQ,EAAE,GAAG,EAAE;gBACX,SAAS,EAAE,CAAC;gBACZ,OAAO;oBACH,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;iBACtE,CAAC;YACN,CAAC;YACD,MAAM,EAAE,GAAG,EAAE,CAAC,aAAa;SAC9B,CAAC;QACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CACnC,GAAG,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,EAC9B,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,CAC5B,CAAC;QAET,MAAM,SAAS,GAAG,IAAI,6CAAqB,CAAC,iBAAiB,CAAC,CAAC;QAE/D,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAEhD,oBAAM,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,gCAAgC,CAAC,CAAC;QACrE,oBAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,EAAE,4CAA4C,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/WebFetcher.d.ts

@@ -0,0 +1,14 @@
+import { TextFetcher } from './types';
+export interface WebFetcherConfig {
+    headers?: Record<string, string>;
+    requestConfig?: RequestInit;
+    htmlToMarkdown: boolean;
+    summarizeHtml: boolean;
+}
+export declare class WebFetcher implements TextFetcher {
+    private readonly _config;
+    constructor(config?: Partial<WebFetcherConfig>);
+    fetch(uri: string, onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>): Promise<boolean>;
+    private htmlToMarkdown;
+}
+//# sourceMappingURL=WebFetcher.d.ts.map

package/lib/WebFetcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebFetcher.d.ts","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AA2BtC,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,WAAW,CAAC;IAC5B,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,qBAAa,UAAW,YAAW,WAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;gBAExB,MAAM,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAUxC,KAAK,CAChB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC5E,OAAO,CAAC,OAAO,CAAC;IAsCnB,OAAO,CAAC,cAAc;CAsCvB"}