vectra 0.12.2 → 0.14.0

package/lib/OpenAIEmbeddings.spec.js

@@ -0,0 +1,298 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_assert_1 = __importDefault(require("node:assert"));
+const sinon_1 = __importDefault(require("sinon"));
+const OpenAIEmbeddings_1 = require("./OpenAIEmbeddings");
+describe('OpenAIEmbeddings', () => {
+    let sandbox;
+    let fetchStub;
+    function makeFetchResponse(status, data, statusText = '') {
+        return {
+            status,
+            statusText,
+            ok: status >= 200 && status < 300,
+            headers: new Headers(),
+            json: () => __awaiter(this, void 0, void 0, function* () { return data; }),
+            text: () => __awaiter(this, void 0, void 0, function* () { return JSON.stringify(data); }),
+        };
+    }
+    const successData = {
+        object: 'list',
+        model: 'any',
+        data: [
+            { index: 0, object: 'embedding', embedding: [0.1, 0.2] }
+        ],
+        usage: { prompt_tokens: 1, total_tokens: 1 }
+    };
+    beforeEach(() => {
+        sandbox = sinon_1.default.createSandbox();
+        fetchStub = sandbox.stub(globalThis, 'fetch');
+    });
+    afterEach(() => {
+        sandbox.restore();
+    });
+    it('Azure: trims trailing slash, enforces https, defaults apiVersion and maxTokens', () => {
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            azureApiKey: 'key',
+            azureEndpoint: 'https://example.com/',
+            azureDeployment: 'dep'
+        });
+        const opts = inst.options;
+        node_assert_1.default.strictEqual(opts.azureEndpoint, 'https://example.com');
+        node_assert_1.default.strictEqual(opts.azureApiVersion, '2023-05-15');
+        node_assert_1.default.strictEqual(inst.maxTokens, 500);
+        // Private enum: ClientType.AzureOpenAI == 1
+        node_assert_1.default.strictEqual(inst._clientType, 1);
+    });
+    it('Azure: throws for non-https endpoint', () => {
+        node_assert_1.default.throws(() => {
+            new OpenAIEmbeddings_1.OpenAIEmbeddings({
+                azureApiKey: 'key',
+                azureEndpoint: 'http://example.com',
+                azureDeployment: 'dep'
+            });
+        }, /Client created with an invalid endpoint of 'http:\/\/example.com'. The endpoint must be a valid HTTPS url\./);
+    });
+    it('OSS: selects OSS client type, defaults retryPolicy, respects maxTokens', () => {
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            ossModel: 'oss-emb',
+            ossEndpoint: 'https://oss.example.com',
+            maxTokens: 1234
+        });
+        const opts = inst.options;
+        node_assert_1.default.deepStrictEqual(opts.retryPolicy, [2000, 5000]);
+        node_assert_1.default.strictEqual(inst.maxTokens, 1234);
+        // Private enum: ClientType.OSS == 2
+        node_assert_1.default.strictEqual(inst._clientType, 2);
+    });
+    it('OpenAI: selects OpenAI client type and defaults retryPolicy', () => {
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'text-embedding-3-small'
+        });
+        const opts = inst.options;
+        node_assert_1.default.deepStrictEqual(opts.retryPolicy, [2000, 5000]);
+        // Private enum: ClientType.OpenAI == 0
+        node_assert_1.default.strictEqual(inst._clientType, 0);
+    });
+    it('Azure: URL formation and dimensions passthrough', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            azureApiKey: 'key',
+            azureEndpoint: 'https://example.com/',
+            azureDeployment: 'dep',
+            dimensions: 256
+        });
+        yield inst.createEmbeddings('hello');
+        const [url, init] = fetchStub.getCall(0).args;
+        node_assert_1.default.strictEqual(url, 'https://example.com/openai/deployments/dep/embeddings?api-version=2023-05-15');
+        const body = JSON.parse(init.body);
+        node_assert_1.default.strictEqual(body.input, 'hello');
+        node_assert_1.default.strictEqual(body.dimensions, 256);
+    }));
+    it('OSS: URL formation and model injection', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            ossModel: 'oss-emb',
+            ossEndpoint: 'https://oss.example.com'
+        });
+        yield inst.createEmbeddings('text');
+        const [url, init] = fetchStub.getCall(0).args;
+        node_assert_1.default.strictEqual(url, 'https://oss.example.com/v1/embeddings');
+        const body = JSON.parse(init.body);
+        node_assert_1.default.strictEqual(body.input, 'text');
+        node_assert_1.default.strictEqual(body.model, 'oss-emb');
+    }));
+    it('OpenAI: URL formation (default endpoint) and model injection', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm'
+        });
+        yield inst.createEmbeddings('x');
+        const [url, init] = fetchStub.getCall(0).args;
+        node_assert_1.default.strictEqual(url, 'https://api.openai.com/v1/embeddings');
+        const body = JSON.parse(init.body);
+        node_assert_1.default.strictEqual(body.model, 'm');
+    }));
+    it('OpenAI: URL formation (custom endpoint)', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            endpoint: 'https://custom'
+        });
+        yield inst.createEmbeddings('x');
+        const [url] = fetchStub.getCall(0).args;
+        node_assert_1.default.strictEqual(url, 'https://custom/v1/embeddings');
+    }));
+    it('Azure: headers include Content-Type, User-Agent, api-key', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            azureApiKey: 'key',
+            azureEndpoint: 'https://example.com/',
+            azureDeployment: 'dep'
+        });
+        yield inst.createEmbeddings('x');
+        const [, init] = fetchStub.getCall(0).args;
+        const headers = new Headers(init.headers);
+        node_assert_1.default.strictEqual(headers.get('Content-Type'), 'application/json');
+        node_assert_1.default.strictEqual(headers.get('User-Agent'), 'AlphaWave');
+        node_assert_1.default.strictEqual(headers.get('api-key'), 'key');
+    }));
+    it('OpenAI: headers include Authorization and OpenAI-Organization when provided', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            organization: 'org1'
+        });
+        yield inst.createEmbeddings('x');
+        const [, init] = fetchStub.getCall(0).args;
+        const headers = new Headers(init.headers);
+        node_assert_1.default.strictEqual(headers.get('Authorization'), 'Bearer sk');
+        node_assert_1.default.strictEqual(headers.get('OpenAI-Organization'), 'org1');
+    }));
+    it('Respects pre-supplied headers from requestConfig and does not overwrite them', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            requestConfig: {
+                headers: {
+                    'Content-Type': 'application/x-custom',
+                    'User-Agent': 'MyUA',
+                    'X-Custom': '1'
+                }
+            }
+        });
+        yield inst.createEmbeddings('x');
+        const [, init] = fetchStub.getCall(0).args;
+        const headers = new Headers(init.headers);
+        node_assert_1.default.strictEqual(headers.get('Content-Type'), 'application/x-custom'); // preserved
+        node_assert_1.default.strictEqual(headers.get('User-Agent'), 'MyUA'); // preserved
+        node_assert_1.default.strictEqual(headers.get('X-Custom'), '1'); // preserved
+        node_assert_1.default.strictEqual(headers.get('Authorization'), 'Bearer sk'); // added
+    }));
+    it('429 retry path obeys retryPolicy delays and eventually succeeds', () => __awaiter(void 0, void 0, void 0, function* () {
+        const clock = sandbox.useFakeTimers();
+        const resp429 = makeFetchResponse(429, {});
+        const resp200 = makeFetchResponse(200, successData);
+        fetchStub.onCall(0).resolves(resp429);
+        fetchStub.onCall(1).resolves(resp429);
+        fetchStub.onCall(2).resolves(resp200);
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            retryPolicy: [10, 20]
+        });
+        const p = inst.createEmbeddings('x');
+        yield clock.tickAsync(10);
+        yield clock.tickAsync(20);
+        const result = yield p;
+        node_assert_1.default.strictEqual(fetchStub.callCount, 3);
+        node_assert_1.default.strictEqual(result.status, 'success');
+        clock.restore();
+    }));
+    it('429 with empty retryPolicy returns rate_limited', () => __awaiter(void 0, void 0, void 0, function* () {
+        const resp429 = makeFetchResponse(429, {});
+        fetchStub.resolves(resp429);
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            retryPolicy: []
+        });
+        const result = yield inst.createEmbeddings('x');
+        node_assert_1.default.strictEqual(result.status, 'rate_limited');
+        node_assert_1.default.ok((result.message || '').includes('rate limit'));
+    }));
+    it('Success maps embeddings sorted by index', () => __awaiter(void 0, void 0, void 0, function* () {
+        const data = {
+            object: 'list',
+            model: 'any',
+            data: [
+                { index: 2, object: 'embedding', embedding: [3] },
+                { index: 0, object: 'embedding', embedding: [1] },
+                { index: 1, object: 'embedding', embedding: [2] }
+            ],
+            usage: { prompt_tokens: 1, total_tokens: 1 }
+        };
+        fetchStub.resolves(makeFetchResponse(200, data));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm'
+        });
+        const result = yield inst.createEmbeddings(['a', 'b', 'c']);
+        node_assert_1.default.strictEqual(result.status, 'success');
+        node_assert_1.default.deepStrictEqual(result.output, [[1], [2], [3]]);
+    }));
+    it('Non-429 error returns status error with message', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(500, {}, 'Internal Server Error'));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm'
+        });
+        const result = yield inst.createEmbeddings('x');
+        node_assert_1.default.strictEqual(result.status, 'error');
+        node_assert_1.default.strictEqual(result.message, 'The embeddings API returned an error status of 500: Internal Server Error');
+    }));
+    it('logRequests true logs request and response details', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const logSpy = sandbox.stub(console, 'log');
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            logRequests: true
+        });
+        yield inst.createEmbeddings('x');
+        node_assert_1.default.ok(logSpy.callCount >= 4);
+    }));
+    it('Input handling: string preserved in body', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm'
+        });
+        yield inst.createEmbeddings('hello');
+        const [, init] = fetchStub.getCall(0).args;
+        const body = JSON.parse(init.body);
+        node_assert_1.default.strictEqual(body.input, 'hello');
+    }));
+    it('Input handling: string[] preserved in body', () => __awaiter(void 0, void 0, void 0, function* () {
+        fetchStub.resolves(makeFetchResponse(200, successData));
+        const inst = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm'
+        });
+        yield inst.createEmbeddings(['a', 'b']);
+        const [, init] = fetchStub.getCall(0).args;
+        const body = JSON.parse(init.body);
+        node_assert_1.default.deepStrictEqual(body.input, ['a', 'b']);
+    }));
+    it('maxTokens default and override', () => {
+        const instDefault = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm'
+        });
+        node_assert_1.default.strictEqual(instDefault.maxTokens, 500);
+        const instOverride = new OpenAIEmbeddings_1.OpenAIEmbeddings({
+            apiKey: 'sk',
+            model: 'm',
+            maxTokens: 1024
+        });
+        node_assert_1.default.strictEqual(instOverride.maxTokens, 1024);
+    });
+});
+//# sourceMappingURL=OpenAIEmbeddings.spec.js.map

package/lib/OpenAIEmbeddings.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OpenAIEmbeddings.spec.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,8DAAiC;AACjC,kDAA0B;AAC1B,yDAK4B;AAE5B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,IAAI,OAA2B,CAAC;IAChC,IAAI,SAA0B,CAAC;IAE/B,SAAS,iBAAiB,CAAC,MAAc,EAAE,IAAS,EAAE,UAAU,GAAG,EAAE;QACnE,OAAO;YACL,MAAM;YACN,UAAU;YACV,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;YACjC,OAAO,EAAE,IAAI,OAAO,EAAE;YACtB,IAAI,EAAE,GAAS,EAAE,gDAAC,OAAA,IAAI,CAAA,GAAA;YACtB,IAAI,EAAE,GAAS,EAAE,gDAAC,OAAA,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA,GAAA;SAC3B,CAAC;IAChB,CAAC;IAED,MAAM,WAAW,GAAG;QAClB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,KAAK;QACZ,IAAI,EAAE;YACJ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;SACzD;QACD,KAAK,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;KAC7C,CAAC;IAEF,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,eAAK,CAAC,aAAa,EAAE,CAAC;QAChC,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,CAAC,OAAO,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gFAAgF,EAAE,GAAG,EAAE;QACxF,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,WAAW,EAAE,KAAK;YAClB,aAAa,EAAE,sBAAsB;YACrC,eAAe,EAAE,KAAK;SACS,CAAC,CAAC;QAEnC,MAAM,IAAI,GAAI,IAAY,CAAC,OAAuC,CAAC;QACnE,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,EAAE,qBAAqB,CAAC,CAAC;QAC9D,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;QACvD,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAExC,4CAA4C;QAC5C,qBAAM,CAAC,WAAW,CAAE,IAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,qBAAM,CAAC,MAAM,CAAC,GAAG,EAAE;YACjB,IAAI,mCAAgB,CAAC;gBACnB,WAAW,EAAE,KAAK;gBAClB,aAAa,EAAE,oBAAoB;gBACnC,eAAe,EAAE,KAAK;aACS,CAAC,CAAC;QACrC,CAAC,EAAE,6GAA6G,CAAC,CAAC;IACpH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAChF,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,QAAQ,EAAE,SAAS;YACnB,WAAW,EAAE,yBAAyB;YACtC,SAAS,EAAE,IAAI;SACQ,CAAC,CAAC;QAE3B,MAAM,IAAI,GAAI,IAAY,CAAC,OAA+B,CAAC;QAC3D,qBAAM,CAAC,eAAe,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QACvD,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QAEzC,oCAAoC;QACpC,qBAAM,CAAC,WAAW,CAAE,IAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACrE,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,wBAAwB;SACL,CAAC,CAAC;QAE9B,MAAM,IAAI,GAAI,IAAY,CAAC,OAAkC,CAAC;QAC9D,qBAAM,CAAC,eAAe,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;QAEvD,uCAAuC;QACvC,qBAAM,CAAC,WAAW,CAAE,IAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAS,EAAE;QAC/D,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,WAAW,EAAE,KAAK;YAClB,aAAa,EAAE,sBAAsB;YACrC,eAAe,EAAE,KAAK;YACtB,UAAU,EAAE,GAAG;SACgB,CAAC,CAAC;QAEnC,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAErC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,qBAAM,CAAC,WAAW,CAAC,GAAG,EAAE,8EAA8E,CAAC,CAAC;QACxG,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACxC,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IAC3C,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAS,EAAE;QACtD,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,QAAQ,EAAE,SAAS;YACnB,WAAW,EAAE,yBAAyB;SACf,CAAC,CAAC;QAE3B,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,qBAAM,CAAC,WAAW,CAAC,GAAG,EAAE,uCAAuC,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACvC,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5C,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAS,EAAE;QAC5E,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;SACgB,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,qBAAM,CAAC,WAAW,CAAC,GAAG,EAAE,sCAAsC,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAS,EAAE;QACvD,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,QAAQ,EAAE,gBAAgB;SACA,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxC,qBAAM,CAAC,WAAW,CAAC,GAAG,EAAE,8BAA8B,CAAC,CAAC;IAC1D,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAS,EAAE;QACxE,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,WAAW,EAAE,KAAK;YAClB,aAAa,EAAE,sBAAsB;YACrC,eAAe,EAAE,KAAK;SACS,CAAC,CAAC;QAEnC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1C,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,kBAAkB,CAAC,CAAC;QACpE,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,WAAW,CAAC,CAAC;QAC3D,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;IACpD,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,GAAS,EAAE;QAC3F,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,YAAY,EAAE,MAAM;SACM,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1C,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,WAAW,CAAC,CAAC;QAC9D,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC,CAAC;IACjE,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,GAAS,EAAE;QAC5F,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,aAAa,EAAE;gBACb,OAAO,EAAE;oBACP,cAAc,EAAE,sBAAsB;oBACtC,YAAY,EAAE,MAAM;oBACpB,UAAU,EAAE,GAAG;iBAChB;aACF;SACyB,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE1C,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,sBAAsB,CAAC,CAAC,CAAC,YAAY;QACrF,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,YAAY;QACnE,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,YAAY;QAC9D,qBAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,QAAQ;IACzE,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAS,EAAE;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;QAEtC,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,EAAE,EAAS,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAEpD,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACtC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACtC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAEtC,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;SACK,CAAC,CAAC;QAE9B,MAAM,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC;QAEvB,qBAAM,CAAC,WAAW,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC3C,qBAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE7C,KAAK,CAAC,OAAO,EAAE,CAAC;IAClB,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAS,EAAE;QAC/D,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,EAAE,EAAS,CAAC,CAAC;QAClD,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAE5B,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,WAAW,EAAE,EAAE;SACW,CAAC,CAAC;QAE9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,qBAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QAClD,qBAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAS,EAAE;QACvD,MAAM,IAAI,GAAG;YACX,MAAM,EAAE,MAAM;YACd,KAAK,EAAE,KAAK;YACZ,IAAI,EAAE;gBACJ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;gBACjD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;gBACjD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE;aAClD;YACD,KAAK,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;SAC7C,CAAC;QACF,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;SACgB,CAAC,CAAC;QAE9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC5D,qBAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC7C,qBAAM,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAS,EAAE;QAC/D,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,EAAS,EAAE,uBAAuB,CAAC,CAAC,CAAC;QAE/E,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;SACgB,CAAC,CAAC;QAE9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,qBAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC3C,qBAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,EAAE,2EAA2E,CAAC,CAAC;IAClH,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAS,EAAE;QAClE,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE5C,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,WAAW,EAAE,IAAI;SACS,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAEjC,qBAAM,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;IACnC,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,GAAS,EAAE;QACxD,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;SACgB,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,qBAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAS,EAAE;QAC1D,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,mCAAgB,CAAC;YAChC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;SACgB,CAAC,CAAC;QAE9B,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,qBAAM,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC,CAAA,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,WAAW,GAAG,IAAI,mCAAgB,CAAC;YACvC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;SACgB,CAAC,CAAC;QAC9B,qBAAM,CAAC,WAAW,CAAC,WAAW,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAG,IAAI,mCAAgB,CAAC;YACxC,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,GAAG;YACV,SAAS,EAAE,IAAI;SACW,CAAC,CAAC;QAC9B,qBAAM,CAAC,WAAW,CAAC,YAAY,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/lib/TextSplitter.d.ts

@@ -0,0 +1,21 @@
+import { TextChunk, Tokenizer } from "./types";
+export interface TextSplitterConfig {
+    separators: string[];
+    keepSeparators: boolean;
+    chunkSize: number;
+    chunkOverlap: number;
+    tokenizer: Tokenizer;
+    docType?: string;
+}
+export declare class TextSplitter {
+    private readonly _config;
+    constructor(config?: Partial<TextSplitterConfig>);
+    split(text: string): TextChunk[];
+    private recursiveSplit;
+    private stripInlineSeparators;
+    private finalizeToChunks;
+    private combineChunks;
+    private splitBySpaces;
+    private getSeparators;
+}
+//# sourceMappingURL=TextSplitter.d.ts.map

package/lib/TextSplitter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAI/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA2BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,aAAa;IA8BrB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,aAAa;CA4WxB"}
+{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IAwBhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IAoBvC,OAAO,CAAC,cAAc;IAkDtB,OAAO,CAAC,qBAAqB;IAwB7B,OAAO,CAAC,gBAAgB;IAqFxB,OAAO,CAAC,aAAa;IAwCrB,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,aAAa;CA8QtB"}