vectra 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/lib/LocalDocumentIndex.d.ts.map +1 -1
- package/lib/LocalDocumentIndex.js +6 -4
- package/lib/LocalDocumentIndex.js.map +1 -1
- package/lib/OpenAIEmbeddings.d.ts +1 -0
- package/lib/OpenAIEmbeddings.d.ts.map +1 -1
- package/lib/OpenAIEmbeddings.js +2 -1
- package/lib/OpenAIEmbeddings.js.map +1 -1
- package/lib/TextSplitter.d.ts.map +1 -1
- package/lib/TextSplitter.js +5 -0
- package/lib/TextSplitter.js.map +1 -1
- package/lib/types.d.ts +4 -0
- package/lib/types.d.ts.map +1 -1
- package/lib/vectra-cli.d.ts.map +1 -1
- package/lib/vectra-cli.js +2 -1
- package/lib/vectra-cli.js.map +1 -1
- package/package.json +1 -1
- package/src/LocalDocumentIndex.ts +6 -5
- package/src/OpenAIEmbeddings.ts +3 -1
- package/src/TextSplitter.ts +6 -0
- package/src/types.ts +5 -0
- package/src/vectra-cli.ts +2 -2
package/README.md
CHANGED
|
@@ -7,6 +7,11 @@ Keep in mind that your entire Vectra index is loaded into memory so it's not wel
|
|
|
7
7
|
|
|
8
8
|
Pinecone style namespaces aren't directly supported but you could easily mimic them by creating a separate Vectra index (and folder) for each namespace.
|
|
9
9
|
|
|
10
|
+
## Other Language Bindings
|
|
11
|
+
This repo contains the TypeScript/JavaScript binding for Vectra but other language bindings are being created. Since Vectra is file based, any language binding can be used to read or write a Vectra index. That means you can build a Vectra index using JS and then read it using Python.
|
|
12
|
+
|
|
13
|
+
- [vectra-py](https://github.com/BMS-geodev/vectra-py) - Python version of Vectra.
|
|
14
|
+
|
|
10
15
|
## Installation
|
|
11
16
|
|
|
12
17
|
```
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAA0D,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"LocalDocumentIndex.d.ts","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAgB,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,SAAS,EAAE,aAAa,EAA0D,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAClK,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,MAAM,WAAW,oBAAoB;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,cAAc,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAwB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,cAAc,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAChD;AAED,qBAAa,kBAAmB,SAAQ,UAAU;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAkB;IAC/C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAC,CAAkB;gBAGnB,MAAM,EAAE,wBAAwB;IAYnD;;OAEG;IACU,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC;IASpC,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAKvD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAK/D,WAAW,CAAC,MAAM,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKtD,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C1C,eAAe,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAU7D;;;;;;;OAOG;IACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IA+G3G,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAqD7F,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAKlC,YAAY,IAAI,IAAI;IAKd,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;cAavB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;CA2BjD"}
|
|
@@ -41,7 +41,6 @@ const LocalIndex_1 = require("./LocalIndex");
|
|
|
41
41
|
const TextSplitter_1 = require("./TextSplitter");
|
|
42
42
|
const LocalDocumentResult_1 = require("./LocalDocumentResult");
|
|
43
43
|
const LocalDocument_1 = require("./LocalDocument");
|
|
44
|
-
const EMBEDDINGS_BATCH_SIZE = 500;
|
|
45
44
|
class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
|
|
46
45
|
constructor(config) {
|
|
47
46
|
var _a, _b;
|
|
@@ -182,14 +181,17 @@ class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
|
|
|
182
181
|
const splitter = new TextSplitter_1.TextSplitter(config);
|
|
183
182
|
const chunks = splitter.split(text);
|
|
184
183
|
// Break chunks into batches for embedding generation
|
|
184
|
+
let totalTokens = 0;
|
|
185
185
|
const chunkBatches = [];
|
|
186
186
|
let currentBatch = [];
|
|
187
187
|
for (const chunk of chunks) {
|
|
188
|
-
|
|
189
|
-
if (
|
|
188
|
+
totalTokens += chunk.tokens.length;
|
|
189
|
+
if (totalTokens > this._embeddings.maxTokens) {
|
|
190
190
|
chunkBatches.push(currentBatch);
|
|
191
191
|
currentBatch = [];
|
|
192
|
+
totalTokens = chunk.tokens.length;
|
|
192
193
|
}
|
|
194
|
+
currentBatch.push(chunk.text.replace(/\n/g, ' '));
|
|
193
195
|
}
|
|
194
196
|
if (currentBatch.length > 0) {
|
|
195
197
|
chunkBatches.push(currentBatch);
|
|
@@ -267,7 +269,7 @@ class LocalDocumentIndex extends LocalIndex_1.LocalIndex {
|
|
|
267
269
|
// Generate embeddings for query
|
|
268
270
|
let embeddings;
|
|
269
271
|
try {
|
|
270
|
-
embeddings = yield this._embeddings.createEmbeddings(query);
|
|
272
|
+
embeddings = yield this._embeddings.createEmbeddings(query.replace(/\n/g, ' '));
|
|
271
273
|
}
|
|
272
274
|
catch (err) {
|
|
273
275
|
throw new Error(`Error generating embeddings for query: ${err.toString()}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalDocumentIndex.js","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,mDAAgD;AAChD,6CAA6D;AAC7D,iDAAkE;AAElE,+DAA4D;AAC5D,mDAAgD;
|
|
1
|
+
{"version":3,"file":"LocalDocumentIndex.js","sourceRoot":"","sources":["../src/LocalDocumentIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,mDAAgD;AAChD,6CAA6D;AAC7D,iDAAkE;AAElE,+DAA4D;AAC5D,mDAAgD;AAehD,MAAa,kBAAmB,SAAQ,uBAAU;IAQ9C,YAAmB,MAAgC;;QAC/C,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;YACjC,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,CAAC;SACI,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAA,MAAA,MAAM,CAAC,SAAS,mCAAI,IAAI,CAAC,eAAe,CAAC,SAAS,mCAAI,IAAI,6BAAa,EAAE,CAAC;QAC5F,IAAI,CAAC,eAAe,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;IACrD,CAAC;IAED;;OAEG;IACU,gBAAgB;;YACzB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC,CAAC;gBAC5D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAEY,aAAa,CAAC,GAAW;;;YAClC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,GAAG,CAAC,CAAC;;KACtC;IAEY,cAAc,CAAC,UAAkB;;;YAC1C,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,UAAU,CAAC,CAAC;;KAC7C;IAEY,WAAW,CAAC,MAA0B;;;;;YAC/C,MAAM,OAAM,WAAW,YAAC,MAAM,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/B,CAAC;KAAA;IAEY,cAAc,CAAC,GAAW;;YACnC,qBAAqB;YACrB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YACjD,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,OAAO;aACV;YAED,4DAA4D;YAC5D,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,kCAAkC;gBAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAwB,EAAE,UAAU,EAAE,CAAC,CAAC;gBAErF,gBAAgB;gBAChB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBACxB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;iBACnC;gBAED,4BAA4B;gBAC5B,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACtC,OAAO,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBAC7C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACnF;YAED,6BAA6B;YAC7B,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,CAAC,CAAC;aACpE;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,gBAAiB,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAC3G;YAED,iCAAiC;YACjC,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC,CAAC;aACrE;YAAC,OAAO,GAAY,EAAE;gBACnB,eAAe;aAClB;QACL,CAAC;KAAA;IAEY,eAAe;;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAA;YACxC,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,QAAS,CAAC,OAAO;gBAC/B,SAAS,EAAE,IAAI,CAAC,QAAS,CAAC,KAAK;gBAC/B,MAAM,EAAE,KAAK,CAAC,KAAK;gBACnB,eAAe,EAAE,KAAK,CAAC,eAAe;aACzC,CAAC;QACN,CAAC;KAAA;IAED;;;;;;;OAOG;IACU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,QAAwC;;YAC3F,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,iCAAiC;YACjC,IAAI,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,UAAU,IAAI,SAAS,EAAE;gBACzB,2BAA2B;gBAC3B,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aAClC;iBAAM;gBACH,2BAA2B;gBAC3B,UAAU,GAAG,IAAA,SAAE,GAAE,CAAC;aACrB;YAED,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC;YACvD,MAAM,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,EAAE;gBACV,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACjD,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC;aACxB;YAED,yBAAyB;YACzB,MAAM,QAAQ,GAAG,IAAI,2BAAY,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEpC,qDAAqD;YACrD,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,MAAM,YAAY,GAAe,EAAE,CAAC;YACpC,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBACxB,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBACnC,IAAI,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE;oBAC1C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAChC,YAAY,GAAG,EAAE,CAAC;oBAClB,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACrC;gBACD,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;aACrD;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE;gBACzB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;aACnC;YAED,iCAAiC;YACjC,MAAM,UAAU,GAAe,EAAE,CAAC;YAClC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE;gBAC9B,IAAI,QAA4B,CAAC;gBACjC,IAAI;oBACA,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;iBAC7D;gBAAC,OAAO,GAAY,EAAE;oBACnB,MAAM,IAAI,KAAK,CAAC,gCAAiC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC9E;gBAED,kBAAkB;gBAClB,IAAI,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE;oBAC9B,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;iBACvE;gBAED,2BAA2B;gBAC3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,MAAO,EAAE;oBACtC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;iBAC9B;aACJ;YAED,+BAA+B;YAC/B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI;gBACA,sBAAsB;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;oBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,aAAa,GAA0B,MAAM,CAAC,MAAM,CAAC;wBACvD,UAAU;wBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACvB,EAAE,QAAQ,CAAC,CAAC;oBACb,MAAM,IAAI,CAAC,UAAU,CAAC;wBAClB,EAAE,EAAE,IAAA,SAAE,GAAE;wBACR,QAAQ,EAAE,aAAa;wBACvB,MAAM,EAAE,SAAS;qBACpB,CAAC,CAAC;iBACN;gBAED,6BAA6B;gBAC7B,IAAI,QAAQ,IAAI,SAAS,EAAE;oBACvB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClG;gBAED,yBAAyB;gBACzB,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;gBAE1E,uBAAuB;gBACvB,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC;gBAC5C,IAAI,CAAC,WAAY,CAAC,KAAK,EAAE,CAAC;gBAE1B,iBAAiB;gBACjB,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;YAAC,OAAO,GAAY,EAAE;gBACnB,gCAAgC;gBAChC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,MAAO,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACjF;YAED,kBAAkB;YAClB,OAAO,IAAI,6BAAa,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC;KAAA;IAGY,cAAc,CAAC,KAAa,EAAE,OAA8B;;YACrE,+BAA+B;YAC/B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;aACvD;YAED,6BAA6B;YAC7B,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpB,YAAY,EAAE,EAAE;gBAChB,SAAS,EAAE,EAAE;aAChB,EAAE,OAAO,CAAC,CAAC;YAEZ,gCAAgC;YAChC,IAAI,UAA8B,CAAC;YACnC,IAAI;gBACA,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;aACnF;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,0CAA2C,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACxF;YAED,kBAAkB;YAClB,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE;gBAChC,MAAM,IAAI,KAAK,CAAC,0CAA0C,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC;aACnF;YAED,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAwB,UAAU,CAAC,MAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,SAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAExH,2BAA2B;YAC3B,MAAM,cAAc,GAAoE,EAAE,CAAC;YAC3F,KAAK,MAAM,MAAM,IAAK,OAAO,EAAE;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,SAAS,EAAE;oBAClD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;iBAC5C;gBACD,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aACpD;YAED,6CAA6C;YAC7C,MAAM,eAAe,GAA0B,EAAE,CAAC;YAClD,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE;gBACrC,MAAM,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAW,CAAC;gBAC5D,MAAM,cAAc,GAAG,IAAI,yCAAmB,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1G,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aACxC;YAED,wDAAwD;YACxD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,YAAa,CAAC,CAAC;QAC7F,CAAC;KAAA;IAED,YAAY;IAEC,WAAW;;;;;YACpB,MAAM,OAAM,WAAW,WAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;KAAA;IAEM,YAAY;QACf,KAAK,CAAC,YAAY,EAAE,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;IACjC,CAAC;IAEY,SAAS;;;;;YAClB,MAAM,OAAM,SAAS,WAAE,CAAC;YAExB,IAAI;gBACA,eAAe;gBACf,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC;gBACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC;aAChC;YAAC,OAAM,GAAY,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,kCAAmC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aAChF;QACL,CAAC;KAAA;IAEe,aAAa;;;;;YACzB,MAAM,OAAM,aAAa,WAAE,CAAC;YAE5B,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,OAAO;aACV;YAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YAC/D,IAAI,MAAM,IAAI,CAAC,gBAAgB,EAAE,EAAE;gBAC/B,eAAe;gBACf,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;aACjD;iBAAM;gBACH,IAAI;oBACA,qBAAqB;oBACrB,IAAI,CAAC,QAAQ,GAAG;wBACZ,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,CAAC;wBACR,OAAO,EAAE,EAAE;wBACX,OAAO,EAAE,EAAE;qBACd,CAAC;oBACF,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;iBAClE;gBAAC,OAAM,GAAY,EAAE;oBAClB,MAAM,IAAI,KAAK,CAAC,oCAAqC,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAClF;aACJ;QACL,CAAC;KAAA;CACJ;AArUD,gDAqUC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenAIEmbeddings.d.ts","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":"AAAA,OAAc,EAAiB,aAAa,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAgC,MAAM,aAAa,CAAC;AAE5G,MAAM,WAAW,2BAA2B;IACxC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,2BAA2B;IACxE;;;;OAIG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC7E;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IAEpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IAEzC;;OAEG;IACH,SAAgB,OAAO,EAAE,uBAAuB,GAAC,4BAA4B,CAAC;IAE9E;;;OAGG;gBACgB,OAAO,EAAE,uBAAuB,GAAC,4BAA4B;IAiChF;;;;;OAKG;IACU,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAerF;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAalH;;OAEG;cACa,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;CAqCxG"}
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddings.d.ts","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":"AAAA,OAAc,EAAiB,aAAa,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAgC,MAAM,aAAa,CAAC;AAE5G,MAAM,WAAW,2BAA2B;IACxC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,2BAA2B;IACxE;;;;OAIG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC7E;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IAEpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IAEzC,SAAgB,SAAS,QAAQ;IAEjC;;OAEG;IACH,SAAgB,OAAO,EAAE,uBAAuB,GAAC,4BAA4B,CAAC;IAE9E;;;OAGG;gBACgB,OAAO,EAAE,uBAAuB,GAAC,4BAA4B;IAiChF;;;;;OAKG;IACU,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAerF;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAalH;;OAEG;cACa,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;CAqCxG"}
|
package/lib/OpenAIEmbeddings.js
CHANGED
|
@@ -25,6 +25,7 @@ class OpenAIEmbeddings {
|
|
|
25
25
|
*/
|
|
26
26
|
constructor(options) {
|
|
27
27
|
this.UserAgent = 'AlphaWave';
|
|
28
|
+
this.maxTokens = 8000;
|
|
28
29
|
// Check for azure config
|
|
29
30
|
if (options.azureApiKey) {
|
|
30
31
|
this._useAzure = true;
|
|
@@ -50,7 +51,7 @@ class OpenAIEmbeddings {
|
|
|
50
51
|
}
|
|
51
52
|
// Create client
|
|
52
53
|
this._httpClient = axios_1.default.create({
|
|
53
|
-
validateStatus: (status) =>
|
|
54
|
+
validateStatus: (status) => true
|
|
54
55
|
});
|
|
55
56
|
}
|
|
56
57
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenAIEmbeddings.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgF;AA2EhF;;;GAGG;AACH,MAAa,gBAAgB;
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddings.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgF;AA2EhF;;;GAGG;AACH,MAAa,gBAAgB;IAazB;;;OAGG;IACH,YAAmB,OAA6D;QAb/D,cAAS,GAAG,WAAW,CAAC;QAEzB,cAAS,GAAG,IAAI,CAAC;QAY7B,yBAAyB;QACzB,IAAK,OAAwC,CAAC,WAAW,EAAE;YACvD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;gBACzB,eAAe,EAAE,YAAY;aAChC,EAAE,OAAO,CAAiC,CAAC;YAE5C,gCAAgC;YAChC,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACxB,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACzD;YAED,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,4CAA4C,CAAC,CAAC;aACxH;YAED,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,QAAQ,CAAC;SACzC;aAAM;YACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;aAC5B,EAAE,OAAO,CAA4B,CAAC;SAC1C;QAED,gBAAgB;QAChB,IAAI,CAAC,WAAW,GAAG,eAAK,CAAC,MAAM,CAAC;YAC5B,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI;SACnC,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACU,gBAAgB,CAAC,MAAyB;;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC;gBAC/C,KAAK,EAAE,MAAM;aAChB,CAAC,CAAC;YAEH,mBAAmB;YACnB,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;gBACvB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;aAC5H;iBAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE;gBAC/B,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAA;aAChG;iBAAM;gBACH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,kDAAkD,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;aACpI;QACL,CAAC;KAAA;IAED;;OAEG;IACO,sBAAsB,CAAC,OAA+B;;QAC5D,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;YAC7D,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,aAAa,uBAAuB,OAAO,CAAC,eAAe,2BAA2B,OAAO,CAAC,eAAgB,EAAE,CAAC;YACxI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;aAAM;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;YACxD,MAAM,GAAG,GAAG,GAAG,MAAA,OAAO,CAAC,QAAQ,mCAAI,wBAAwB,gBAAgB,CAAC;YAC3E,OAAwC,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAChE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;IACL,CAAC;IAED;;OAEG;IACa,IAAI,CAAQ,GAAW,EAAE,IAAY,EAAE,UAAU,GAAG,CAAC;;YACjE,4BAA4B;YAC5B,MAAM,aAAa,GAAuB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAExF,6BAA6B;YAC7B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;gBACxB,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;aAC9B;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;gBACxC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;aAC9D;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBACtC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;aACxD;YACD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;gBAC7D,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC;aAC1D;iBAAM;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;gBACxD,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpE,IAAI,OAAO,CAAC,YAAY,EAAE;oBACtB,aAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;iBACvE;aACJ;YAED,eAAe;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YAEvE,6BAA6B;YAC7B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE;gBACnH,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBACnD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;aAC/C;iBAAM;gBACH,OAAO,QAAQ,CAAC;aACnB;QACL,CAAC;KAAA;CACJ;AA/HD,4CA+HC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA4BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;
|
|
1
|
+
{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA4BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IAgDtB,OAAO,CAAC,aAAa;CA8WxB"}
|
package/lib/TextSplitter.js
CHANGED
|
@@ -68,6 +68,11 @@ class TextSplitter {
|
|
|
68
68
|
if (this._config.keepSeparators && !lastChunk) {
|
|
69
69
|
chunk += separator;
|
|
70
70
|
}
|
|
71
|
+
// Check for empty chunk
|
|
72
|
+
const trimmed = chunk.trim();
|
|
73
|
+
if (trimmed.length === 0 || trimmed == '\n') {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
71
76
|
// Encode chunk text
|
|
72
77
|
const tokens = this._config.tokenizer.encode(chunk);
|
|
73
78
|
if (tokens.length > this._config.chunkSize) {
|
package/lib/TextSplitter.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAYhD,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,UAAU,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YACnC,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACtC;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,oBAAoB;gBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACxC,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,yBAAyB;oBACzB,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,KAAK;wBACX,MAAM,EAAE,MAAM;wBACd,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,MAAM;wBACd,YAAY,EAAE,EAAE;wBAChB,UAAU,EAAE,EAAE;qBACjB,CAAC,CAAC;iBACN;gBAED,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;
|
|
1
|
+
{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAYhD,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,UAAU,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;YACnC,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aACtC;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,wBAAwB;gBACxB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;gBAC7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,IAAI,IAAI,EAAE;oBACzC,SAAS;iBACZ;gBAED,oBAAoB;gBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACxC,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,yBAAyB;oBACzB,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,KAAK;wBACX,MAAM,EAAE,MAAM;wBACd,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,MAAM;wBACd,YAAY,EAAE,EAAE;wBAChB,UAAU,EAAE,EAAE;qBACjB,CAAC,CAAC;iBACN;gBAED,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;oBACH,EAAE;iBACL,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAzdD,oCAydC"}
|
package/lib/types.d.ts
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
* An AI model that can be used to create embeddings.
|
|
3
3
|
*/
|
|
4
4
|
export interface EmbeddingsModel {
|
|
5
|
+
/**
|
|
6
|
+
* Maximum number of tokens
|
|
7
|
+
*/
|
|
8
|
+
readonly maxTokens: number;
|
|
5
9
|
/**
|
|
6
10
|
* Creates embeddings for the given inputs.
|
|
7
11
|
* @param inputs Text inputs to create embeddings for.
|
package/lib/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAC,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC1E;AAED;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,SAAS,GAAG,OAAO,GAAG,cAAc,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,wBAAwB,CAAC;IAEjC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAEpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,SAAS,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAE3B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE3B;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAE1B;;OAEG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IAEzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;AAElD,MAAM,WAAW,WAAW,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjE,IAAI,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACjB"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAC,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC1E;AAED;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,SAAS,GAAG,OAAO,GAAG,cAAc,CAAC;AAE5E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,wBAAwB,CAAC;IAEjC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAEpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,SAAS,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAE3B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE3B;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAE1B;;OAEG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IAEzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;AAElD,MAAM,WAAW,WAAW,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjE,IAAI,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACjB"}
|
package/lib/vectra-cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vectra-cli.d.ts","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"vectra-cli.d.ts","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":"AAQA,wBAAsB,GAAG,kBAyNxB"}
|
package/lib/vectra-cli.js
CHANGED
|
@@ -47,6 +47,7 @@ function run() {
|
|
|
47
47
|
return __awaiter(this, void 0, void 0, function* () {
|
|
48
48
|
// prettier-ignore
|
|
49
49
|
const args = yield (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
|
|
50
|
+
.scriptName('vectra')
|
|
50
51
|
.command('create <index>', `create a new local index`, {}, (args) => __awaiter(this, void 0, void 0, function* () {
|
|
51
52
|
const folderPath = args.index;
|
|
52
53
|
const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
|
|
@@ -59,7 +60,7 @@ function run() {
|
|
|
59
60
|
const index = new LocalDocumentIndex_1.LocalDocumentIndex({ folderPath });
|
|
60
61
|
yield index.deleteIndex();
|
|
61
62
|
}))
|
|
62
|
-
.command('add
|
|
63
|
+
.command('add <index>', `adds one or more web pages to an index`, (yargs) => {
|
|
63
64
|
return yargs
|
|
64
65
|
.option('keys', {
|
|
65
66
|
alias: 'k',
|
package/lib/vectra-cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vectra-cli.js","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,wDAAgC;AAChC,2CAAwC;AACxC,6DAA0D;AAC1D,6CAA0C;AAC1C,yDAAsD;AACtD,2CAAuC;
|
|
1
|
+
{"version":3,"file":"vectra-cli.js","sourceRoot":"","sources":["../src/vectra-cli.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,wDAAgC;AAChC,2CAAwC;AACxC,6DAA0D;AAC1D,6CAA0C;AAC1C,yDAAsD;AACtD,2CAAuC;AAEvC,SAAsB,GAAG;;QACrB,kBAAkB;QAClB,MAAM,IAAI,GAAG,MAAM,IAAA,eAAK,EAAC,IAAA,iBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;aAC1C,UAAU,CAAC,QAAQ,CAAC;aACpB,OAAO,CAAC,gBAAgB,EAAE,0BAA0B,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YACtE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,gCAAgC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC5E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAC,CAAC;YAChE,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,aAAa,EAAE,wCAAwC,EAAE,CAAC,KAAK,EAAE,EAAE;YACxE,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;gBAC1F,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,sCAAsC;gBAChD,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,sDAAsD;gBAChE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,0DAA0D;gBACpE,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,GAAG;aACf,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChD,OAAO,IAAI,CAAC;iBACf;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpE,OAAO,IAAI,CAAC;iBACf;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;iBACxK;YACL,CAAC,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;YAEzD,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAElG,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;gBACV,cAAc,EAAE;oBACZ,SAAS,EAAE,IAAI,CAAC,SAAS;iBAC5B;aACJ,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,kBAAkB;YAClB,MAAM,OAAO,GAAG,IAAI,uBAAU,EAAE,CAAC;YACjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACpB,IAAI;oBACA,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC;oBAClD,MAAM,OAAO,GAAI,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;oBACxE,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;iBACvE;gBAAC,OAAO,GAAY,EAAE;oBACnB,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,WAAW,CAAC,oBAAQ,CAAC,KAAK,CAAC,iBAAiB,GAAG,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;iBACxG;aACJ;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,gBAAgB,EAAE,6CAA6C,EAAE,CAAC,KAAK,EAAE,EAAE;YAChF,OAAO,KAAK;iBACP,MAAM,CAAC,KAAK,EAAE;gBACX,KAAK,EAAE,GAAG;gBACV,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,6BAA6B;gBACvC,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,yDAAyD;gBACnE,IAAI,EAAE,QAAQ;aACjB,CAAC;iBACD,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE;oBAChD,OAAO,IAAI,CAAC;iBACf;qBAAM,IAAI,OAAO,IAAI,CAAC,IAAI,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpE,OAAO,IAAI,CAAC;iBACf;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,mJAAmJ,CAAC,CAAC;iBACxK;YACL,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YAErD,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAe,EAAE,IAAI,CAAC,IAAc,EAAE,UAAU,CAAC,CAAC;YAEtF,mBAAmB;YACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACpB,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;gBAC/B,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;aACnC;QACL,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,eAAe,EAAE,oCAAoC,EAAE,EAAE,EAAE,CAAO,IAAI,EAAE,EAAE;YAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,CAAC,CAAA,CAAC;aACD,OAAO,CAAC,uBAAuB,EAAE,uBAAuB,EAAE,CAAC,KAAK,EAAE,EAAE;YACjE,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,gFAAgF;aAC7F,CAAC;iBACD,MAAM,CAAC,gBAAgB,EAAE;gBACtB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,oDAAoD;gBAC9D,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,iDAAiD;gBAC3D,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,EAAE;aACd,CAAC;iBACD,MAAM,CAAC,eAAe,EAAE;gBACrB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,2DAA2D;gBACrE,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,CAAC;aACb,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,6EAA6E;gBACvF,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,IAAI;aAChB,CAAC;iBACD,MAAM,CAAC,QAAQ,EAAE;gBACd,KAAK,EAAE,GAAG;gBACV,QAAQ,EAAE,wDAAwD;gBAClE,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,CAAC;gBACxC,OAAO,EAAE,UAAU;aACtB,CAAC;iBACD,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,CAAC,EAAE,CAAO,IAAI,EAAE,EAAE;YACd,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAE9C,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAc,EAAE,OAAO,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,mCAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,wBAAwB,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAElG,mBAAmB;YACnB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAe,CAAC;YACxC,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC;gBACjC,UAAU;gBACV,UAAU;aACb,CAAC,CAAC;YAEH,cAAc;YACd,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;YACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE;gBAC9C,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,SAAS,EAAE,IAAI,CAAC,UAAU;aAC7B,CAAC,CAAC;YAEH,iBAAiB;YACjB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;gBAC1B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE;oBAC3B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBACtC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC5B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChF,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;wBACpD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;wBAC1D,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;qBAC9C;iBACJ;qBAAM,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE;oBAChC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC1C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;wBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;qBACtE;iBACJ;aACJ;QACL,CAAC,CAAA,CAAC;aACD,IAAI,EAAE;aACN,aAAa,EAAE;aACf,UAAU,EAAE,CAAC;IACtB,CAAC;CAAA;AAzND,kBAyNC;AAGD,SAAe,WAAW,CAAC,KAAe,EAAE,QAAgB,EAAE,OAAe;;QACzE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,OAAO,KAAK,CAAC;SAChB;aAAM,IAAI,OAAO,QAAQ,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE;YAClE,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;SACxF;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,6EAA6E,CAAC,CAAA;SACvJ;IACL,CAAC;CAAA"}
|
package/package.json
CHANGED
|
@@ -8,8 +8,6 @@ import { MetadataFilter, EmbeddingsModel, Tokenizer, MetadataTypes, EmbeddingsRe
|
|
|
8
8
|
import { LocalDocumentResult } from './LocalDocumentResult';
|
|
9
9
|
import { LocalDocument } from './LocalDocument';
|
|
10
10
|
|
|
11
|
-
const EMBEDDINGS_BATCH_SIZE = 500;
|
|
12
|
-
|
|
13
11
|
export interface DocumentQueryOptions {
|
|
14
12
|
maxDocuments?: number;
|
|
15
13
|
maxChunks?: number;
|
|
@@ -163,14 +161,17 @@ export class LocalDocumentIndex extends LocalIndex {
|
|
|
163
161
|
const chunks = splitter.split(text);
|
|
164
162
|
|
|
165
163
|
// Break chunks into batches for embedding generation
|
|
164
|
+
let totalTokens = 0;
|
|
166
165
|
const chunkBatches: string[][] = [];
|
|
167
166
|
let currentBatch: string[] = [];
|
|
168
167
|
for (const chunk of chunks) {
|
|
169
|
-
|
|
170
|
-
if (
|
|
168
|
+
totalTokens += chunk.tokens.length;
|
|
169
|
+
if (totalTokens > this._embeddings.maxTokens) {
|
|
171
170
|
chunkBatches.push(currentBatch);
|
|
172
171
|
currentBatch = [];
|
|
172
|
+
totalTokens = chunk.tokens.length;
|
|
173
173
|
}
|
|
174
|
+
currentBatch.push(chunk.text.replace(/\n/g, ' '));
|
|
174
175
|
}
|
|
175
176
|
if (currentBatch.length > 0) {
|
|
176
177
|
chunkBatches.push(currentBatch);
|
|
@@ -257,7 +258,7 @@ export class LocalDocumentIndex extends LocalIndex {
|
|
|
257
258
|
// Generate embeddings for query
|
|
258
259
|
let embeddings: EmbeddingsResponse;
|
|
259
260
|
try {
|
|
260
|
-
embeddings = await this._embeddings.createEmbeddings(query);
|
|
261
|
+
embeddings = await this._embeddings.createEmbeddings(query.replace(/\n/g, ' '));
|
|
261
262
|
} catch (err: unknown) {
|
|
262
263
|
throw new Error(`Error generating embeddings for query: ${(err as any).toString()}`);
|
|
263
264
|
}
|
package/src/OpenAIEmbeddings.ts
CHANGED
|
@@ -83,6 +83,8 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
83
83
|
|
|
84
84
|
private readonly UserAgent = 'AlphaWave';
|
|
85
85
|
|
|
86
|
+
public readonly maxTokens = 8000;
|
|
87
|
+
|
|
86
88
|
/**
|
|
87
89
|
* Options the client was configured with.
|
|
88
90
|
*/
|
|
@@ -121,7 +123,7 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
121
123
|
|
|
122
124
|
// Create client
|
|
123
125
|
this._httpClient = axios.create({
|
|
124
|
-
validateStatus: (status) =>
|
|
126
|
+
validateStatus: (status) => true
|
|
125
127
|
});
|
|
126
128
|
}
|
|
127
129
|
|
package/src/TextSplitter.ts
CHANGED
|
@@ -85,6 +85,12 @@ export class TextSplitter {
|
|
|
85
85
|
chunk += separator;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
// Check for empty chunk
|
|
89
|
+
const trimmed = chunk.trim();
|
|
90
|
+
if (trimmed.length === 0 || trimmed == '\n') {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
|
|
88
94
|
// Encode chunk text
|
|
89
95
|
const tokens = this._config.tokenizer.encode(chunk);
|
|
90
96
|
if (tokens.length > this._config.chunkSize) {
|
package/src/types.ts
CHANGED
|
@@ -4,6 +4,11 @@
|
|
|
4
4
|
* An AI model that can be used to create embeddings.
|
|
5
5
|
*/
|
|
6
6
|
export interface EmbeddingsModel {
|
|
7
|
+
/**
|
|
8
|
+
* Maximum number of tokens
|
|
9
|
+
*/
|
|
10
|
+
readonly maxTokens: number;
|
|
11
|
+
|
|
7
12
|
/**
|
|
8
13
|
* Creates embeddings for the given inputs.
|
|
9
14
|
* @param inputs Text inputs to create embeddings for.
|
package/src/vectra-cli.ts
CHANGED
|
@@ -5,11 +5,11 @@ import { LocalDocumentIndex } from "./LocalDocumentIndex";
|
|
|
5
5
|
import { WebFetcher } from './WebFetcher';
|
|
6
6
|
import { OpenAIEmbeddings } from './OpenAIEmbeddings';
|
|
7
7
|
import { Colorize } from './internals';
|
|
8
|
-
import { Console } from 'console';
|
|
9
8
|
|
|
10
9
|
export async function run() {
|
|
11
10
|
// prettier-ignore
|
|
12
11
|
const args = await yargs(hideBin(process.argv))
|
|
12
|
+
.scriptName('vectra')
|
|
13
13
|
.command('create <index>', `create a new local index`, {}, async (args) => {
|
|
14
14
|
const folderPath = args.index as string;
|
|
15
15
|
const index = new LocalDocumentIndex({ folderPath });
|
|
@@ -22,7 +22,7 @@ export async function run() {
|
|
|
22
22
|
const index = new LocalDocumentIndex({ folderPath });
|
|
23
23
|
await index.deleteIndex();
|
|
24
24
|
})
|
|
25
|
-
.command('add
|
|
25
|
+
.command('add <index>', `adds one or more web pages to an index`, (yargs) => {
|
|
26
26
|
return yargs
|
|
27
27
|
.option('keys', {
|
|
28
28
|
alias: 'k',
|