vectra 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/lib/LocalDocumentResult.d.ts +2 -0
- package/lib/LocalDocumentResult.d.ts.map +1 -1
- package/lib/LocalDocumentResult.js +21 -9
- package/lib/LocalDocumentResult.js.map +1 -1
- package/lib/OpenAIEmbeddings.d.ts +24 -3
- package/lib/OpenAIEmbeddings.d.ts.map +1 -1
- package/lib/OpenAIEmbeddings.js +37 -9
- package/lib/OpenAIEmbeddings.js.map +1 -1
- package/lib/TextSplitter.d.ts.map +1 -1
- package/lib/TextSplitter.js +1 -4
- package/lib/TextSplitter.js.map +1 -1
- package/package.json +1 -1
- package/src/LocalDocumentResult.ts +24 -9
- package/src/OpenAIEmbeddings.ts +67 -13
- package/src/TextSplitter.ts +2 -5
package/README.md
CHANGED
|
@@ -73,7 +73,7 @@ Then query for items:
|
|
|
73
73
|
|
|
74
74
|
```typescript
|
|
75
75
|
async function query(text: string) {
|
|
76
|
-
const vector = await getVector(
|
|
76
|
+
const vector = await getVector(text);
|
|
77
77
|
const results = await index.queryItems(vector, 3);
|
|
78
78
|
if (results.length > 0) {
|
|
79
79
|
for (const result of results) {
|
|
@@ -8,5 +8,7 @@ export declare class LocalDocumentResult extends LocalDocument {
|
|
|
8
8
|
get chunks(): QueryResult<DocumentChunkMetadata>[];
|
|
9
9
|
get score(): number;
|
|
10
10
|
renderSections(maxTokens: number, maxSections: number): Promise<DocumentTextSection[]>;
|
|
11
|
+
private encodeBeforeText;
|
|
12
|
+
private encodeAfterText;
|
|
11
13
|
}
|
|
12
14
|
//# sourceMappingURL=LocalDocumentResult.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalDocumentResult.d.ts","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE7F,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS;IAWlI,IAAW,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAExD;IAED,IAAW,KAAK,IAAI,MAAM,CAEzB;IAEY,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"LocalDocumentResult.d.ts","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE7F,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS;IAWlI,IAAW,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAExD;IAED,IAAW,KAAK,IAAI,MAAM,CAEzB;IAEY,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAsKnG,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,eAAe;CAM1B"}
|
|
@@ -32,13 +32,15 @@ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
|
32
32
|
// Load text from disk
|
|
33
33
|
const text = yield this.loadText();
|
|
34
34
|
// First check to see if the entire document is less than maxTokens
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
35
|
+
if (text.length <= (maxTokens * 8)) {
|
|
36
|
+
const tokens = this._tokenizer.encode(text);
|
|
37
|
+
if (tokens.length < maxTokens) {
|
|
38
|
+
return [{
|
|
39
|
+
text,
|
|
40
|
+
tokenCount: tokens.length,
|
|
41
|
+
score: 1.0
|
|
42
|
+
}];
|
|
43
|
+
}
|
|
42
44
|
}
|
|
43
45
|
// Otherwise, we need to split the document into sections
|
|
44
46
|
// - Add each chunk to a temp array and filter out any chunk that's longer then maxTokens.
|
|
@@ -139,7 +141,7 @@ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
|
139
141
|
const sectionEnd = section.chunks[section.chunks.length - 1].endPos;
|
|
140
142
|
if (sectionStart > 0) {
|
|
141
143
|
const beforeTex = text.substring(0, section.chunks[0].startPos);
|
|
142
|
-
const beforeTokens = this.
|
|
144
|
+
const beforeTokens = this.encodeBeforeText(beforeTex, Math.ceil(budget / 2));
|
|
143
145
|
const beforeBudget = sectionEnd < text.length - 1 ? Math.min(beforeTokens.length, Math.ceil(budget / 2)) : Math.min(beforeTokens.length, budget);
|
|
144
146
|
const chunk = {
|
|
145
147
|
text: this._tokenizer.decode(beforeTokens.slice(-beforeBudget)),
|
|
@@ -154,7 +156,7 @@ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
|
154
156
|
}
|
|
155
157
|
if (sectionEnd < text.length - 1) {
|
|
156
158
|
const afterText = text.substring(sectionEnd + 1);
|
|
157
|
-
const afterTokens = this.
|
|
159
|
+
const afterTokens = this.encodeAfterText(afterText, budget);
|
|
158
160
|
const afterBudget = Math.min(afterTokens.length, budget);
|
|
159
161
|
const chunk = {
|
|
160
162
|
text: this._tokenizer.decode(afterTokens.slice(0, afterBudget)),
|
|
@@ -181,6 +183,16 @@ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
|
181
183
|
});
|
|
182
184
|
});
|
|
183
185
|
}
|
|
186
|
+
encodeBeforeText(text, budget) {
|
|
187
|
+
const maxLength = budget * 8;
|
|
188
|
+
const substr = text.length <= maxLength ? text : text.substring(text.length - maxLength);
|
|
189
|
+
return this._tokenizer.encode(substr);
|
|
190
|
+
}
|
|
191
|
+
encodeAfterText(text, budget) {
|
|
192
|
+
const maxLength = budget * 8;
|
|
193
|
+
const substr = text.length <= maxLength ? text : text.substring(0, maxLength);
|
|
194
|
+
return this._tokenizer.encode(substr);
|
|
195
|
+
}
|
|
184
196
|
}
|
|
185
197
|
exports.LocalDocumentResult = LocalDocumentResult;
|
|
186
198
|
//# sourceMappingURL=LocalDocumentResult.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;AAGhD,MAAa,mBAAoB,SAAQ,6BAAa;IAKlD,YAAmB,UAAkB,EAAE,EAAU,EAAE,GAAW,EAAE,MAA4C,EAAE,SAAoB;QAC9H,KAAK,CAAC,UAAU,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,wBAAwB;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEY,cAAc,CAAC,SAAiB,EAAE,WAAmB;;YAC9D,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,mEAAmE;YACnE,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;AAGhD,MAAa,mBAAoB,SAAQ,6BAAa;IAKlD,YAAmB,UAAkB,EAAE,EAAU,EAAE,GAAW,EAAE,MAA4C,EAAE,SAAoB;QAC9H,KAAK,CAAC,UAAU,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,wBAAwB;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEY,cAAc,CAAC,SAAiB,EAAE,WAAmB;;YAC9D,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,mEAAmE;YACnE,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE;gBAChC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5C,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE;oBAC3B,OAAO,CAAC;4BACJ,IAAI;4BACJ,UAAU,EAAE,MAAM,CAAC,MAAM;4BACzB,KAAK,EAAE,GAAG;yBACb,CAAC,CAAC;iBACN;aACJ;YAED,yDAAyD;YACzD,0FAA0F;YAC1F,oEAAoE;YACpE,0GAA0G;YAC1G,sGAAsG;YACtG,yDAAyD;YACzD,gEAAgE;YAChE,+FAA+F;YAC/F,MAAM,MAAM,GAAmB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gBACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,OAAO;oBACH,IAAI,EAAE,SAAS;oBACf,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM;iBACvD,CAAC;YACN,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE1F,sBAAsB;YACtB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBACrB,qDAAqD;gBACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,OAAO,CAAC;wBACJ,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxD,UAAU,EAAE,SAAS;wBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACxB,CAAC,CAAC;aACN;YAED,oBAAoB;YACpB,MAAM,QAAQ,GAAc,CAAC;oBACzB,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,CAAC;iBAChB,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE;oBACnD,QAAQ,CAAC,IAAI,CAAC;wBACV,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC,CAAC;iBACN;gBACD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACjD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBACnD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;aAChE;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,kDAAkD;YAClD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW,EAAE;gBAC/B,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;aAC/D;YAED,kCAAkC;YAClC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;oBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE;wBACzC,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC;wBAC7B,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;wBAChC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBACzC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChC,CAAC,EAAE,CAAC;qBACP;iBACJ;YACL,CAAC,CAAC,CAAC;YAEH,gFAAgF;YAChF,MAAM,SAAS,GAAiB;gBAC5B,IAAI,EAAE,aAAa;gBACnB,QAAQ,EAAE,CAAC,CAAC;gBACZ,MAAM,EAAE,CAAC,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM;aAC3D,CAAC;YACF,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,mCAAmC;gBACnC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;wBAChD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;wBAC3C,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBAC3C,CAAC,EAAE,CAAC;qBACP;iBACJ;gBAED,4EAA4E;gBAC5E,IAAI,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;gBAC5C,IAAI,MAAM,GAAG,EAAE,EAAE;oBACb,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;oBAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;oBACpE,IAAI,YAAY,GAAG,CAAC,EAAE;wBAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;wBAChE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC;wBAC3E,MAAM,YAAY,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBAC/I,MAAM,KAAK,GAAiB;4BACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;4BAC/D,QAAQ,EAAE,YAAY,GAAG,YAAY;4BACrC,MAAM,EAAE,YAAY,GAAG,CAAC;4BACxB,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,YAAY;yBAC3B,CAAC;wBACF,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBAC9B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;wBACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;qBAC9B;oBAED,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;wBAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;wBACjD,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;wBAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBACzD,MAAM,KAAK,GAAiB;4BACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;4BAC/D,QAAQ,EAAE,UAAU,GAAG,CAAC;4BACxB,MAAM,EAAE,UAAU,GAAG,WAAW;4BAChC,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,WAAW;yBAC1B,CAAC;wBACF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC3B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;wBACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;qBAC9B;iBACJ;YACL,CAAC,CAAC,CAAC;YAEH,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACvB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAEO,gBAAgB,CAAC,IAAY,EAAE,MAAc;QACjD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QACzF,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe,CAAC,IAAY,EAAE,MAAc;QAChD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;CAEJ;AA1MD,kDA0MC"}
|
|
@@ -2,6 +2,12 @@ import { AxiosResponse, AxiosRequestConfig } from 'axios';
|
|
|
2
2
|
import { EmbeddingsModel, EmbeddingsResponse } from "./types";
|
|
3
3
|
import { CreateEmbeddingRequest, CreateEmbeddingResponse } from "./internals";
|
|
4
4
|
export interface BaseOpenAIEmbeddingsOptions {
|
|
5
|
+
/**
|
|
6
|
+
* Optional. Whether to log requests to the console.
|
|
7
|
+
* @remarks
|
|
8
|
+
* This is useful for debugging prompts and defaults to `false`.
|
|
9
|
+
*/
|
|
10
|
+
logRequests?: boolean;
|
|
5
11
|
/**
|
|
6
12
|
* Optional. Retry policy to use when calling the OpenAI API.
|
|
7
13
|
* @remarks
|
|
@@ -14,6 +20,21 @@ export interface BaseOpenAIEmbeddingsOptions {
|
|
|
14
20
|
*/
|
|
15
21
|
requestConfig?: AxiosRequestConfig;
|
|
16
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Options for configuring an `OpenAIEmbeddings` to generate embeddings using an OSS hosted model.
|
|
25
|
+
*/
|
|
26
|
+
export interface OSSEmbeddingsOptions extends BaseOpenAIEmbeddingsOptions {
|
|
27
|
+
/**
|
|
28
|
+
* Model to use for completion.
|
|
29
|
+
*/
|
|
30
|
+
ossModel: string;
|
|
31
|
+
/**
|
|
32
|
+
* Optional. Endpoint to use when calling the OpenAI API.
|
|
33
|
+
* @remarks
|
|
34
|
+
* For Azure OpenAI this is the deployment endpoint.
|
|
35
|
+
*/
|
|
36
|
+
ossEndpoint: string;
|
|
37
|
+
}
|
|
17
38
|
/**
|
|
18
39
|
* Options for configuring an `OpenAIEmbeddings` to generate embeddings using an OpenAI hosted model.
|
|
19
40
|
*/
|
|
@@ -68,18 +89,18 @@ export interface AzureOpenAIEmbeddingsOptions extends BaseOpenAIEmbeddingsOption
|
|
|
68
89
|
*/
|
|
69
90
|
export declare class OpenAIEmbeddings implements EmbeddingsModel {
|
|
70
91
|
private readonly _httpClient;
|
|
71
|
-
private readonly
|
|
92
|
+
private readonly _clientType;
|
|
72
93
|
private readonly UserAgent;
|
|
73
94
|
readonly maxTokens = 8000;
|
|
74
95
|
/**
|
|
75
96
|
* Options the client was configured with.
|
|
76
97
|
*/
|
|
77
|
-
readonly options: OpenAIEmbeddingsOptions | AzureOpenAIEmbeddingsOptions;
|
|
98
|
+
readonly options: OSSEmbeddingsOptions | OpenAIEmbeddingsOptions | AzureOpenAIEmbeddingsOptions;
|
|
78
99
|
/**
|
|
79
100
|
* Creates a new `OpenAIClient` instance.
|
|
80
101
|
* @param options Options for configuring an `OpenAIClient`.
|
|
81
102
|
*/
|
|
82
|
-
constructor(options: OpenAIEmbeddingsOptions | AzureOpenAIEmbeddingsOptions);
|
|
103
|
+
constructor(options: OSSEmbeddingsOptions | OpenAIEmbeddingsOptions | AzureOpenAIEmbeddingsOptions);
|
|
83
104
|
/**
|
|
84
105
|
* Creates embeddings for the given inputs using the OpenAI API.
|
|
85
106
|
* @param model Name of the model to use (or deployment for Azure).
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenAIEmbeddings.d.ts","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":"AAAA,OAAc,EAAiB,aAAa,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAgC,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddings.d.ts","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":"AAAA,OAAc,EAAiB,aAAa,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAgC,MAAM,aAAa,CAAC;AAG5G,MAAM,WAAW,2BAA2B;IACxC;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACtC;AAGD;;GAEG;AACH,MAAM,WAAW,oBAAqB,SAAQ,2BAA2B;IACrE;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,2BAA2B;IACxE;;;;OAIG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC7E;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAa;IAEzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IAEzC,SAAgB,SAAS,QAAQ;IAEjC;;OAEG;IACH,SAAgB,OAAO,EAAE,oBAAoB,GAAC,uBAAuB,GAAC,4BAA4B,CAAC;IAEnG;;;OAGG;gBACgB,OAAO,EAAE,oBAAoB,GAAC,uBAAuB,GAAC,4BAA4B;IAsCrG;;;;;OAKG;IACU,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA6BrF;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAkBlH;;OAEG;cACa,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;CAqCxG"}
|
package/lib/OpenAIEmbeddings.js
CHANGED
|
@@ -14,6 +14,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
14
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
15
|
exports.OpenAIEmbeddings = void 0;
|
|
16
16
|
const axios_1 = __importDefault(require("axios"));
|
|
17
|
+
const internals_1 = require("./internals");
|
|
17
18
|
/**
|
|
18
19
|
* A `PromptCompletionModel` for calling OpenAI and Azure OpenAI hosted models.
|
|
19
20
|
* @remarks
|
|
@@ -28,7 +29,7 @@ class OpenAIEmbeddings {
|
|
|
28
29
|
this.maxTokens = 8000;
|
|
29
30
|
// Check for azure config
|
|
30
31
|
if (options.azureApiKey) {
|
|
31
|
-
this.
|
|
32
|
+
this._clientType = ClientType.AzureOpenAI;
|
|
32
33
|
this.options = Object.assign({
|
|
33
34
|
retryPolicy: [2000, 5000],
|
|
34
35
|
azureApiVersion: '2023-05-15',
|
|
@@ -43,15 +44,21 @@ class OpenAIEmbeddings {
|
|
|
43
44
|
}
|
|
44
45
|
this.options.azureEndpoint = endpoint;
|
|
45
46
|
}
|
|
47
|
+
else if (options.ossModel) {
|
|
48
|
+
this._clientType = ClientType.OSS;
|
|
49
|
+
this.options = Object.assign({
|
|
50
|
+
retryPolicy: [2000, 5000]
|
|
51
|
+
}, options);
|
|
52
|
+
}
|
|
46
53
|
else {
|
|
47
|
-
this.
|
|
54
|
+
this._clientType = ClientType.OpenAI;
|
|
48
55
|
this.options = Object.assign({
|
|
49
56
|
retryPolicy: [2000, 5000]
|
|
50
57
|
}, options);
|
|
51
58
|
}
|
|
52
59
|
// Create client
|
|
53
60
|
this._httpClient = axios_1.default.create({
|
|
54
|
-
validateStatus: (status) =>
|
|
61
|
+
validateStatus: (status) => status < 400 || status == 429
|
|
55
62
|
});
|
|
56
63
|
}
|
|
57
64
|
/**
|
|
@@ -62,19 +69,28 @@ class OpenAIEmbeddings {
|
|
|
62
69
|
*/
|
|
63
70
|
createEmbeddings(inputs) {
|
|
64
71
|
return __awaiter(this, void 0, void 0, function* () {
|
|
72
|
+
if (this.options.logRequests) {
|
|
73
|
+
console.log(internals_1.Colorize.title('EMBEDDINGS REQUEST:'));
|
|
74
|
+
console.log(internals_1.Colorize.output(inputs));
|
|
75
|
+
}
|
|
76
|
+
const startTime = Date.now();
|
|
65
77
|
const response = yield this.createEmbeddingRequest({
|
|
66
78
|
input: inputs,
|
|
67
79
|
});
|
|
80
|
+
if (this.options.logRequests) {
|
|
81
|
+
console.log(internals_1.Colorize.title('RESPONSE:'));
|
|
82
|
+
console.log(internals_1.Colorize.value('status', response.status));
|
|
83
|
+
console.log(internals_1.Colorize.value('duration', Date.now() - startTime, 'ms'));
|
|
84
|
+
console.log(internals_1.Colorize.output(response.data));
|
|
85
|
+
}
|
|
68
86
|
// Process response
|
|
69
87
|
if (response.status < 300) {
|
|
70
|
-
|
|
71
|
-
return { status: 'success', output: data.sort((a, b) => a.index - b.index).map((item) => item.embedding), model, usage };
|
|
88
|
+
return { status: 'success', output: response.data.data.sort((a, b) => a.index - b.index).map((item) => item.embedding) };
|
|
72
89
|
}
|
|
73
90
|
else if (response.status == 429) {
|
|
74
91
|
return { status: 'rate_limited', message: `The embeddings API returned a rate limit error.` };
|
|
75
92
|
}
|
|
76
93
|
else {
|
|
77
|
-
console.log(inputs);
|
|
78
94
|
return { status: 'error', message: `The embeddings API returned an error status of ${response.status}: ${response.statusText}` };
|
|
79
95
|
}
|
|
80
96
|
});
|
|
@@ -84,11 +100,17 @@ class OpenAIEmbeddings {
|
|
|
84
100
|
*/
|
|
85
101
|
createEmbeddingRequest(request) {
|
|
86
102
|
var _a;
|
|
87
|
-
if (this.
|
|
103
|
+
if (this._clientType == ClientType.AzureOpenAI) {
|
|
88
104
|
const options = this.options;
|
|
89
105
|
const url = `${options.azureEndpoint}/openai/deployments/${options.azureDeployment}/embeddings?api-version=${options.azureApiVersion}`;
|
|
90
106
|
return this.post(url, request);
|
|
91
107
|
}
|
|
108
|
+
else if (this._clientType == ClientType.OSS) {
|
|
109
|
+
const options = this.options;
|
|
110
|
+
const url = `${options.ossEndpoint}/v1/embeddings`;
|
|
111
|
+
request.model = options.ossModel;
|
|
112
|
+
return this.post(url, request);
|
|
113
|
+
}
|
|
92
114
|
else {
|
|
93
115
|
const options = this.options;
|
|
94
116
|
const url = `${(_a = options.endpoint) !== null && _a !== void 0 ? _a : 'https://api.openai.com'}/v1/embeddings`;
|
|
@@ -113,11 +135,11 @@ class OpenAIEmbeddings {
|
|
|
113
135
|
if (!requestConfig.headers['User-Agent']) {
|
|
114
136
|
requestConfig.headers['User-Agent'] = this.UserAgent;
|
|
115
137
|
}
|
|
116
|
-
if (this.
|
|
138
|
+
if (this._clientType == ClientType.AzureOpenAI) {
|
|
117
139
|
const options = this.options;
|
|
118
140
|
requestConfig.headers['api-key'] = options.azureApiKey;
|
|
119
141
|
}
|
|
120
|
-
else {
|
|
142
|
+
else if (this._clientType == ClientType.OpenAI) {
|
|
121
143
|
const options = this.options;
|
|
122
144
|
requestConfig.headers['Authorization'] = `Bearer ${options.apiKey}`;
|
|
123
145
|
if (options.organization) {
|
|
@@ -139,4 +161,10 @@ class OpenAIEmbeddings {
|
|
|
139
161
|
}
|
|
140
162
|
}
|
|
141
163
|
exports.OpenAIEmbeddings = OpenAIEmbeddings;
|
|
164
|
+
var ClientType;
|
|
165
|
+
(function (ClientType) {
|
|
166
|
+
ClientType[ClientType["OpenAI"] = 0] = "OpenAI";
|
|
167
|
+
ClientType[ClientType["AzureOpenAI"] = 1] = "AzureOpenAI";
|
|
168
|
+
ClientType[ClientType["OSS"] = 2] = "OSS";
|
|
169
|
+
})(ClientType || (ClientType = {}));
|
|
142
170
|
//# sourceMappingURL=OpenAIEmbeddings.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenAIEmbeddings.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgF;
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddings.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgF;AAGhF,2CAAuC;AAkGvC;;;GAGG;AACH,MAAa,gBAAgB;IAazB;;;OAGG;IACH,YAAmB,OAAkF;QAbpF,cAAS,GAAG,WAAW,CAAC;QAEzB,cAAS,GAAG,IAAI,CAAC;QAY7B,yBAAyB;QACzB,IAAK,OAAwC,CAAC,WAAW,EAAE;YACvD,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;YAC1C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;gBACzB,eAAe,EAAE,YAAY;aAChC,EAAE,OAAO,CAAiC,CAAC;YAE5C,gCAAgC;YAChC,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACxB,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACzD;YAED,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,4CAA4C,CAAC,CAAC;aACxH;YAED,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,QAAQ,CAAC;SACzC;aAAM,IAAK,OAAgC,CAAC,QAAQ,EAAE;YACnD,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;aAC5B,EAAE,OAAO,CAAyB,CAAC;SACvC;aAAM;YACH,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC;YACrC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;aAC5B,EAAE,OAAO,CAA4B,CAAC;SAC1C;QAED,gBAAgB;QAChB,IAAI,CAAC,WAAW,GAAG,eAAK,CAAC,MAAM,CAAC;YAC5B,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG;SAC5D,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACU,gBAAgB,CAAC,MAAyB;;YACnD,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;gBAC1B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;aACxC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC;gBAC/C,KAAK,EAAE,MAAM;aAChB,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;gBAC1B,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;gBACvD,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;gBACtE,OAAO,CAAC,GAAG,CAAC,oBAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;aAC/C;YAGD,mBAAmB;YACnB,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;gBACvB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;aAC5H;iBAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE;gBAC/B,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAA;aAChG;iBAAM;gBACH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,kDAAkD,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;aACpI;QACL,CAAC;KAAA;IAED;;OAEG;IACO,sBAAsB,CAAC,OAA+B;;QAC5D,IAAI,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,WAAW,EAAE;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;YAC7D,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,aAAa,uBAAuB,OAAO,CAAC,eAAe,2BAA2B,OAAO,CAAC,eAAgB,EAAE,CAAC;YACxI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;aAAM,IAAI,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,GAAG,EAAE;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAA+B,CAAC;YACrD,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,WAAW,gBAAgB,CAAC;YAClD,OAAwC,CAAC,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC;YACnE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;aAAM;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;YACxD,MAAM,GAAG,GAAG,GAAG,MAAA,OAAO,CAAC,QAAQ,mCAAI,wBAAwB,gBAAgB,CAAC;YAC3E,OAAwC,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAChE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;IACL,CAAC;IAED;;OAEG;IACa,IAAI,CAAQ,GAAW,EAAE,IAAY,EAAE,UAAU,GAAG,CAAC;;YACjE,4BAA4B;YAC5B,MAAM,aAAa,GAAuB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAExF,6BAA6B;YAC7B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;gBACxB,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;aAC9B;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;gBACxC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;aAC9D;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBACtC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;aACxD;YACD,IAAI,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,WAAW,EAAE;gBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;gBAC7D,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC;aAC1D;iBAAM,IAAI,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,EAAE;gBAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;gBACxD,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpE,IAAI,OAAO,CAAC,YAAY,EAAE;oBACtB,aAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;iBACvE;aACJ;YAED,eAAe;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YAEvE,6BAA6B;YAC7B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE;gBACnH,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBACnD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;aAC/C;iBAAM;gBACH,OAAO,QAAQ,CAAC;aACnB;QACL,CAAC;KAAA;CACJ;AAvJD,4CAuJC;AAED,IAAK,UAIJ;AAJD,WAAK,UAAU;IACX,+CAAM,CAAA;IACN,yDAAW,CAAA;IACX,yCAAG,CAAA;AACP,CAAC,EAJI,UAAU,KAAV,UAAU,QAId"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAI/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA2BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;
|
|
1
|
+
{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAI/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA2BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,aAAa;IA6BrB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;CA4WxB"}
|
package/lib/TextSplitter.js
CHANGED
|
@@ -36,7 +36,7 @@ class TextSplitter {
|
|
|
36
36
|
function getOverlapTokens(tokens) {
|
|
37
37
|
if (tokens != undefined) {
|
|
38
38
|
const len = tokens.length > that._config.chunkOverlap ? that._config.chunkOverlap : tokens.length;
|
|
39
|
-
return tokens.slice(
|
|
39
|
+
return tokens.slice(0, len);
|
|
40
40
|
}
|
|
41
41
|
else {
|
|
42
42
|
return [];
|
|
@@ -113,9 +113,6 @@ class TextSplitter {
|
|
|
113
113
|
// Update startPos
|
|
114
114
|
startPos = endPos + 1;
|
|
115
115
|
}
|
|
116
|
-
if (separator == '// LLM-REGION' && chunks.length > 1) {
|
|
117
|
-
console.log(this.combineChunks(chunks));
|
|
118
|
-
}
|
|
119
116
|
}
|
|
120
117
|
return this.combineChunks(chunks);
|
|
121
118
|
}
|
package/lib/TextSplitter.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAGhD,MAAM,kBAAkB,GAAG,gEAAgE,CAAC;AAW5F,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAGhD,MAAM,kBAAkB,GAAG,gEAAgE,CAAC;AAW5F,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC/B;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;YACjB,wBAAwB;YACxB,IAAI,KAAe,CAAC;YACpB,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,qBAAqB;gBACrB,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;aACjC;iBAAM;gBACH,mBAAmB;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzC,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;aAC3D;YAED,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,6BAA6B;gBAC7B,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;oBACnC,SAAS;iBACZ;gBAED,qDAAqD;gBACrD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBAC3C,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,oBAAoB;oBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;wBACxC,qCAAqC;wBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;wBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;qBAC7B;yBAAM;wBACH,yBAAyB;wBACzB,MAAM,CAAC,IAAI,CAAC;4BACR,IAAI,EAAE,KAAK;4BACX,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,QAAQ;4BAClB,MAAM,EAAE,MAAM;4BACd,YAAY,EAAE,EAAE;4BAChB,UAAU,EAAE,EAAE;yBACjB,CAAC,CAAC;qBACN;iBAEJ;gBAGD,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,MAAmB;QACrC,MAAM,cAAc,GAAgB,EAAE,CAAC;QACvC,IAAI,YAAiC,CAAC;QACtC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,YAAY,EAAE;gBACd,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBAChE,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACjC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAClC,YAAY,GAAG,KAAK,CAAC;oBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACvC;qBAAM;oBACH,YAAY,CAAC,IAAI,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;oBAC5C,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBAC1C,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACxC;aACJ;iBAAM;gBACH,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;aACvC;SACJ;QACD,IAAI,YAAY,EAAE;YACd,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SACrC;QACD,OAAO,cAAc,CAAC;IAC1B,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;gBACtC,OAAO,IAAI,CAAC;aACf;SACJ;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,gBAAgB;oBAChB,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,UAAU;oBACV,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAhhBD,oCAghBC"}
|
package/package.json
CHANGED
|
@@ -30,13 +30,15 @@ export class LocalDocumentResult extends LocalDocument {
|
|
|
30
30
|
const text = await this.loadText();
|
|
31
31
|
|
|
32
32
|
// First check to see if the entire document is less than maxTokens
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
if (text.length <= (maxTokens * 8)) {
|
|
34
|
+
const tokens = this._tokenizer.encode(text);
|
|
35
|
+
if (tokens.length < maxTokens) {
|
|
36
|
+
return [{
|
|
37
|
+
text,
|
|
38
|
+
tokenCount: tokens.length,
|
|
39
|
+
score: 1.0
|
|
40
|
+
}];
|
|
41
|
+
}
|
|
40
42
|
}
|
|
41
43
|
|
|
42
44
|
// Otherwise, we need to split the document into sections
|
|
@@ -145,7 +147,7 @@ export class LocalDocumentResult extends LocalDocument {
|
|
|
145
147
|
const sectionEnd = section.chunks[section.chunks.length - 1].endPos;
|
|
146
148
|
if (sectionStart > 0) {
|
|
147
149
|
const beforeTex = text.substring(0, section.chunks[0].startPos);
|
|
148
|
-
const beforeTokens = this.
|
|
150
|
+
const beforeTokens = this.encodeBeforeText(beforeTex, Math.ceil(budget/2));
|
|
149
151
|
const beforeBudget = sectionEnd < text.length - 1 ? Math.min(beforeTokens.length, Math.ceil(budget/2)) : Math.min(beforeTokens.length, budget);
|
|
150
152
|
const chunk: SectionChunk = {
|
|
151
153
|
text: this._tokenizer.decode(beforeTokens.slice(-beforeBudget)),
|
|
@@ -161,7 +163,7 @@ export class LocalDocumentResult extends LocalDocument {
|
|
|
161
163
|
|
|
162
164
|
if (sectionEnd < text.length - 1) {
|
|
163
165
|
const afterText = text.substring(sectionEnd + 1);
|
|
164
|
-
const afterTokens = this.
|
|
166
|
+
const afterTokens = this.encodeAfterText(afterText, budget);
|
|
165
167
|
const afterBudget = Math.min(afterTokens.length, budget);
|
|
166
168
|
const chunk: SectionChunk = {
|
|
167
169
|
text: this._tokenizer.decode(afterTokens.slice(0, afterBudget)),
|
|
@@ -188,6 +190,19 @@ export class LocalDocumentResult extends LocalDocument {
|
|
|
188
190
|
};
|
|
189
191
|
});
|
|
190
192
|
}
|
|
193
|
+
|
|
194
|
+
private encodeBeforeText(text: string, budget: number): number[] {
|
|
195
|
+
const maxLength = budget * 8;
|
|
196
|
+
const substr = text.length <= maxLength ? text : text.substring(text.length - maxLength);
|
|
197
|
+
return this._tokenizer.encode(substr);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private encodeAfterText(text: string, budget: number): number[] {
|
|
201
|
+
const maxLength = budget * 8;
|
|
202
|
+
const substr = text.length <= maxLength ? text : text.substring(0, maxLength);
|
|
203
|
+
return this._tokenizer.encode(substr);
|
|
204
|
+
}
|
|
205
|
+
|
|
191
206
|
}
|
|
192
207
|
|
|
193
208
|
interface SectionChunk {
|
package/src/OpenAIEmbeddings.ts
CHANGED
|
@@ -1,8 +1,16 @@
|
|
|
1
1
|
import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios';
|
|
2
2
|
import { EmbeddingsModel, EmbeddingsResponse } from "./types";
|
|
3
3
|
import { CreateEmbeddingRequest, CreateEmbeddingResponse, OpenAICreateEmbeddingRequest } from "./internals";
|
|
4
|
+
import { Colorize } from "./internals";
|
|
4
5
|
|
|
5
6
|
export interface BaseOpenAIEmbeddingsOptions {
|
|
7
|
+
/**
|
|
8
|
+
* Optional. Whether to log requests to the console.
|
|
9
|
+
* @remarks
|
|
10
|
+
* This is useful for debugging prompts and defaults to `false`.
|
|
11
|
+
*/
|
|
12
|
+
logRequests?: boolean;
|
|
13
|
+
|
|
6
14
|
/**
|
|
7
15
|
* Optional. Retry policy to use when calling the OpenAI API.
|
|
8
16
|
* @remarks
|
|
@@ -17,6 +25,24 @@ export interface BaseOpenAIEmbeddingsOptions {
|
|
|
17
25
|
requestConfig?: AxiosRequestConfig;
|
|
18
26
|
}
|
|
19
27
|
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Options for configuring an `OpenAIEmbeddings` to generate embeddings using an OSS hosted model.
|
|
31
|
+
*/
|
|
32
|
+
export interface OSSEmbeddingsOptions extends BaseOpenAIEmbeddingsOptions {
|
|
33
|
+
/**
|
|
34
|
+
* Model to use for completion.
|
|
35
|
+
*/
|
|
36
|
+
ossModel: string;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Optional. Endpoint to use when calling the OpenAI API.
|
|
40
|
+
* @remarks
|
|
41
|
+
* For Azure OpenAI this is the deployment endpoint.
|
|
42
|
+
*/
|
|
43
|
+
ossEndpoint: string;
|
|
44
|
+
}
|
|
45
|
+
|
|
20
46
|
/**
|
|
21
47
|
* Options for configuring an `OpenAIEmbeddings` to generate embeddings using an OpenAI hosted model.
|
|
22
48
|
*/
|
|
@@ -79,25 +105,25 @@ export interface AzureOpenAIEmbeddingsOptions extends BaseOpenAIEmbeddingsOption
|
|
|
79
105
|
*/
|
|
80
106
|
export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
81
107
|
private readonly _httpClient: AxiosInstance;
|
|
82
|
-
private readonly
|
|
108
|
+
private readonly _clientType: ClientType;
|
|
83
109
|
|
|
84
110
|
private readonly UserAgent = 'AlphaWave';
|
|
85
111
|
|
|
86
112
|
public readonly maxTokens = 8000;
|
|
87
|
-
|
|
113
|
+
|
|
88
114
|
/**
|
|
89
115
|
* Options the client was configured with.
|
|
90
116
|
*/
|
|
91
|
-
public readonly options: OpenAIEmbeddingsOptions|AzureOpenAIEmbeddingsOptions;
|
|
117
|
+
public readonly options: OSSEmbeddingsOptions|OpenAIEmbeddingsOptions|AzureOpenAIEmbeddingsOptions;
|
|
92
118
|
|
|
93
119
|
/**
|
|
94
120
|
* Creates a new `OpenAIClient` instance.
|
|
95
121
|
* @param options Options for configuring an `OpenAIClient`.
|
|
96
122
|
*/
|
|
97
|
-
public constructor(options: OpenAIEmbeddingsOptions|AzureOpenAIEmbeddingsOptions) {
|
|
123
|
+
public constructor(options: OSSEmbeddingsOptions|OpenAIEmbeddingsOptions|AzureOpenAIEmbeddingsOptions) {
|
|
98
124
|
// Check for azure config
|
|
99
125
|
if ((options as AzureOpenAIEmbeddingsOptions).azureApiKey) {
|
|
100
|
-
this.
|
|
126
|
+
this._clientType = ClientType.AzureOpenAI;
|
|
101
127
|
this.options = Object.assign({
|
|
102
128
|
retryPolicy: [2000, 5000],
|
|
103
129
|
azureApiVersion: '2023-05-15',
|
|
@@ -114,8 +140,13 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
114
140
|
}
|
|
115
141
|
|
|
116
142
|
this.options.azureEndpoint = endpoint;
|
|
143
|
+
} else if ((options as OSSEmbeddingsOptions).ossModel) {
|
|
144
|
+
this._clientType = ClientType.OSS;
|
|
145
|
+
this.options = Object.assign({
|
|
146
|
+
retryPolicy: [2000, 5000]
|
|
147
|
+
}, options) as OSSEmbeddingsOptions;
|
|
117
148
|
} else {
|
|
118
|
-
this.
|
|
149
|
+
this._clientType = ClientType.OpenAI;
|
|
119
150
|
this.options = Object.assign({
|
|
120
151
|
retryPolicy: [2000, 5000]
|
|
121
152
|
}, options) as OpenAIEmbeddingsOptions;
|
|
@@ -123,7 +154,7 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
123
154
|
|
|
124
155
|
// Create client
|
|
125
156
|
this._httpClient = axios.create({
|
|
126
|
-
validateStatus: (status) =>
|
|
157
|
+
validateStatus: (status) => status < 400 || status == 429
|
|
127
158
|
});
|
|
128
159
|
}
|
|
129
160
|
|
|
@@ -134,18 +165,30 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
134
165
|
* @returns A `EmbeddingsResponse` with a status and the generated embeddings or a message when an error occurs.
|
|
135
166
|
*/
|
|
136
167
|
public async createEmbeddings(inputs: string | string[]): Promise<EmbeddingsResponse> {
|
|
168
|
+
if (this.options.logRequests) {
|
|
169
|
+
console.log(Colorize.title('EMBEDDINGS REQUEST:'));
|
|
170
|
+
console.log(Colorize.output(inputs));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const startTime = Date.now();
|
|
137
174
|
const response = await this.createEmbeddingRequest({
|
|
138
175
|
input: inputs,
|
|
139
176
|
});
|
|
140
177
|
|
|
178
|
+
if (this.options.logRequests) {
|
|
179
|
+
console.log(Colorize.title('RESPONSE:'));
|
|
180
|
+
console.log(Colorize.value('status', response.status));
|
|
181
|
+
console.log(Colorize.value('duration', Date.now() - startTime, 'ms'));
|
|
182
|
+
console.log(Colorize.output(response.data));
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
141
186
|
// Process response
|
|
142
187
|
if (response.status < 300) {
|
|
143
|
-
|
|
144
|
-
return { status: 'success', output: data.sort((a, b) => a.index - b.index).map((item) => item.embedding), model, usage };
|
|
188
|
+
return { status: 'success', output: response.data.data.sort((a, b) => a.index - b.index).map((item) => item.embedding) };
|
|
145
189
|
} else if (response.status == 429) {
|
|
146
190
|
return { status: 'rate_limited', message: `The embeddings API returned a rate limit error.` }
|
|
147
191
|
} else {
|
|
148
|
-
console.log(inputs);
|
|
149
192
|
return { status: 'error', message: `The embeddings API returned an error status of ${response.status}: ${response.statusText}` };
|
|
150
193
|
}
|
|
151
194
|
}
|
|
@@ -154,10 +197,15 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
154
197
|
* @private
|
|
155
198
|
*/
|
|
156
199
|
protected createEmbeddingRequest(request: CreateEmbeddingRequest): Promise<AxiosResponse<CreateEmbeddingResponse>> {
|
|
157
|
-
if (this.
|
|
200
|
+
if (this._clientType == ClientType.AzureOpenAI) {
|
|
158
201
|
const options = this.options as AzureOpenAIEmbeddingsOptions;
|
|
159
202
|
const url = `${options.azureEndpoint}/openai/deployments/${options.azureDeployment}/embeddings?api-version=${options.azureApiVersion!}`;
|
|
160
203
|
return this.post(url, request);
|
|
204
|
+
} else if (this._clientType == ClientType.OSS) {
|
|
205
|
+
const options = this.options as OSSEmbeddingsOptions;
|
|
206
|
+
const url = `${options.ossEndpoint}/v1/embeddings`;
|
|
207
|
+
(request as OpenAICreateEmbeddingRequest).model = options.ossModel;
|
|
208
|
+
return this.post(url, request);
|
|
161
209
|
} else {
|
|
162
210
|
const options = this.options as OpenAIEmbeddingsOptions;
|
|
163
211
|
const url = `${options.endpoint ?? 'https://api.openai.com'}/v1/embeddings`;
|
|
@@ -183,10 +231,10 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
183
231
|
if (!requestConfig.headers['User-Agent']) {
|
|
184
232
|
requestConfig.headers['User-Agent'] = this.UserAgent;
|
|
185
233
|
}
|
|
186
|
-
if (this.
|
|
234
|
+
if (this._clientType == ClientType.AzureOpenAI) {
|
|
187
235
|
const options = this.options as AzureOpenAIEmbeddingsOptions;
|
|
188
236
|
requestConfig.headers['api-key'] = options.azureApiKey;
|
|
189
|
-
} else {
|
|
237
|
+
} else if (this._clientType == ClientType.OpenAI) {
|
|
190
238
|
const options = this.options as OpenAIEmbeddingsOptions;
|
|
191
239
|
requestConfig.headers['Authorization'] = `Bearer ${options.apiKey}`;
|
|
192
240
|
if (options.organization) {
|
|
@@ -207,3 +255,9 @@ export class OpenAIEmbeddings implements EmbeddingsModel {
|
|
|
207
255
|
}
|
|
208
256
|
}
|
|
209
257
|
}
|
|
258
|
+
|
|
259
|
+
enum ClientType {
|
|
260
|
+
OpenAI,
|
|
261
|
+
AzureOpenAI,
|
|
262
|
+
OSS
|
|
263
|
+
}
|
package/src/TextSplitter.ts
CHANGED
|
@@ -50,7 +50,7 @@ export class TextSplitter {
|
|
|
50
50
|
function getOverlapTokens(tokens?: number[]): number[] {
|
|
51
51
|
if (tokens != undefined) {
|
|
52
52
|
const len = tokens.length > that._config.chunkOverlap ? that._config.chunkOverlap : tokens.length;
|
|
53
|
-
return tokens.slice(
|
|
53
|
+
return tokens.slice(0, len);
|
|
54
54
|
} else {
|
|
55
55
|
return [];
|
|
56
56
|
}
|
|
@@ -133,9 +133,6 @@ export class TextSplitter {
|
|
|
133
133
|
// Update startPos
|
|
134
134
|
startPos = endPos + 1;
|
|
135
135
|
}
|
|
136
|
-
if (separator == '// LLM-REGION' && chunks.length > 1) {
|
|
137
|
-
console.log(this.combineChunks(chunks));
|
|
138
|
-
}
|
|
139
136
|
}
|
|
140
137
|
|
|
141
138
|
return this.combineChunks(chunks);
|
|
@@ -543,4 +540,4 @@ export class TextSplitter {
|
|
|
543
540
|
];
|
|
544
541
|
}
|
|
545
542
|
}
|
|
546
|
-
}
|
|
543
|
+
}
|