vectra 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/bin/vectra.js +3 -0
- package/lib/GPT3Tokenizer.d.ts +9 -0
- package/lib/GPT3Tokenizer.d.ts.map +1 -0
- package/lib/GPT3Tokenizer.js +17 -0
- package/lib/GPT3Tokenizer.js.map +1 -0
- package/lib/ItemSelector.d.ts +1 -1
- package/lib/ItemSelector.d.ts.map +1 -1
- package/lib/ItemSelector.js.map +1 -1
- package/lib/LocalDocument.d.ts +16 -0
- package/lib/LocalDocument.d.ts.map +1 -0
- package/lib/LocalDocument.js +99 -0
- package/lib/LocalDocument.js.map +1 -0
- package/lib/LocalDocumentIndex.d.ts +48 -0
- package/lib/LocalDocumentIndex.d.ts.map +1 -0
- package/lib/LocalDocumentIndex.js +367 -0
- package/lib/LocalDocumentIndex.js.map +1 -0
- package/lib/LocalDocumentResult.d.ts +12 -0
- package/lib/LocalDocumentResult.d.ts.map +1 -0
- package/lib/LocalDocumentResult.js +186 -0
- package/lib/LocalDocumentResult.js.map +1 -0
- package/lib/LocalIndex.d.ts +9 -63
- package/lib/LocalIndex.d.ts.map +1 -1
- package/lib/LocalIndex.js +14 -1
- package/lib/LocalIndex.js.map +1 -1
- package/lib/OpenAIEmbeddings.d.ts +98 -0
- package/lib/OpenAIEmbeddings.d.ts.map +1 -0
- package/lib/OpenAIEmbeddings.js +139 -0
- package/lib/OpenAIEmbeddings.js.map +1 -0
- package/lib/TextSplitter.d.ts +17 -0
- package/lib/TextSplitter.d.ts.map +1 -0
- package/lib/TextSplitter.js +460 -0
- package/lib/TextSplitter.js.map +1 -0
- package/lib/WebFetcher.d.ts +16 -0
- package/lib/WebFetcher.d.ts.map +1 -0
- package/lib/WebFetcher.js +144 -0
- package/lib/WebFetcher.js.map +1 -0
- package/lib/index.d.ts +8 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +13 -1
- package/lib/index.js.map +1 -1
- package/lib/internals/Colorize.d.ts +14 -0
- package/lib/internals/Colorize.d.ts.map +1 -0
- package/lib/internals/Colorize.js +64 -0
- package/lib/internals/Colorize.js.map +1 -0
- package/lib/internals/index.d.ts +3 -0
- package/lib/internals/index.d.ts.map +1 -0
- package/lib/internals/index.js +19 -0
- package/lib/internals/index.js.map +1 -0
- package/lib/internals/types.d.ts +42 -0
- package/lib/internals/types.d.ts.map +1 -0
- package/lib/internals/types.js +3 -0
- package/lib/internals/types.js.map +1 -0
- package/lib/types.d.ts +133 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +3 -0
- package/lib/types.js.map +1 -0
- package/lib/vectra-cli.d.ts +2 -0
- package/lib/vectra-cli.d.ts.map +1 -0
- package/lib/vectra-cli.js +277 -0
- package/lib/vectra-cli.js.map +1 -0
- package/package.json +21 -3
- package/src/GPT3Tokenizer.ts +15 -0
- package/src/ItemSelector.ts +9 -9
- package/src/LocalDocument.ts +70 -0
- package/src/LocalDocumentIndex.ts +355 -0
- package/src/LocalDocumentResult.ts +206 -0
- package/src/LocalIndex.ts +12 -78
- package/src/OpenAIEmbeddings.ts +205 -0
- package/src/TextSplitter.ts +480 -0
- package/src/WebFetcher.ts +128 -0
- package/src/index.ts +8 -0
- package/src/internals/Colorize.ts +64 -0
- package/src/internals/index.ts +2 -0
- package/src/internals/types.ts +46 -0
- package/src/types.ts +160 -0
- package/src/vectra-cli.ts +238 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { LocalDocument } from "./LocalDocument";
|
|
2
|
+
import { QueryResult, DocumentChunkMetadata, Tokenizer, DocumentTextSection } from "./types";
|
|
3
|
+
export declare class LocalDocumentResult extends LocalDocument {
|
|
4
|
+
private readonly _chunks;
|
|
5
|
+
private readonly _tokenizer;
|
|
6
|
+
private readonly _score;
|
|
7
|
+
constructor(folderPath: string, id: string, uri: string, chunks: QueryResult<DocumentChunkMetadata>[], tokenizer: Tokenizer);
|
|
8
|
+
get chunks(): QueryResult<DocumentChunkMetadata>[];
|
|
9
|
+
get score(): number;
|
|
10
|
+
renderSections(maxTokens: number, maxSections: number): Promise<DocumentTextSection[]>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=LocalDocumentResult.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalDocumentResult.d.ts","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,qBAAqB,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAE7F,qBAAa,mBAAoB,SAAQ,aAAa;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEb,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAAE,SAAS,EAAE,SAAS;IAWlI,IAAW,MAAM,IAAI,WAAW,CAAC,qBAAqB,CAAC,EAAE,CAExD;IAED,IAAW,KAAK,IAAI,MAAM,CAEzB;IAEY,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;CAmKtG"}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.LocalDocumentResult = void 0;
|
|
13
|
+
const LocalDocument_1 = require("./LocalDocument");
|
|
14
|
+
class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
15
|
+
constructor(folderPath, id, uri, chunks, tokenizer) {
|
|
16
|
+
super(folderPath, id, uri);
|
|
17
|
+
this._chunks = chunks;
|
|
18
|
+
this._tokenizer = tokenizer;
|
|
19
|
+
// Compute average score
|
|
20
|
+
let score = 0;
|
|
21
|
+
this._chunks.forEach(chunk => score += chunk.score);
|
|
22
|
+
this._score = score / this._chunks.length;
|
|
23
|
+
}
|
|
24
|
+
get chunks() {
|
|
25
|
+
return this._chunks;
|
|
26
|
+
}
|
|
27
|
+
get score() {
|
|
28
|
+
return this._score;
|
|
29
|
+
}
|
|
30
|
+
renderSections(maxTokens, maxSections) {
|
|
31
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
32
|
+
// Load text from disk
|
|
33
|
+
const text = yield this.loadText();
|
|
34
|
+
// First check to see if the entire document is less than maxTokens
|
|
35
|
+
const tokens = this._tokenizer.encode(text);
|
|
36
|
+
if (tokens.length < maxTokens) {
|
|
37
|
+
return [{
|
|
38
|
+
text,
|
|
39
|
+
tokenCount: tokens.length,
|
|
40
|
+
score: 1.0
|
|
41
|
+
}];
|
|
42
|
+
}
|
|
43
|
+
// Otherwise, we need to split the document into sections
|
|
44
|
+
// - Add each chunk to a temp array and filter out any chunk that's longer then maxTokens.
|
|
45
|
+
// - Sort the array by startPos to arrange chunks in document order.
|
|
46
|
+
// - Generate a new array of sections by combining chunks until the maxTokens is reached for each section.
|
|
47
|
+
// - Generate an aggregate score for each section by averaging the score of each chunk in the section.
|
|
48
|
+
// - Sort the sections by score and limit to maxSections.
|
|
49
|
+
// - For each remaining section combine adjacent chunks of text.
|
|
50
|
+
// - Dynamically add overlapping chunks of text to each section until the maxTokens is reached.
|
|
51
|
+
const chunks = this._chunks.map(chunk => {
|
|
52
|
+
const startPos = chunk.item.metadata.startPos;
|
|
53
|
+
const endPos = chunk.item.metadata.endPos;
|
|
54
|
+
const chunkText = text.substring(startPos, endPos + 1);
|
|
55
|
+
return {
|
|
56
|
+
text: chunkText,
|
|
57
|
+
startPos,
|
|
58
|
+
endPos,
|
|
59
|
+
score: chunk.score,
|
|
60
|
+
tokenCount: this._tokenizer.encode(chunkText).length
|
|
61
|
+
};
|
|
62
|
+
}).filter(chunk => chunk.tokenCount <= maxTokens).sort((a, b) => a.startPos - b.startPos);
|
|
63
|
+
// Check for no chunks
|
|
64
|
+
if (chunks.length === 0) {
|
|
65
|
+
// Take the top chunk and return a subset of its text
|
|
66
|
+
const topChunk = this._chunks[0];
|
|
67
|
+
const startPos = topChunk.item.metadata.startPos;
|
|
68
|
+
const endPos = topChunk.item.metadata.endPos;
|
|
69
|
+
const chunkText = text.substring(startPos, endPos + 1);
|
|
70
|
+
const tokens = this._tokenizer.encode(chunkText);
|
|
71
|
+
return [{
|
|
72
|
+
text: this._tokenizer.decode(tokens.slice(0, maxTokens)),
|
|
73
|
+
tokenCount: maxTokens,
|
|
74
|
+
score: topChunk.score
|
|
75
|
+
}];
|
|
76
|
+
}
|
|
77
|
+
// Generate sections
|
|
78
|
+
const sections = [{
|
|
79
|
+
chunks: [],
|
|
80
|
+
score: 0,
|
|
81
|
+
tokenCount: 0
|
|
82
|
+
}];
|
|
83
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
84
|
+
const chunk = chunks[i];
|
|
85
|
+
let section = sections[sections.length - 1];
|
|
86
|
+
if (section.tokenCount + chunk.tokenCount > maxTokens) {
|
|
87
|
+
sections.push({
|
|
88
|
+
chunks: [],
|
|
89
|
+
score: 0,
|
|
90
|
+
tokenCount: 0
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
sections[sections.length - 1].chunks.push(chunk);
|
|
94
|
+
sections[sections.length - 1].score += chunk.score;
|
|
95
|
+
sections[sections.length - 1].tokenCount += chunk.tokenCount;
|
|
96
|
+
}
|
|
97
|
+
// Normalize section scores
|
|
98
|
+
sections.forEach(section => section.score /= section.chunks.length);
|
|
99
|
+
// Sort sections by score and limit to maxSections
|
|
100
|
+
sections.sort((a, b) => b.score - a.score);
|
|
101
|
+
if (sections.length > maxSections) {
|
|
102
|
+
sections.splice(maxSections, sections.length - maxSections);
|
|
103
|
+
}
|
|
104
|
+
// Combine adjacent chunks of text
|
|
105
|
+
sections.forEach(section => {
|
|
106
|
+
for (let i = 0; i < section.chunks.length - 1; i++) {
|
|
107
|
+
const chunk = section.chunks[i];
|
|
108
|
+
const nextChunk = section.chunks[i + 1];
|
|
109
|
+
if (chunk.endPos + 1 === nextChunk.startPos) {
|
|
110
|
+
chunk.text += nextChunk.text;
|
|
111
|
+
chunk.endPos = nextChunk.endPos;
|
|
112
|
+
chunk.tokenCount += nextChunk.tokenCount;
|
|
113
|
+
section.chunks.splice(i + 1, 1);
|
|
114
|
+
i--;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
// Add overlapping chunks of text to each section until the maxTokens is reached
|
|
119
|
+
const connector = {
|
|
120
|
+
text: '\n\n...\n\n',
|
|
121
|
+
startPos: -1,
|
|
122
|
+
endPos: -1,
|
|
123
|
+
score: 0,
|
|
124
|
+
tokenCount: this._tokenizer.encode('\n\n...\n\n').length
|
|
125
|
+
};
|
|
126
|
+
sections.forEach(section => {
|
|
127
|
+
// Insert connectors between chunks
|
|
128
|
+
if (section.chunks.length > 1) {
|
|
129
|
+
for (let i = 0; i < section.chunks.length - 1; i++) {
|
|
130
|
+
section.chunks.splice(i + 1, 0, connector);
|
|
131
|
+
section.tokenCount += connector.tokenCount;
|
|
132
|
+
i++;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
// Add chunks to beginning and end of the section until maxTokens is reached
|
|
136
|
+
let budget = maxTokens - section.tokenCount;
|
|
137
|
+
if (budget > 40) {
|
|
138
|
+
const sectionStart = section.chunks[0].startPos;
|
|
139
|
+
const sectionEnd = section.chunks[section.chunks.length - 1].endPos;
|
|
140
|
+
if (sectionStart > 0) {
|
|
141
|
+
const beforeTex = text.substring(0, section.chunks[0].startPos);
|
|
142
|
+
const beforeTokens = this._tokenizer.encode(beforeTex);
|
|
143
|
+
const beforeBudget = sectionEnd < text.length - 1 ? Math.min(beforeTokens.length, Math.ceil(budget / 2)) : Math.min(beforeTokens.length, budget);
|
|
144
|
+
const chunk = {
|
|
145
|
+
text: this._tokenizer.decode(beforeTokens.slice(-beforeBudget)),
|
|
146
|
+
startPos: sectionStart - beforeBudget,
|
|
147
|
+
endPos: sectionStart - 1,
|
|
148
|
+
score: 0,
|
|
149
|
+
tokenCount: beforeBudget
|
|
150
|
+
};
|
|
151
|
+
section.chunks.unshift(chunk);
|
|
152
|
+
section.tokenCount += chunk.tokenCount;
|
|
153
|
+
budget -= chunk.tokenCount;
|
|
154
|
+
}
|
|
155
|
+
if (sectionEnd < text.length - 1) {
|
|
156
|
+
const afterText = text.substring(sectionEnd + 1);
|
|
157
|
+
const afterTokens = this._tokenizer.encode(afterText);
|
|
158
|
+
const afterBudget = Math.min(afterTokens.length, budget);
|
|
159
|
+
const chunk = {
|
|
160
|
+
text: this._tokenizer.decode(afterTokens.slice(0, afterBudget)),
|
|
161
|
+
startPos: sectionEnd + 1,
|
|
162
|
+
endPos: sectionEnd + afterBudget,
|
|
163
|
+
score: 0,
|
|
164
|
+
tokenCount: afterBudget
|
|
165
|
+
};
|
|
166
|
+
section.chunks.push(chunk);
|
|
167
|
+
section.tokenCount += chunk.tokenCount;
|
|
168
|
+
budget -= chunk.tokenCount;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
// Return final rendered sections
|
|
173
|
+
return sections.map(section => {
|
|
174
|
+
let text = '';
|
|
175
|
+
section.chunks.forEach(chunk => text += chunk.text);
|
|
176
|
+
return {
|
|
177
|
+
text: text,
|
|
178
|
+
tokenCount: section.tokenCount,
|
|
179
|
+
score: section.score
|
|
180
|
+
};
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
exports.LocalDocumentResult = LocalDocumentResult;
|
|
186
|
+
//# sourceMappingURL=LocalDocumentResult.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;AAGhD,MAAa,mBAAoB,SAAQ,6BAAa;IAKlD,YAAmB,UAAkB,EAAE,EAAU,EAAE,GAAW,EAAE,MAA4C,EAAE,SAAoB;QAC9H,KAAK,CAAC,UAAU,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,wBAAwB;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEY,cAAc,CAAC,SAAiB,EAAE,WAAmB;;YAC9D,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,mEAAmE;YACnE,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE;gBAC3B,OAAO,CAAC;wBACJ,IAAI;wBACJ,UAAU,EAAE,MAAM,CAAC,MAAM;wBACzB,KAAK,EAAE,GAAG;qBACb,CAAC,CAAC;aACN;YAED,yDAAyD;YACzD,0FAA0F;YAC1F,oEAAoE;YACpE,0GAA0G;YAC1G,sGAAsG;YACtG,yDAAyD;YACzD,gEAAgE;YAChE,+FAA+F;YAC/F,MAAM,MAAM,GAAmB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gBACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,OAAO;oBACH,IAAI,EAAE,SAAS;oBACf,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM;iBACvD,CAAC;YACN,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE1F,sBAAsB;YACtB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBACrB,qDAAqD;gBACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,OAAO,CAAC;wBACJ,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxD,UAAU,EAAE,SAAS;wBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACxB,CAAC,CAAC;aACN;YAED,oBAAoB;YACpB,MAAM,QAAQ,GAAc,CAAC;oBACzB,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,CAAC;iBAChB,CAAC,CAAC;YACH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE;oBACnD,QAAQ,CAAC,IAAI,CAAC;wBACV,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC,CAAC;iBACN;gBACD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACjD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBACnD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;aAChE;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,kDAAkD;YAClD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW,EAAE;gBAC/B,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;aAC/D;YAED,kCAAkC;YAClC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;oBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE;wBACzC,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC;wBAC7B,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;wBAChC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBACzC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChC,CAAC,EAAE,CAAC;qBACP;iBACJ;YACL,CAAC,CAAC,CAAC;YAEH,gFAAgF;YAChF,MAAM,SAAS,GAAiB;gBAC5B,IAAI,EAAE,aAAa;gBACnB,QAAQ,EAAE,CAAC,CAAC;gBACZ,MAAM,EAAE,CAAC,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM;aAC3D,CAAC;YACF,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,mCAAmC;gBACnC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;wBAChD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;wBAC3C,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBAC3C,CAAC,EAAE,CAAC;qBACP;iBACJ;gBAED,4EAA4E;gBAC5E,IAAI,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;gBAC5C,IAAI,MAAM,GAAG,EAAE,EAAE;oBACb,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;oBAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;oBACpE,IAAI,YAAY,GAAG,CAAC,EAAE;wBAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;wBAChE,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;wBACvD,MAAM,YAAY,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBAC/I,MAAM,KAAK,GAAiB;4BACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;4BAC/D,QAAQ,EAAE,YAAY,GAAG,YAAY;4BACrC,MAAM,EAAE,YAAY,GAAG,CAAC;4BACxB,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,YAAY;yBAC3B,CAAC;wBACF,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBAC9B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;wBACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;qBAC9B;oBAED,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;wBAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;wBACjD,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;wBACtD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;wBACzD,MAAM,KAAK,GAAiB;4BACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;4BAC/D,QAAQ,EAAE,UAAU,GAAG,CAAC;4BACxB,MAAM,EAAE,UAAU,GAAG,WAAW;4BAChC,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,WAAW;yBAC1B,CAAC;wBACF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC3B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;wBACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;qBAC9B;iBACJ;YACL,CAAC,CAAC,CAAC;YAEH,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACvB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;CACJ;AA3LD,kDA2LC"}
|
package/lib/LocalIndex.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { IndexItem, IndexStats, MetadataFilter, MetadataTypes, QueryResult } from './types';
|
|
1
2
|
export interface CreateIndexConfig {
|
|
2
3
|
version: number;
|
|
3
4
|
deleteIfExists?: boolean;
|
|
@@ -5,68 +6,6 @@ export interface CreateIndexConfig {
|
|
|
5
6
|
indexed?: string[];
|
|
6
7
|
};
|
|
7
8
|
}
|
|
8
|
-
export interface IndexStats {
|
|
9
|
-
version: number;
|
|
10
|
-
metadata_config: {
|
|
11
|
-
indexed?: string[];
|
|
12
|
-
};
|
|
13
|
-
items: number;
|
|
14
|
-
}
|
|
15
|
-
export interface IndexItem<TMetadata = Record<string, MetadataTypes>> {
|
|
16
|
-
id: string;
|
|
17
|
-
metadata: TMetadata;
|
|
18
|
-
vector: number[];
|
|
19
|
-
norm: number;
|
|
20
|
-
metadataFile?: string;
|
|
21
|
-
}
|
|
22
|
-
export interface QueryResult<TMetadata = Record<string, MetadataTypes>> {
|
|
23
|
-
item: IndexItem<TMetadata>;
|
|
24
|
-
score: number;
|
|
25
|
-
}
|
|
26
|
-
export interface MetadataFilter {
|
|
27
|
-
[key: string]: MetadataTypes | MetadataFilter | (number | string)[] | MetadataFilter[];
|
|
28
|
-
/**
|
|
29
|
-
* Equal to (number, string, boolean)
|
|
30
|
-
*/
|
|
31
|
-
'$eq': number | string | boolean;
|
|
32
|
-
/**
|
|
33
|
-
* Not equal to (number, string, boolean)
|
|
34
|
-
*/
|
|
35
|
-
'$ne': number | string | boolean;
|
|
36
|
-
/**
|
|
37
|
-
* Greater than (number)
|
|
38
|
-
*/
|
|
39
|
-
'$gt': number;
|
|
40
|
-
/**
|
|
41
|
-
* Greater than or equal to (number)
|
|
42
|
-
*/
|
|
43
|
-
'$gte': number;
|
|
44
|
-
/**
|
|
45
|
-
* Less than (number)
|
|
46
|
-
*/
|
|
47
|
-
'$lt': number;
|
|
48
|
-
/**
|
|
49
|
-
* Less than or equal to (number)
|
|
50
|
-
*/
|
|
51
|
-
'$lte': number;
|
|
52
|
-
/**
|
|
53
|
-
* In array (string or number)
|
|
54
|
-
*/
|
|
55
|
-
'$in': (number | string)[];
|
|
56
|
-
/**
|
|
57
|
-
* Not in array (string or number)
|
|
58
|
-
*/
|
|
59
|
-
'$nin': (number | string)[];
|
|
60
|
-
/**
|
|
61
|
-
* AND (MetadataFilter[])
|
|
62
|
-
*/
|
|
63
|
-
'$and': MetadataFilter[];
|
|
64
|
-
/**
|
|
65
|
-
* OR (MetadataFilter[])
|
|
66
|
-
*/
|
|
67
|
-
'$or': MetadataFilter[];
|
|
68
|
-
}
|
|
69
|
-
export declare type MetadataTypes = number | string | boolean;
|
|
70
9
|
/**
|
|
71
10
|
* Local vector index instance.
|
|
72
11
|
* @remarks
|
|
@@ -82,6 +21,10 @@ export declare class LocalIndex {
|
|
|
82
21
|
* @param folderPath - Path to the index folder
|
|
83
22
|
*/
|
|
84
23
|
constructor(folderPath: string);
|
|
24
|
+
/**
|
|
25
|
+
* Path to the index folder.
|
|
26
|
+
*/
|
|
27
|
+
get folderPath(): string;
|
|
85
28
|
/**
|
|
86
29
|
* Begins an update to the index.
|
|
87
30
|
* @remarks
|
|
@@ -178,7 +121,10 @@ export declare class LocalIndex {
|
|
|
178
121
|
* @returns Upserted item
|
|
179
122
|
*/
|
|
180
123
|
upsertItem<TMetadata = Record<string, MetadataTypes>>(item: Partial<IndexItem<TMetadata>>): Promise<IndexItem<TMetadata>>;
|
|
181
|
-
|
|
124
|
+
/**
|
|
125
|
+
* Ensures that the index has been loaded into memory.
|
|
126
|
+
*/
|
|
127
|
+
protected loadIndexData(): Promise<void>;
|
|
182
128
|
private addItemToUpdate;
|
|
183
129
|
}
|
|
184
130
|
//# sourceMappingURL=LocalIndex.d.ts.map
|
package/lib/LocalIndex.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE5F,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED;;;;;GAKG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B;;;OAGG;gBACgB,UAAU,EAAE,MAAM;IAIrC;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;IAKrH;;;;;;;OAOG;IACU,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAWrI;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,KAAK,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;IAKnG;;;;;;OAMG;IACU,mBAAmB,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;IAKnI;;;;;;;;;OASG;IACU,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;IAyC7J;;;;;;;OAOG;IACU,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAWrI;;OAEG;cACa,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAahC,eAAe;CA8DhC"}
|
package/lib/LocalIndex.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
|
@@ -47,6 +51,12 @@ class LocalIndex {
|
|
|
47
51
|
constructor(folderPath) {
|
|
48
52
|
this._folderPath = folderPath;
|
|
49
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Path to the index folder.
|
|
56
|
+
*/
|
|
57
|
+
get folderPath() {
|
|
58
|
+
return this._folderPath;
|
|
59
|
+
}
|
|
50
60
|
/**
|
|
51
61
|
* Begins an update to the index.
|
|
52
62
|
* @remarks
|
|
@@ -312,6 +322,9 @@ class LocalIndex {
|
|
|
312
322
|
}
|
|
313
323
|
});
|
|
314
324
|
}
|
|
325
|
+
/**
|
|
326
|
+
* Ensures that the index has been loaded into memory.
|
|
327
|
+
*/
|
|
315
328
|
loadIndexData() {
|
|
316
329
|
return __awaiter(this, void 0, void 0, function* () {
|
|
317
330
|
if (this._data) {
|
package/lib/LocalIndex.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalIndex.js","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"LocalIndex.js","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,2CAA6B;AAC7B,+BAA0B;AAC1B,iDAA8C;AAW9C;;;;;GAKG;AACH,MAAa,UAAU;IAKnB;;;OAGG;IACH,YAAmB,UAAkB;QACjC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACU,WAAW;;YACpB,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;aACjD;YAED,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC;KAAA;IAED;;;;OAIG;IACI,YAAY;QACf,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACU,WAAW,CAAC,SAA4B,EAAC,OAAO,EAAE,CAAC,EAAC;;;YAC7D,mBAAmB;YACnB,IAAI,MAAM,IAAI,CAAC,cAAc,EAAE,EAAE;gBAC7B,IAAI,MAAM,CAAC,cAAc,EAAE;oBACvB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;iBAC5B;qBAAM;oBACH,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;iBAC3C;aACJ;YAED,IAAI;gBACA,0BAA0B;gBAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAEtD,6BAA6B;gBAC7B,IAAI,CAAC,KAAK,GAAG;oBACT,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,eAAe,EAAE,MAAA,MAAM,CAAC,eAAe,mCAAI,EAAE;oBAC7C,KAAK,EAAE,EAAE;iBACZ,CAAC;gBACF,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;aAC7F;YAAC,OAAO,GAAY,EAAE;gBACnB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;aAC3C;;KACJ;IAED;;;;OAIG;IACI,WAAW;QACd,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,OAAO,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE;YAC3B,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,CAAC;SAChB,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACU,UAAU,CAAC,EAAU;;YAC9B,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC7D,IAAI,KAAK,IAAI,CAAC,EAAE;oBACZ,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;iBACvC;aACJ;iBAAM;gBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC9D,IAAI,KAAK,IAAI,CAAC,EAAE;oBACZ,IAAI,CAAC,OAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;iBACxC;gBACD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;aAC1B;QACL,CAAC;KAAA;IAED;;;;OAIG;IACU,SAAS;;YAClB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBACf,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;aAC5C;YAED,IAAI;gBACA,aAAa;gBACb,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC5F,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;gBAC1B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;aAC5B;YAAC,OAAM,GAAY,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,uBAAwB,GAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;aACrE;QACL,CAAC;KAAA;IAED;;;OAGG;IACU,aAAa;;YACtB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,KAAM,CAAC,OAAO;gBAC5B,eAAe,EAAE,IAAI,CAAC,KAAM,CAAC,eAAe;gBAC5C,KAAK,EAAE,IAAI,CAAC,KAAM,CAAC,KAAK,CAAC,MAAM;aAClC,CAAC;QACN,CAAC;KAAA;IAED;;;;OAIG;IACU,OAAO,CAA2C,EAAU;;YACrE,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,KAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAoB,CAAC;QACvE,CAAC;KAAA;IAED;;;;;;;OAOG;IACU,UAAU,CAA2C,IAAmC;;YACjG,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAQ,CAAC;aACxD;iBAAM;gBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACvD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO,OAAc,CAAC;aACzB;QACL,CAAC;KAAA;IAED;;OAEG;IACU,cAAc;;YACvB,IAAI;gBACA,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;gBAC3D,OAAO,IAAI,CAAC;aACf;YAAC,OAAO,GAAY,EAAE;gBACnB,OAAO,KAAK,CAAC;aAChB;QACL,CAAC;KAAA;IAED;;;;;;OAMG;IACU,SAAS;;YAClB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,KAAM,CAAC,KAAK,CAAC,KAAK,EAAS,CAAC;QAC5C,CAAC;KAAA;IAED;;;;;;OAMG;IACU,mBAAmB,CAA2C,MAAsB;;YAC7F,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,KAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,2BAAY,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAQ,CAAC;QACzF,CAAC;KAAA;IAED;;;;;;;;;OASG;IACU,UAAU,CAA2C,MAAgB,EAAE,IAAY,EAAE,MAAuB;;YACrH,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAE3B,eAAe;YACf,IAAI,KAAK,GAAG,IAAI,CAAC,KAAM,CAAC,KAAK,CAAC;YAC9B,IAAI,MAAM,EAAE;gBACR,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,2BAAY,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;aACtE;YAED,sBAAsB;YACtB,MAAM,IAAI,GAAG,2BAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,SAAS,GAA0C,EAAE,CAAC;YAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,QAAQ,GAAG,2BAAY,CAAC,0BAA0B,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/F,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;aACpD;YAED,8BAA8B;YAC9B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAElD,aAAa;YACb,MAAM,GAAG,GAA6B,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACnE,OAAO;oBACH,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAQ;oBAC9C,KAAK,EAAE,CAAC,CAAC,QAAQ;iBACpB,CAAC;YACN,CAAC,CAAC,CAAC;YAEH,yBAAyB;YACzB,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE;gBACpB,IAAI,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;oBACxB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACzE,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;oBACjD,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;iBACxD;aACJ;YAED,OAAO,GAAG,CAAC;QACf,CAAC;KAAA;IAED;;;;;;;OAOG;IACU,UAAU,CAA2C,IAAmC;;YACjG,IAAI,IAAI,CAAC,OAAO,EAAE;gBACd,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAQ,CAAC;aACzD;iBAAM;gBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACxD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO,OAAc,CAAC;aACzB;QACL,CAAC;KAAA;IAED;;OAEG;IACa,aAAa;;YACzB,IAAI,IAAI,CAAC,KAAK,EAAE;gBACZ,OAAO;aACV;YAED,IAAI,CAAC,CAAA,MAAM,IAAI,CAAC,cAAc,EAAE,CAAA,EAAE;gBAC9B,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;aAC3C;YAED,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;YAC1E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC7C,CAAC;KAAA;IAEa,eAAe,CAAC,IAA6B,EAAE,MAAe;;;YACxE,4BAA4B;YAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;gBACd,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;aACzC;YAED,gBAAgB;YAChB,MAAM,EAAE,GAAG,MAAA,IAAI,CAAC,EAAE,mCAAI,IAAA,SAAE,GAAE,CAAC;YAC3B,IAAI,MAAM,EAAE;gBACR,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5D,IAAI,QAAQ,EAAE;oBACV,MAAM,IAAI,KAAK,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;iBACxD;aACJ;YAED,6BAA6B;YAC7B,IAAI,QAAQ,GAAuB,EAAE,CAAC;YACtC,IAAI,YAAgC,CAAC;YACrC,IAAI,IAAI,CAAC,OAAQ,CAAC,eAAe,CAAC,OAAO,IAAI,IAAI,CAAC,OAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE;gBAC5G,6BAA6B;gBAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAQ,CAAC,eAAe,CAAC,OAAO,EAAE;oBACrD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;wBACrC,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;qBACtC;iBACJ;gBAED,kCAAkC;gBAClC,YAAY,GAAG,GAAG,SAAE,OAAO,CAAC;gBAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;gBAC/D,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;aACnE;iBAAM,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACtB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;aAC5B;YAED,kBAAkB;YAClB,MAAM,OAAO,GAAc;gBACvB,EAAE,EAAE,EAAE;gBACN,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,IAAI,EAAE,2BAAY,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;aAC5C,CAAC;YACF,IAAI,YAAY,EAAE;gBACd,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;aACvC;YAED,oBAAoB;YACpB,IAAI,CAAC,MAAM,EAAE;gBACT,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5D,IAAI,QAAQ,EAAE;oBACV,QAAQ,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;oBACrC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBACjC,QAAQ,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;oBAC7C,OAAO,QAAQ,CAAC;iBACnB;qBAAM;oBACH,IAAI,CAAC,OAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAClC,OAAO,OAAO,CAAC;iBAClB;aACJ;iBAAM;gBACH,IAAI,CAAC,OAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO,OAAO,CAAC;aAClB;;KACJ;CACJ;AAnWD,gCAmWC"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { AxiosResponse, AxiosRequestConfig } from 'axios';
|
|
2
|
+
import { EmbeddingsModel, EmbeddingsResponse } from "./types";
|
|
3
|
+
import { CreateEmbeddingRequest, CreateEmbeddingResponse } from "./internals";
|
|
4
|
+
export interface BaseOpenAIEmbeddingsOptions {
|
|
5
|
+
/**
|
|
6
|
+
* Optional. Retry policy to use when calling the OpenAI API.
|
|
7
|
+
* @remarks
|
|
8
|
+
* The default retry policy is `[2000, 5000]` which means that the first retry will be after
|
|
9
|
+
* 2 seconds and the second retry will be after 5 seconds.
|
|
10
|
+
*/
|
|
11
|
+
retryPolicy?: number[];
|
|
12
|
+
/**
|
|
13
|
+
* Optional. Request options to use when calling the OpenAI API.
|
|
14
|
+
*/
|
|
15
|
+
requestConfig?: AxiosRequestConfig;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Options for configuring an `OpenAIEmbeddings` to generate embeddings using an OpenAI hosted model.
|
|
19
|
+
*/
|
|
20
|
+
export interface OpenAIEmbeddingsOptions extends BaseOpenAIEmbeddingsOptions {
|
|
21
|
+
/**
|
|
22
|
+
* API key to use when calling the OpenAI API.
|
|
23
|
+
* @remarks
|
|
24
|
+
* A new API key can be created at https://platform.openai.com/account/api-keys.
|
|
25
|
+
*/
|
|
26
|
+
apiKey: string;
|
|
27
|
+
/**
|
|
28
|
+
* Model to use for completion.
|
|
29
|
+
* @remarks
|
|
30
|
+
* For Azure OpenAI this is the name of the deployment to use.
|
|
31
|
+
*/
|
|
32
|
+
model: string;
|
|
33
|
+
/**
|
|
34
|
+
* Optional. Organization to use when calling the OpenAI API.
|
|
35
|
+
*/
|
|
36
|
+
organization?: string;
|
|
37
|
+
/**
|
|
38
|
+
* Optional. Endpoint to use when calling the OpenAI API.
|
|
39
|
+
* @remarks
|
|
40
|
+
* For Azure OpenAI this is the deployment endpoint.
|
|
41
|
+
*/
|
|
42
|
+
endpoint?: string;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Options for configuring an `OpenAIEmbeddings` to generate embeddings using an Azure OpenAI hosted model.
|
|
46
|
+
*/
|
|
47
|
+
export interface AzureOpenAIEmbeddingsOptions extends BaseOpenAIEmbeddingsOptions {
|
|
48
|
+
/**
|
|
49
|
+
* API key to use when making requests to Azure OpenAI.
|
|
50
|
+
*/
|
|
51
|
+
azureApiKey: string;
|
|
52
|
+
/**
|
|
53
|
+
* Deployment endpoint to use.
|
|
54
|
+
*/
|
|
55
|
+
azureEndpoint: string;
|
|
56
|
+
/**
|
|
57
|
+
* Name of the Azure OpenAI deployment (model) to use.
|
|
58
|
+
*/
|
|
59
|
+
azureDeployment: string;
|
|
60
|
+
/**
|
|
61
|
+
* Optional. Version of the API being called. Defaults to `2023-05-15`.
|
|
62
|
+
*/
|
|
63
|
+
azureApiVersion?: string;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* A `PromptCompletionModel` for calling OpenAI and Azure OpenAI hosted models.
|
|
67
|
+
* @remarks
|
|
68
|
+
*/
|
|
69
|
+
export declare class OpenAIEmbeddings implements EmbeddingsModel {
|
|
70
|
+
private readonly _httpClient;
|
|
71
|
+
private readonly _useAzure;
|
|
72
|
+
private readonly UserAgent;
|
|
73
|
+
/**
|
|
74
|
+
* Options the client was configured with.
|
|
75
|
+
*/
|
|
76
|
+
readonly options: OpenAIEmbeddingsOptions | AzureOpenAIEmbeddingsOptions;
|
|
77
|
+
/**
|
|
78
|
+
* Creates a new `OpenAIClient` instance.
|
|
79
|
+
* @param options Options for configuring an `OpenAIClient`.
|
|
80
|
+
*/
|
|
81
|
+
constructor(options: OpenAIEmbeddingsOptions | AzureOpenAIEmbeddingsOptions);
|
|
82
|
+
/**
|
|
83
|
+
* Creates embeddings for the given inputs using the OpenAI API.
|
|
84
|
+
* @param model Name of the model to use (or deployment for Azure).
|
|
85
|
+
* @param inputs Text inputs to create embeddings for.
|
|
86
|
+
* @returns A `EmbeddingsResponse` with a status and the generated embeddings or a message when an error occurs.
|
|
87
|
+
*/
|
|
88
|
+
createEmbeddings(inputs: string | string[]): Promise<EmbeddingsResponse>;
|
|
89
|
+
/**
|
|
90
|
+
* @private
|
|
91
|
+
*/
|
|
92
|
+
protected createEmbeddingRequest(request: CreateEmbeddingRequest): Promise<AxiosResponse<CreateEmbeddingResponse>>;
|
|
93
|
+
/**
|
|
94
|
+
* @private
|
|
95
|
+
*/
|
|
96
|
+
protected post<TData>(url: string, body: object, retryCount?: number): Promise<AxiosResponse<TData>>;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=OpenAIEmbeddings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddings.d.ts","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":"AAAA,OAAc,EAAiB,aAAa,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,uBAAuB,EAAgC,MAAM,aAAa,CAAC;AAE5G,MAAM,WAAW,2BAA2B;IACxC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,aAAa,CAAC,EAAE,kBAAkB,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,2BAA2B;IACxE;;;;OAIG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC7E;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;GAGG;AACH,qBAAa,gBAAiB,YAAW,eAAe;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgB;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IAEpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IAEzC;;OAEG;IACH,SAAgB,OAAO,EAAE,uBAAuB,GAAC,4BAA4B,CAAC;IAE9E;;;OAGG;gBACgB,OAAO,EAAE,uBAAuB,GAAC,4BAA4B;IAiChF;;;;;OAKG;IACU,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAerF;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC,uBAAuB,CAAC,CAAC;IAalH;;OAEG;cACa,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,SAAI,GAAG,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;CAqCxG"}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.OpenAIEmbeddings = void 0;
|
|
16
|
+
const axios_1 = __importDefault(require("axios"));
|
|
17
|
+
/**
|
|
18
|
+
* A `PromptCompletionModel` for calling OpenAI and Azure OpenAI hosted models.
|
|
19
|
+
* @remarks
|
|
20
|
+
*/
|
|
21
|
+
class OpenAIEmbeddings {
|
|
22
|
+
/**
|
|
23
|
+
* Creates a new `OpenAIClient` instance.
|
|
24
|
+
* @param options Options for configuring an `OpenAIClient`.
|
|
25
|
+
*/
|
|
26
|
+
constructor(options) {
|
|
27
|
+
this.UserAgent = 'AlphaWave';
|
|
28
|
+
// Check for azure config
|
|
29
|
+
if (options.azureApiKey) {
|
|
30
|
+
this._useAzure = true;
|
|
31
|
+
this.options = Object.assign({
|
|
32
|
+
retryPolicy: [2000, 5000],
|
|
33
|
+
azureApiVersion: '2023-05-15',
|
|
34
|
+
}, options);
|
|
35
|
+
// Cleanup and validate endpoint
|
|
36
|
+
let endpoint = this.options.azureEndpoint.trim();
|
|
37
|
+
if (endpoint.endsWith('/')) {
|
|
38
|
+
endpoint = endpoint.substring(0, endpoint.length - 1);
|
|
39
|
+
}
|
|
40
|
+
if (!endpoint.toLowerCase().startsWith('https://')) {
|
|
41
|
+
throw new Error(`Client created with an invalid endpoint of '${endpoint}'. The endpoint must be a valid HTTPS url.`);
|
|
42
|
+
}
|
|
43
|
+
this.options.azureEndpoint = endpoint;
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
this._useAzure = false;
|
|
47
|
+
this.options = Object.assign({
|
|
48
|
+
retryPolicy: [2000, 5000]
|
|
49
|
+
}, options);
|
|
50
|
+
}
|
|
51
|
+
// Create client
|
|
52
|
+
this._httpClient = axios_1.default.create({
|
|
53
|
+
validateStatus: (status) => status < 400 || status == 429
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Creates embeddings for the given inputs using the OpenAI API.
|
|
58
|
+
* @param model Name of the model to use (or deployment for Azure).
|
|
59
|
+
* @param inputs Text inputs to create embeddings for.
|
|
60
|
+
* @returns A `EmbeddingsResponse` with a status and the generated embeddings or a message when an error occurs.
|
|
61
|
+
*/
|
|
62
|
+
createEmbeddings(inputs) {
|
|
63
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
64
|
+
const response = yield this.createEmbeddingRequest({
|
|
65
|
+
input: inputs,
|
|
66
|
+
});
|
|
67
|
+
// Process response
|
|
68
|
+
if (response.status < 300) {
|
|
69
|
+
return { status: 'success', output: response.data.data.sort((a, b) => a.index - b.index).map((item) => item.embedding) };
|
|
70
|
+
}
|
|
71
|
+
else if (response.status == 429) {
|
|
72
|
+
return { status: 'rate_limited', message: `The embeddings API returned a rate limit error.` };
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
return { status: 'error', message: `The embeddings API returned an error status of ${response.status}: ${response.statusText}` };
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* @private
|
|
81
|
+
*/
|
|
82
|
+
createEmbeddingRequest(request) {
|
|
83
|
+
var _a;
|
|
84
|
+
if (this._useAzure) {
|
|
85
|
+
const options = this.options;
|
|
86
|
+
const url = `${options.azureEndpoint}/openai/deployments/${options.azureDeployment}/embeddings?api-version=${options.azureApiVersion}`;
|
|
87
|
+
return this.post(url, request);
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
const options = this.options;
|
|
91
|
+
const url = `${(_a = options.endpoint) !== null && _a !== void 0 ? _a : 'https://api.openai.com'}/v1/embeddings`;
|
|
92
|
+
request.model = options.model;
|
|
93
|
+
return this.post(url, request);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* @private
|
|
98
|
+
*/
|
|
99
|
+
post(url, body, retryCount = 0) {
|
|
100
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
101
|
+
// Initialize request config
|
|
102
|
+
const requestConfig = Object.assign({}, this.options.requestConfig);
|
|
103
|
+
// Initialize request headers
|
|
104
|
+
if (!requestConfig.headers) {
|
|
105
|
+
requestConfig.headers = {};
|
|
106
|
+
}
|
|
107
|
+
if (!requestConfig.headers['Content-Type']) {
|
|
108
|
+
requestConfig.headers['Content-Type'] = 'application/json';
|
|
109
|
+
}
|
|
110
|
+
if (!requestConfig.headers['User-Agent']) {
|
|
111
|
+
requestConfig.headers['User-Agent'] = this.UserAgent;
|
|
112
|
+
}
|
|
113
|
+
if (this._useAzure) {
|
|
114
|
+
const options = this.options;
|
|
115
|
+
requestConfig.headers['api-key'] = options.azureApiKey;
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
const options = this.options;
|
|
119
|
+
requestConfig.headers['Authorization'] = `Bearer ${options.apiKey}`;
|
|
120
|
+
if (options.organization) {
|
|
121
|
+
requestConfig.headers['OpenAI-Organization'] = options.organization;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
// Send request
|
|
125
|
+
const response = yield this._httpClient.post(url, body, requestConfig);
|
|
126
|
+
// Check for rate limit error
|
|
127
|
+
if (response.status == 429 && Array.isArray(this.options.retryPolicy) && retryCount < this.options.retryPolicy.length) {
|
|
128
|
+
const delay = this.options.retryPolicy[retryCount];
|
|
129
|
+
yield new Promise((resolve) => setTimeout(resolve, delay));
|
|
130
|
+
return this.post(url, body, retryCount + 1);
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
return response;
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
exports.OpenAIEmbeddings = OpenAIEmbeddings;
|
|
139
|
+
//# sourceMappingURL=OpenAIEmbeddings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddings.js","sourceRoot":"","sources":["../src/OpenAIEmbeddings.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kDAAgF;AA2EhF;;;GAGG;AACH,MAAa,gBAAgB;IAWzB;;;OAGG;IACH,YAAmB,OAA6D;QAX/D,cAAS,GAAG,WAAW,CAAC;QAYrC,yBAAyB;QACzB,IAAK,OAAwC,CAAC,WAAW,EAAE;YACvD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;gBACzB,eAAe,EAAE,YAAY;aAChC,EAAE,OAAO,CAAiC,CAAC;YAE5C,gCAAgC;YAChC,IAAI,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBACxB,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;aACzD;YAED,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CAAC,+CAA+C,QAAQ,4CAA4C,CAAC,CAAC;aACxH;YAED,IAAI,CAAC,OAAO,CAAC,aAAa,GAAG,QAAQ,CAAC;SACzC;aAAM;YACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;gBACzB,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;aAC5B,EAAE,OAAO,CAA4B,CAAC;SAC1C;QAED,gBAAgB;QAChB,IAAI,CAAC,WAAW,GAAG,eAAK,CAAC,MAAM,CAAC;YAC5B,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG;SAC5D,CAAC,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACU,gBAAgB,CAAC,MAAyB;;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC;gBAC/C,KAAK,EAAE,MAAM;aAChB,CAAC,CAAC;YAEH,mBAAmB;YACnB,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE;gBACvB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;aAC5H;iBAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE;gBAC/B,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,iDAAiD,EAAE,CAAA;aAChG;iBAAM;gBACH,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,kDAAkD,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;aACpI;QACL,CAAC;KAAA;IAED;;OAEG;IACO,sBAAsB,CAAC,OAA+B;;QAC5D,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;YAC7D,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,aAAa,uBAAuB,OAAO,CAAC,eAAe,2BAA2B,OAAO,CAAC,eAAgB,EAAE,CAAC;YACxI,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;aAAM;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;YACxD,MAAM,GAAG,GAAG,GAAG,MAAA,OAAO,CAAC,QAAQ,mCAAI,wBAAwB,gBAAgB,CAAC;YAC3E,OAAwC,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAChE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAClC;IACL,CAAC;IAED;;OAEG;IACa,IAAI,CAAQ,GAAW,EAAE,IAAY,EAAE,UAAU,GAAG,CAAC;;YACjE,4BAA4B;YAC5B,MAAM,aAAa,GAAuB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAExF,6BAA6B;YAC7B,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE;gBACxB,aAAa,CAAC,OAAO,GAAG,EAAE,CAAC;aAC9B;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;gBACxC,aAAa,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;aAC9D;YACD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;gBACtC,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;aACxD;YACD,IAAI,IAAI,CAAC,SAAS,EAAE;gBAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAuC,CAAC;gBAC7D,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC;aAC1D;iBAAM;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAkC,CAAC;gBACxD,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpE,IAAI,OAAO,CAAC,YAAY,EAAE;oBACtB,aAAa,CAAC,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;iBACvE;aACJ;YAED,eAAe;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC;YAEvE,6BAA6B;YAC7B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE;gBACnH,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBACnD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC3D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;aAC/C;iBAAM;gBACH,OAAO,QAAQ,CAAC;aACnB;QACL,CAAC;KAAA;CACJ;AA7HD,4CA6HC"}
|