vectra 0.12.2 → 0.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.draft.md +499 -0
- package/README.draft.outline.md +160 -0
- package/README.research.md +2159 -0
- package/bin/vectra.js +3 -0
- package/lib/FileFetcher.d.ts +5 -0
- package/lib/FileFetcher.d.ts.map +1 -0
- package/lib/FileFetcher.js +79 -0
- package/lib/FileFetcher.js.map +1 -0
- package/lib/GPT3Tokenizer.d.ts +9 -0
- package/lib/ItemSelector.d.ts +41 -0
- package/lib/ItemSelector.d.ts.map +1 -0
- package/lib/ItemSelector.js +168 -0
- package/lib/ItemSelector.js.map +1 -0
- package/lib/LocalDocument.d.ts +54 -0
- package/lib/LocalDocument.js +156 -0
- package/lib/LocalDocument.js.map +1 -0
- package/lib/LocalDocumentIndex.d.ts +132 -0
- package/lib/LocalDocumentIndex.js +456 -0
- package/lib/LocalDocumentIndex.js.map +1 -0
- package/lib/LocalDocumentResult.d.ts +45 -0
- package/lib/LocalDocumentResult.js +328 -0
- package/lib/LocalDocumentResult.js.map +1 -0
- package/lib/LocalIndex.d.ts +150 -0
- package/lib/LocalIndex.d.ts.map +1 -1
- package/lib/LocalIndex.js +515 -0
- package/lib/LocalIndex.js.map +1 -0
- package/lib/LocalIndex.spec.d.ts +2 -0
- package/lib/LocalIndex.spec.js +218 -7
- package/lib/LocalIndex.spec.js.map +1 -1
- package/lib/OpenAIEmbeddings.d.ts +126 -0
- package/lib/OpenAIEmbeddings.d.ts.map +1 -0
- package/lib/OpenAIEmbeddings.js +174 -0
- package/lib/OpenAIEmbeddings.js.map +1 -0
- package/lib/TextSplitter.d.ts +19 -0
- package/lib/TextSplitter.d.ts.map +1 -1
- package/lib/TextSplitter.js +457 -0
- package/lib/TextSplitter.js.map +1 -0
- package/lib/TextSplitter.spec.d.ts +2 -0
- package/lib/TextSplitter.spec.d.ts.map +1 -0
- package/lib/TextSplitter.spec.js +109 -0
- package/lib/TextSplitter.spec.js.map +1 -0
- package/lib/WebFetcher.d.ts +15 -0
- package/lib/WebFetcher.d.ts.map +1 -0
- package/lib/WebFetcher.js +234 -0
- package/lib/WebFetcher.js.map +1 -0
- package/lib/index.d.ts +12 -0
- package/lib/index.js +28 -0
- package/lib/index.js.map +1 -0
- package/lib/internals/Colorize.d.ts +14 -0
- package/lib/internals/Colorize.d.ts.map +1 -0
- package/lib/internals/Colorize.js +64 -0
- package/lib/internals/Colorize.js.map +1 -0
- package/lib/internals/index.d.ts +3 -0
- package/lib/internals/index.d.ts.map +1 -0
- package/lib/internals/index.js +19 -0
- package/lib/internals/index.js.map +1 -0
- package/lib/internals/types.d.ts +43 -0
- package/lib/internals/types.d.ts.map +1 -0
- package/lib/internals/types.js +3 -0
- package/lib/internals/types.js.map +1 -0
- package/lib/types.d.ts +146 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +3 -0
- package/lib/types.js.map +1 -0
- package/lib/vectra-cli.d.ts +2 -0
- package/lib/vectra-cli.js +323 -0
- package/lib/vectra-cli.js.map +1 -0
- package/package.json +3 -1
- package/src/LocalIndex.spec.ts +265 -8
- package/src/LocalIndex.ts +1 -0
- package/src/TextSplitter.spec.ts +87 -0
- package/src/TextSplitter.ts +459 -531
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
45
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
46
|
+
};
|
|
47
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
+
exports.WebFetcher = void 0;
|
|
49
|
+
const axios_1 = __importDefault(require("axios"));
|
|
50
|
+
const cheerio = __importStar(require("cheerio"));
|
|
51
|
+
const turndown_1 = __importDefault(require("turndown"));
|
|
52
|
+
const ALLOWED_CONTENT_TYPES = [
|
|
53
|
+
"text/html",
|
|
54
|
+
"application/json",
|
|
55
|
+
"application/xml",
|
|
56
|
+
"application/javascript",
|
|
57
|
+
"text/plain",
|
|
58
|
+
];
|
|
59
|
+
const DEFAULT_HEADERS = {
|
|
60
|
+
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
61
|
+
"Accept-Encoding": "gzip, deflate, sdch, br",
|
|
62
|
+
"Accept-Language": "en-US,en;q=0.8,ms;q=0.6",
|
|
63
|
+
"Alt-Used": "LEAVE-THIS-KEY-SET-BY-TOOL",
|
|
64
|
+
Connection: "keep-alive",
|
|
65
|
+
Host: "LEAVE-THIS-KEY-SET-BY-TOOL",
|
|
66
|
+
Referer: "https://www.google.com/",
|
|
67
|
+
"Sec-Fetch-Dest": "document",
|
|
68
|
+
"Sec-Fetch-Mode": "navigate",
|
|
69
|
+
"Sec-Fetch-Site": "cross-site",
|
|
70
|
+
"Upgrade-Insecure-Requests": "1",
|
|
71
|
+
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
|
|
72
|
+
};
|
|
73
|
+
class WebFetcher {
|
|
74
|
+
constructor(config) {
|
|
75
|
+
this._config = Object.assign({
|
|
76
|
+
htmlToMarkdown: true,
|
|
77
|
+
summarizeHtml: false,
|
|
78
|
+
}, config);
|
|
79
|
+
}
|
|
80
|
+
fetch(uri, onDocument) {
|
|
81
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
82
|
+
const httpClient = axios_1.default.create({
|
|
83
|
+
validateStatus: () => true,
|
|
84
|
+
});
|
|
85
|
+
// Clone headers to avoid mutating the original
|
|
86
|
+
const headers = Object.assign({}, DEFAULT_HEADERS, this._config.headers);
|
|
87
|
+
// get hostname from url
|
|
88
|
+
const host = new URL(uri).hostname;
|
|
89
|
+
headers['Host'] = host;
|
|
90
|
+
headers['Alt-Used'] = host;
|
|
91
|
+
// Fetch page and check for errors
|
|
92
|
+
const response = yield httpClient.get(uri, Object.assign({ headers }, this._config.requestConfig));
|
|
93
|
+
if (response.status >= 400) {
|
|
94
|
+
throw new Error(`Site returned an HTTP status of ${response.status}`);
|
|
95
|
+
}
|
|
96
|
+
// Check for valid content type
|
|
97
|
+
const contentType = response.headers['content-type'];
|
|
98
|
+
const contentTypeArray = contentType.split(';');
|
|
99
|
+
if (!contentTypeArray[0] || !ALLOWED_CONTENT_TYPES.includes(contentTypeArray[0])) {
|
|
100
|
+
throw new Error(`Site returned an invalid content type of ${contentType}`);
|
|
101
|
+
}
|
|
102
|
+
// Convert content type to doc type
|
|
103
|
+
const docType = contentTypeArray[0] != 'text/plain' ? contentTypeArray[0].split('/')[1] : undefined;
|
|
104
|
+
if (docType == 'html' && this._config.htmlToMarkdown) {
|
|
105
|
+
const text = this.htmlToMarkdown(response.data, uri);
|
|
106
|
+
return yield onDocument(uri, text, 'md');
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const text = response.data;
|
|
110
|
+
return yield onDocument(uri, text, docType);
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
htmlToMarkdown(html, baseUrl) {
|
|
115
|
+
var _a;
|
|
116
|
+
// Parse HTML and remove scripts
|
|
117
|
+
const $ = cheerio.load(html, { scriptingEnabled: true });
|
|
118
|
+
// Remove scripts and convert relative links to absolute
|
|
119
|
+
$('script').remove();
|
|
120
|
+
$('a').each((i, elem) => {
|
|
121
|
+
const $el = $(elem);
|
|
122
|
+
const href = $el.attr("href");
|
|
123
|
+
if (href && !href.startsWith("http")) {
|
|
124
|
+
// Try converting to an absolute link
|
|
125
|
+
try {
|
|
126
|
+
$el.attr("href", new URL(href, baseUrl).toString());
|
|
127
|
+
}
|
|
128
|
+
catch (_a) {
|
|
129
|
+
// Leave as is
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
// Convert to markdown
|
|
134
|
+
const body = (_a = $('body').html()) !== null && _a !== void 0 ? _a : '';
|
|
135
|
+
const turndownService = new turndown_1.default({
|
|
136
|
+
hr: '\n\n---\n\n',
|
|
137
|
+
});
|
|
138
|
+
convertTables(turndownService);
|
|
139
|
+
const md = turndownService.turndown(body);
|
|
140
|
+
// Remove any overly long header text
|
|
141
|
+
const contentStart = Math.min(md.indexOf('\n'), md.indexOf(' '));
|
|
142
|
+
if (contentStart > 64) {
|
|
143
|
+
return md.slice(contentStart);
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
return md;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
exports.WebFetcher = WebFetcher;
|
|
151
|
+
function convertTables(turndownService) {
|
|
152
|
+
turndownService.addRule('tableCell', {
|
|
153
|
+
filter: ['th', 'td'],
|
|
154
|
+
replacement: function (content, node) {
|
|
155
|
+
return cell(content, node);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
turndownService.addRule('tableRow', {
|
|
159
|
+
filter: 'tr',
|
|
160
|
+
replacement: function (content, node) {
|
|
161
|
+
var borderCells = '';
|
|
162
|
+
var alignMap = { left: ':--', right: '--:', center: ':-:' };
|
|
163
|
+
if (isHeadingRow(node)) {
|
|
164
|
+
for (var i = 0; i < node.childNodes.length; i++) {
|
|
165
|
+
var border = '---';
|
|
166
|
+
var align = (node.childNodes[i].getAttribute('align') || '').toLowerCase();
|
|
167
|
+
if (align)
|
|
168
|
+
border = alignMap[align] || border;
|
|
169
|
+
borderCells += cell(border, node.childNodes[i]);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return '\n' + content + (borderCells ? '\n' + borderCells : '');
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
turndownService.addRule('table', {
|
|
176
|
+
filter: ['table'],
|
|
177
|
+
replacement: function (content, node) {
|
|
178
|
+
// Ensure there are no blank lines
|
|
179
|
+
content = content.replace('\n\n', '\n');
|
|
180
|
+
return '\n\n' + content + '\n\n';
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
turndownService.addRule('tableSection', {
|
|
184
|
+
filter: ['thead', 'tbody', 'tfoot'],
|
|
185
|
+
replacement: function (content) {
|
|
186
|
+
return content;
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
const indexOf = Array.prototype.indexOf;
|
|
191
|
+
const every = Array.prototype.every;
|
|
192
|
+
// A tr is a heading row if:
|
|
193
|
+
// - the parent is a THEAD
|
|
194
|
+
// - or if its the first child of the TABLE or the first TBODY (possibly
|
|
195
|
+
// following a blank THEAD)
|
|
196
|
+
// - and every cell is a TH
|
|
197
|
+
function isHeadingRow(tr) {
|
|
198
|
+
var parentNode = tr.parentNode;
|
|
199
|
+
return (parentNode.nodeName === 'THEAD' ||
|
|
200
|
+
(parentNode.firstChild === tr &&
|
|
201
|
+
(parentNode.nodeName === 'TABLE' || isFirstTbody(parentNode)) &&
|
|
202
|
+
every.call(tr.childNodes, function (n) { return n.nodeName === 'TH'; })));
|
|
203
|
+
}
|
|
204
|
+
function isFirstTbody(element) {
|
|
205
|
+
var previousSibling = element.previousSibling;
|
|
206
|
+
return (element.nodeName === 'TBODY' && (!previousSibling ||
|
|
207
|
+
(previousSibling.nodeName === 'THEAD' &&
|
|
208
|
+
/^\s*$/i.test(previousSibling.textContent))));
|
|
209
|
+
}
|
|
210
|
+
function cell(content, node) {
|
|
211
|
+
var index = indexOf.call(node.parentNode.childNodes, node);
|
|
212
|
+
var prefix = ' ';
|
|
213
|
+
if (index === 0) {
|
|
214
|
+
prefix = '| ';
|
|
215
|
+
}
|
|
216
|
+
return cleanContent(prefix + content + ' |');
|
|
217
|
+
}
|
|
218
|
+
function cleanContent(content) {
|
|
219
|
+
let output = '';
|
|
220
|
+
const chars = ['\n', '\r', '\t', '\f', '\v', '\u00a0', '\u2028', '\u2029', ' '];
|
|
221
|
+
for (let i = 0; i < content.length; i++) {
|
|
222
|
+
if (chars.includes(content[i])) {
|
|
223
|
+
if (output[output.length - 1] != ' ') {
|
|
224
|
+
output += ' ';
|
|
225
|
+
}
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
output += content[i];
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return output;
|
|
233
|
+
}
|
|
234
|
+
//# sourceMappingURL=WebFetcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WebFetcher.js","sourceRoot":"","sources":["../src/WebFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kDAAkD;AAElD,iDAAmC;AACnC,wDAAwC;AAExC,MAAM,qBAAqB,GAAG;IAC1B,WAAW;IACX,kBAAkB;IAClB,iBAAiB;IACjB,wBAAwB;IACxB,YAAY;CACf,CAAC;AAEF,MAAM,eAAe,GAAG;IACpB,MAAM,EAAE,uFAAuF;IAC/F,iBAAiB,EAAE,yBAAyB;IAC5C,iBAAiB,EAAE,yBAAyB;IAC5C,UAAU,EAAE,4BAA4B;IACxC,UAAU,EAAE,YAAY;IACxB,IAAI,EAAE,4BAA4B;IAClC,OAAO,EAAE,yBAAyB;IAClC,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,YAAY;IAC9B,2BAA2B,EAAE,GAAG;IAChC,YAAY,EAAE,0GAA0G;CAC3H,CAAC;AASF,MAAa,UAAU;IAGnB,YAAmB,MAAkC;QACjD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK;SACH,EAAE,MAAM,CAAC,CAAC;IACnC,CAAC;IAEY,KAAK,CAAC,GAAW,EAAE,UAA6E;;YACzG,MAAM,UAAU,GAAG,eAAK,CAAC,MAAM,CAAC;gBAC5B,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI;aAC7B,CAAC,CAAC;YAEH,+CAA+C;YAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAExE,wBAAwB;YACxB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;YAE3B,kCAAkC;YAClC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,GAAG,kBACrC,OAAO,IACJ,IAAI,CAAC,OAAO,CAAC,aAAa,EAC/B,CAAC;YACH,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1E,CAAC;YAED,+BAA+B;YAC/B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACrD,MAAM,gBAAgB,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/E,MAAM,IAAI,KAAK,CAAC,4CAA4C,WAAW,EAAE,CAAC,CAAC;YAC/E,CAAC;YAED,mCAAmC;YACnC,MAAM,OAAO,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACpG,IAAI,OAAO,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;gBACnD,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACrD,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC3B,OAAO,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC;QACL,CAAC;KAAA;IAGO,cAAc,CAAC,IAAY,EAAE,OAAe;;QAChD,gCAAgC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzD,wDAAwD;QACxD,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;QACrB,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;YACpB,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,qCAAqC;gBACrC,IAAI,CAAC;oBACD,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACxD,CAAC;gBAAC,WAAM,CAAC;oBACL,cAAc;gBAClB,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,sBAAsB;QACtB,MAAM,IAAI,GAAG,MAAA,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,mCAAI,EAAE,CAAC;QACpC,MAAM,eAAe,GAAG,IAAI,kBAAe,CAAC;YACxC,EAAE,EAAE,aAAa;SACpB,CAAC,CAAC;QACH,aAAa,CAAC,eAAe,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE1C,qCAAqC;QACrC,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,IAAI,YAAY,GAAG,EAAE,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACJ,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ;AAtFD,gCAsFC;AAED,SAAS,aAAa,CAAC,eAAgC;IACnD,eAAe,CAAC,OAAO,CAAC,WAAW,EAAE;QACjC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;QACpB,WAAW,EAAE,UAAU,OAAO,EAAE,IAAI;YAChC,OAAO,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC9B,CAAC;KACJ,CAAC,CAAC;IAEH,eAAe,CAAC,OAAO,CAAC,UAAU,EAAE;QAChC,MAAM,EAAE,IAAI;QACZ,WAAW,EAAE,UAAU,OAAO,EAAE,IAAI;YAChC,IAAI,WAAW,GAAG,EAAE,CAAA;YACpB,IAAI,QAAQ,GAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;YAEhE,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9C,IAAI,MAAM,GAAG,KAAK,CAAA;oBAClB,IAAI,KAAK,GAAW,CAChB,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CACjD,CAAC,WAAW,EAAE,CAAA;oBAEf,IAAI,KAAK;wBAAE,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAA;oBAE7C,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAA;gBACnD,CAAC;YACL,CAAC;YACD,OAAO,IAAI,GAAG,OAAO,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QACnE,CAAC;KACJ,CAAC,CAAC;IAEH,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,MAAM,EAAE,CAAC,OAAO,CAAC;QACjB,WAAW,EAAE,UAAU,OAAO,EAAE,IAAI;YAChC,kCAAkC;YAClC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YACvC,OAAO,MAAM,GAAG,OAAO,GAAG,MAAM,CAAA;QACpC,CAAC;KACJ,CAAC,CAAC;IAEH,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE;QACpC,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;QACnC,WAAW,EAAE,UAAU,OAAO;YAC1B,OAAO,OAAO,CAAA;QAClB,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAA;AACvC,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAA;AAEnC,4BAA4B;AAC5B,0BAA0B;AAC1B,wEAAwE;AACxE,6BAA6B;AAC7B,2BAA2B;AAC3B,SAAS,YAAY,CAAC,EAAO;IACzB,IAAI,UAAU,GAAG,EAAE,CAAC,UAAU,CAAA;IAC9B,OAAO,CACH,UAAU,CAAC,QAAQ,KAAK,OAAO;QAC/B,CACI,UAAU,CAAC,UAAU,KAAK,EAAE;YAC5B,CAAC,UAAU,CAAC,QAAQ,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAA,CAAC,CAAC,CAAC,CACzE,CACJ,CAAA;AACL,CAAC;AAED,SAAS,YAAY,CAAC,OAAY;IAC9B,IAAI,eAAe,GAAG,OAAO,CAAC,eAAe,CAAA;IAC7C,OAAO,CACH,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,CAC5B,CAAC,eAAe;QAChB,CACI,eAAe,CAAC,QAAQ,KAAK,OAAO;YACpC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAC7C,CACJ,CACJ,CAAA;AACL,CAAC;AAED,SAAS,IAAI,CAAC,OAAe,EAAE,IAAS;IACpC,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;IAC1D,IAAI,MAAM,GAAG,GAAG,CAAA;IAChB,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACd,MAAM,GAAG,IAAI,CAAA;IACjB,CAAC;IACD,OAAO,YAAY,CAAC,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACjC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,MAAM,KAAK,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;IAChF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,CAAC;YAClB,CAAC;YACD,SAAS;QACb,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC"}
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export * from './FileFetcher';
|
|
2
|
+
export * from './GPT3Tokenizer';
|
|
3
|
+
export * from './ItemSelector';
|
|
4
|
+
export * from './LocalIndex';
|
|
5
|
+
export * from './LocalDocument';
|
|
6
|
+
export * from './LocalDocumentIndex';
|
|
7
|
+
export * from './LocalDocumentResult';
|
|
8
|
+
export * from './OpenAIEmbeddings';
|
|
9
|
+
export * from './TextSplitter';
|
|
10
|
+
export * from './types';
|
|
11
|
+
export * from './WebFetcher';
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./FileFetcher"), exports);
|
|
18
|
+
__exportStar(require("./GPT3Tokenizer"), exports);
|
|
19
|
+
__exportStar(require("./ItemSelector"), exports);
|
|
20
|
+
__exportStar(require("./LocalIndex"), exports);
|
|
21
|
+
__exportStar(require("./LocalDocument"), exports);
|
|
22
|
+
__exportStar(require("./LocalDocumentIndex"), exports);
|
|
23
|
+
__exportStar(require("./LocalDocumentResult"), exports);
|
|
24
|
+
__exportStar(require("./OpenAIEmbeddings"), exports);
|
|
25
|
+
__exportStar(require("./TextSplitter"), exports);
|
|
26
|
+
__exportStar(require("./types"), exports);
|
|
27
|
+
__exportStar(require("./WebFetcher"), exports);
|
|
28
|
+
//# sourceMappingURL=index.js.map
|
package/lib/index.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,gDAA8B;AAC9B,kDAAgC;AAChC,iDAA+B;AAC/B,+CAA6B;AAC7B,kDAAgC;AAChC,uDAAqC;AACrC,wDAAsC;AACtC,qDAAmC;AACnC,iDAA+B;AAC/B,0CAAwB;AACxB,+CAA6B"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @private
|
|
3
|
+
*/
|
|
4
|
+
export declare class Colorize {
|
|
5
|
+
static replaceLine(text: string): string;
|
|
6
|
+
static error(error: Error | string): string;
|
|
7
|
+
static output(output: object | string, isBm25?: boolean, quote?: string, units?: string): string;
|
|
8
|
+
static progress(message: string): string;
|
|
9
|
+
static success(message: string): string;
|
|
10
|
+
static title(title: string): string;
|
|
11
|
+
static value(field: string, value: any, units?: string): string;
|
|
12
|
+
static warning(warning: string): string;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=Colorize.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Colorize.d.ts","sourceRoot":"","sources":["../../src/internals/Colorize.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,qBAAa,QAAQ;WACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;WAIjC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAC,MAAM,GAAG,MAAM;WAQlC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,MAAM,GAAE,OAAe,EAAE,KAAK,GAAE,MAAW,EAAE,KAAK,GAAE,MAAW,GAAG,MAAM;WAyBxG,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;WAIjC,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;WAIhC,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;WAI5B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAE,MAAW,GAAG,MAAM;WAI5D,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;CAIjD"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Colorize = void 0;
|
|
4
|
+
const colorizer = require('json-colorizer');
|
|
5
|
+
/**
|
|
6
|
+
* @private
|
|
7
|
+
*/
|
|
8
|
+
class Colorize {
|
|
9
|
+
static replaceLine(text) {
|
|
10
|
+
return '\x1b[A\x1b[2K' + text;
|
|
11
|
+
}
|
|
12
|
+
static error(error) {
|
|
13
|
+
if (typeof error === 'string') {
|
|
14
|
+
return `\x1b[31;1m${error}\x1b[0m`;
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
return `\x1b[31;1m${error.message}\x1b[0m`;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
static output(output, isBm25 = false, quote = '', units = '') {
|
|
21
|
+
if (typeof output === 'string') {
|
|
22
|
+
return isBm25 ? `\x1b[34m${quote}${output}${quote}\x1b[0m` : `\x1b[32m${quote}${output}${quote}\x1b[0m`;
|
|
23
|
+
}
|
|
24
|
+
else if (typeof output === 'object' && output !== null) {
|
|
25
|
+
return colorizer(output, {
|
|
26
|
+
pretty: true,
|
|
27
|
+
colors: {
|
|
28
|
+
BRACE: 'white',
|
|
29
|
+
BRACKET: 'white',
|
|
30
|
+
COLON: 'white',
|
|
31
|
+
COMMA: 'white',
|
|
32
|
+
STRING_KEY: 'white',
|
|
33
|
+
STRING_LITERAL: 'green',
|
|
34
|
+
NUMBER_LITERAL: 'blue',
|
|
35
|
+
BOOLEAN_LITERAL: 'blue',
|
|
36
|
+
NULL_LITERAL: 'blue'
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
else if (typeof output == 'number') {
|
|
41
|
+
return `\x1b[34m${output}${units}\x1b[0m`;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
return `\x1b[34m${output}\x1b[0m`;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
static progress(message) {
|
|
48
|
+
return message;
|
|
49
|
+
}
|
|
50
|
+
static success(message) {
|
|
51
|
+
return `\x1b[32;1m${message}\x1b[0m`;
|
|
52
|
+
}
|
|
53
|
+
static title(title) {
|
|
54
|
+
return `\x1b[35;1m${title}\x1b[0m`;
|
|
55
|
+
}
|
|
56
|
+
static value(field, value, units = '') {
|
|
57
|
+
return `${field}: ${Colorize.output(value, false, '"', units)}`;
|
|
58
|
+
}
|
|
59
|
+
static warning(warning) {
|
|
60
|
+
return `\x1b[33m${warning}\x1b[0m`;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
exports.Colorize = Colorize;
|
|
64
|
+
//# sourceMappingURL=Colorize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Colorize.js","sourceRoot":"","sources":["../../src/internals/Colorize.ts"],"names":[],"mappings":";;;AAAA,MAAM,SAAS,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE5C;;GAEG;AACH,MAAa,QAAQ;IACV,MAAM,CAAC,WAAW,CAAC,IAAY;QAClC,OAAO,eAAe,GAAG,IAAI,CAAC;IAClC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,KAAmB;QACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,OAAO,aAAa,KAAK,SAAS,CAAC;QACvC,CAAC;aAAM,CAAC;YACJ,OAAO,aAAa,KAAK,CAAC,OAAO,SAAS,CAAC;QAC/C,CAAC;IACL,CAAC;IAEM,MAAM,CAAC,MAAM,CAAC,MAAuB,EAAE,SAAkB,KAAK,EAAE,QAAgB,EAAE,EAAE,QAAgB,EAAE;QACzG,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,KAAK,GAAG,MAAM,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,KAAK,GAAG,MAAM,GAAG,KAAK,SAAS,CAAC;QAC5G,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACvD,OAAO,SAAS,CAAC,MAAM,EAAE;gBACrB,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE;oBACJ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,OAAO;oBAChB,KAAK,EAAE,OAAO;oBACd,KAAK,EAAE,OAAO;oBACd,UAAU,EAAE,OAAO;oBACnB,cAAc,EAAE,OAAO;oBACvB,cAAc,EAAE,MAAM;oBACtB,eAAe,EAAE,MAAM;oBACvB,YAAY,EAAE,MAAM;iBACvB;aACJ,CAAC,CAAC;QACP,CAAC;aAAM,IAAI,OAAO,MAAM,IAAI,QAAQ,EAAE,CAAC;YACnC,OAAO,WAAW,MAAM,GAAG,KAAK,SAAS,CAAC;QAC9C,CAAC;aAAM,CAAC;YACJ,OAAO,WAAW,MAAM,SAAS,CAAC;QACtC,CAAC;IACL,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,OAAe;QAClC,OAAO,OAAO,CAAC;IACnB,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,OAAe;QACjC,OAAO,aAAa,OAAO,SAAS,CAAC;IACzC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,KAAa;QAC7B,OAAO,aAAa,KAAK,SAAS,CAAC;IACvC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,KAAa,EAAE,KAAU,EAAE,QAAgB,EAAE;QAC7D,OAAO,GAAG,KAAK,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;IACpE,CAAC;IAEM,MAAM,CAAC,OAAO,CAAC,OAAe;QACjC,OAAO,WAAW,OAAO,SAAS,CAAC;IAEvC,CAAC;CACJ;AA1DD,4BA0DC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/internals/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./Colorize"), exports);
|
|
18
|
+
__exportStar(require("./types"), exports);
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/internals/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA2B;AAC3B,0CAAwB"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @private
|
|
3
|
+
*/
|
|
4
|
+
export interface CreateEmbeddingRequest {
|
|
5
|
+
input: CreateEmbeddingRequestInput;
|
|
6
|
+
dimensions?: number;
|
|
7
|
+
user?: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* @private
|
|
11
|
+
*/
|
|
12
|
+
export interface OpenAICreateEmbeddingRequest extends CreateEmbeddingRequest {
|
|
13
|
+
model: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* @private
|
|
17
|
+
*/
|
|
18
|
+
export interface CreateEmbeddingResponse {
|
|
19
|
+
object: string;
|
|
20
|
+
model: string;
|
|
21
|
+
data: Array<CreateEmbeddingResponseDataInner>;
|
|
22
|
+
usage: CreateEmbeddingResponseUsage;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* @private
|
|
26
|
+
*/
|
|
27
|
+
export interface CreateEmbeddingResponseDataInner {
|
|
28
|
+
index: number;
|
|
29
|
+
object: string;
|
|
30
|
+
embedding: Array<number>;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* @private
|
|
34
|
+
*/
|
|
35
|
+
export interface CreateEmbeddingResponseUsage {
|
|
36
|
+
prompt_tokens: number;
|
|
37
|
+
total_tokens: number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* @private
|
|
41
|
+
*/
|
|
42
|
+
export type CreateEmbeddingRequestInput = Array<any> | Array<number> | Array<string> | string;
|
|
43
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/internals/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACnC,KAAK,EAAE,2BAA2B,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,4BAA6B,SAAQ,sBAAsB;IACxE,KAAK,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC9C,KAAK,EAAE,4BAA4B,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,2BAA2B,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/internals/types.ts"],"names":[],"mappings":""}
|
package/lib/types.d.ts
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An AI model that can be used to create embeddings.
|
|
3
|
+
*/
|
|
4
|
+
export interface EmbeddingsModel {
|
|
5
|
+
/**
|
|
6
|
+
* Maximum number of tokens
|
|
7
|
+
*/
|
|
8
|
+
readonly maxTokens: number;
|
|
9
|
+
/**
|
|
10
|
+
* Creates embeddings for the given inputs.
|
|
11
|
+
* @param inputs Text inputs to create embeddings for.
|
|
12
|
+
* @returns A `EmbeddingsResponse` with a status and the generated embeddings or a message when an error occurs.
|
|
13
|
+
*/
|
|
14
|
+
createEmbeddings(inputs: string | string[]): Promise<EmbeddingsResponse>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Status of the embeddings response.
|
|
18
|
+
* @remarks
|
|
19
|
+
* `success` - The embeddings were successfully created.
|
|
20
|
+
* `error` - An error occurred while creating the embeddings.
|
|
21
|
+
* `rate_limited` - The request was rate limited.
|
|
22
|
+
*/
|
|
23
|
+
export type EmbeddingsResponseStatus = 'success' | 'error' | 'rate_limited' | 'cancelled';
|
|
24
|
+
/**
|
|
25
|
+
* Response returned by a `EmbeddingsClient`.
|
|
26
|
+
*/
|
|
27
|
+
export interface EmbeddingsResponse {
|
|
28
|
+
/**
|
|
29
|
+
* Status of the embeddings response.
|
|
30
|
+
*/
|
|
31
|
+
status: EmbeddingsResponseStatus;
|
|
32
|
+
/**
|
|
33
|
+
* Optional. Embeddings for the given inputs.
|
|
34
|
+
*/
|
|
35
|
+
output?: number[][];
|
|
36
|
+
/**
|
|
37
|
+
* Optional. Message when status is not equal to `success`.
|
|
38
|
+
*/
|
|
39
|
+
message?: string;
|
|
40
|
+
/**
|
|
41
|
+
* Optional. Model used to create the embeddings.
|
|
42
|
+
*/
|
|
43
|
+
model?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Optional. Usage statistics for the request.
|
|
46
|
+
*/
|
|
47
|
+
usage?: Record<string, any>;
|
|
48
|
+
}
|
|
49
|
+
export interface TextChunk {
|
|
50
|
+
text: string;
|
|
51
|
+
tokens: number[];
|
|
52
|
+
startPos: number;
|
|
53
|
+
endPos: number;
|
|
54
|
+
startOverlap: number[];
|
|
55
|
+
endOverlap: number[];
|
|
56
|
+
}
|
|
57
|
+
export interface TextFetcher {
|
|
58
|
+
fetch(uri: string, onDocument: (uri: string, text: string, docType?: string) => Promise<boolean>): Promise<boolean>;
|
|
59
|
+
}
|
|
60
|
+
export interface IndexStats {
|
|
61
|
+
version: number;
|
|
62
|
+
metadata_config: {
|
|
63
|
+
indexed?: string[];
|
|
64
|
+
};
|
|
65
|
+
items: number;
|
|
66
|
+
}
|
|
67
|
+
export interface IndexItem<TMetadata = Record<string, MetadataTypes>> {
|
|
68
|
+
id: string;
|
|
69
|
+
metadata: TMetadata;
|
|
70
|
+
vector: number[];
|
|
71
|
+
norm: number;
|
|
72
|
+
metadataFile?: string;
|
|
73
|
+
}
|
|
74
|
+
export interface MetadataFilter {
|
|
75
|
+
/**
|
|
76
|
+
* Equal to (number, string, boolean)
|
|
77
|
+
*/
|
|
78
|
+
'$eq'?: number | string | boolean;
|
|
79
|
+
/**
|
|
80
|
+
* Not equal to (number, string, boolean)
|
|
81
|
+
*/
|
|
82
|
+
'$ne'?: number | string | boolean;
|
|
83
|
+
/**
|
|
84
|
+
* Greater than (number)
|
|
85
|
+
*/
|
|
86
|
+
'$gt'?: number;
|
|
87
|
+
/**
|
|
88
|
+
* Greater than or equal to (number)
|
|
89
|
+
*/
|
|
90
|
+
'$gte'?: number;
|
|
91
|
+
/**
|
|
92
|
+
* Less than (number)
|
|
93
|
+
*/
|
|
94
|
+
'$lt'?: number;
|
|
95
|
+
/**
|
|
96
|
+
* Less than or equal to (number)
|
|
97
|
+
*/
|
|
98
|
+
'$lte'?: number;
|
|
99
|
+
/**
|
|
100
|
+
* In array (string or number)
|
|
101
|
+
*/
|
|
102
|
+
'$in'?: (number | string)[];
|
|
103
|
+
/**
|
|
104
|
+
* Not in array (string or number)
|
|
105
|
+
*/
|
|
106
|
+
'$nin'?: (number | string)[];
|
|
107
|
+
/**
|
|
108
|
+
* AND (MetadataFilter[])
|
|
109
|
+
*/
|
|
110
|
+
'$and'?: MetadataFilter[];
|
|
111
|
+
/**
|
|
112
|
+
* OR (MetadataFilter[])
|
|
113
|
+
*/
|
|
114
|
+
'$or'?: MetadataFilter[];
|
|
115
|
+
[key: string]: unknown;
|
|
116
|
+
}
|
|
117
|
+
export type MetadataTypes = number | string | boolean;
|
|
118
|
+
export interface QueryResult<TMetadata = Record<string, MetadataTypes>> {
|
|
119
|
+
item: IndexItem<TMetadata>;
|
|
120
|
+
score: number;
|
|
121
|
+
}
|
|
122
|
+
export interface Tokenizer {
|
|
123
|
+
decode(tokens: number[]): string;
|
|
124
|
+
encode(text: string): number[];
|
|
125
|
+
}
|
|
126
|
+
export interface DocumentChunkMetadata {
|
|
127
|
+
documentId: string;
|
|
128
|
+
startPos: number;
|
|
129
|
+
endPos: number;
|
|
130
|
+
[key: string]: MetadataTypes;
|
|
131
|
+
}
|
|
132
|
+
export interface DocumentCatalogStats {
|
|
133
|
+
version: number;
|
|
134
|
+
documents: number;
|
|
135
|
+
chunks: number;
|
|
136
|
+
metadata_config: {
|
|
137
|
+
indexed?: string[];
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
export interface DocumentTextSection {
|
|
141
|
+
text: string;
|
|
142
|
+
tokenCount: number;
|
|
143
|
+
score: number;
|
|
144
|
+
isBm25: boolean;
|
|
145
|
+
}
|
|
146
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAC,MAAM,EAAE,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;CAC1E;AAED;;;;;;GAMG;AACH,MAAM,MAAM,wBAAwB,GAAG,SAAS,GAAG,OAAO,GAAG,cAAc,GAAG,WAAW,CAAC;AAE1F;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,wBAAwB,CAAC;IAEjC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAEpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,SAAS;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,WAAW;IACxB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACvH;AAED,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,SAAS,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAE3B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE9B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE1B;;OAEG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE3B;;OAEG;IACH,MAAM,CAAC,EAAE,cAAc,EAAE,CAAC;IAE1B;;OAEG;IACH,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IAEzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,MAAM,aAAa,GAAG,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;AAElD,MAAM,WAAW,WAAW,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjE,IAAI,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;CACnB"}
|
package/lib/types.js
ADDED
package/lib/types.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|