vectra 0.12.2 → 0.12.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +1 -1
  2. package/README.draft.md +499 -0
  3. package/README.draft.outline.md +160 -0
  4. package/README.research.md +2159 -0
  5. package/bin/vectra.js +3 -0
  6. package/lib/FileFetcher.d.ts +5 -0
  7. package/lib/FileFetcher.d.ts.map +1 -0
  8. package/lib/FileFetcher.js +79 -0
  9. package/lib/FileFetcher.js.map +1 -0
  10. package/lib/GPT3Tokenizer.d.ts +9 -0
  11. package/lib/ItemSelector.d.ts +41 -0
  12. package/lib/ItemSelector.d.ts.map +1 -0
  13. package/lib/ItemSelector.js +168 -0
  14. package/lib/ItemSelector.js.map +1 -0
  15. package/lib/LocalDocument.d.ts +54 -0
  16. package/lib/LocalDocument.js +156 -0
  17. package/lib/LocalDocument.js.map +1 -0
  18. package/lib/LocalDocumentIndex.d.ts +132 -0
  19. package/lib/LocalDocumentIndex.js +456 -0
  20. package/lib/LocalDocumentIndex.js.map +1 -0
  21. package/lib/LocalDocumentResult.d.ts +45 -0
  22. package/lib/LocalDocumentResult.js +328 -0
  23. package/lib/LocalDocumentResult.js.map +1 -0
  24. package/lib/LocalIndex.d.ts +150 -0
  25. package/lib/LocalIndex.d.ts.map +1 -1
  26. package/lib/LocalIndex.js +515 -0
  27. package/lib/LocalIndex.js.map +1 -0
  28. package/lib/LocalIndex.spec.d.ts +2 -0
  29. package/lib/LocalIndex.spec.js +218 -7
  30. package/lib/LocalIndex.spec.js.map +1 -1
  31. package/lib/OpenAIEmbeddings.d.ts +126 -0
  32. package/lib/OpenAIEmbeddings.d.ts.map +1 -0
  33. package/lib/OpenAIEmbeddings.js +174 -0
  34. package/lib/OpenAIEmbeddings.js.map +1 -0
  35. package/lib/TextSplitter.d.ts +19 -0
  36. package/lib/TextSplitter.d.ts.map +1 -1
  37. package/lib/TextSplitter.js +457 -0
  38. package/lib/TextSplitter.js.map +1 -0
  39. package/lib/TextSplitter.spec.d.ts +2 -0
  40. package/lib/TextSplitter.spec.d.ts.map +1 -0
  41. package/lib/TextSplitter.spec.js +109 -0
  42. package/lib/TextSplitter.spec.js.map +1 -0
  43. package/lib/WebFetcher.d.ts +15 -0
  44. package/lib/WebFetcher.d.ts.map +1 -0
  45. package/lib/WebFetcher.js +234 -0
  46. package/lib/WebFetcher.js.map +1 -0
  47. package/lib/index.d.ts +12 -0
  48. package/lib/index.js +28 -0
  49. package/lib/index.js.map +1 -0
  50. package/lib/internals/Colorize.d.ts +14 -0
  51. package/lib/internals/Colorize.d.ts.map +1 -0
  52. package/lib/internals/Colorize.js +64 -0
  53. package/lib/internals/Colorize.js.map +1 -0
  54. package/lib/internals/index.d.ts +3 -0
  55. package/lib/internals/index.d.ts.map +1 -0
  56. package/lib/internals/index.js +19 -0
  57. package/lib/internals/index.js.map +1 -0
  58. package/lib/internals/types.d.ts +43 -0
  59. package/lib/internals/types.d.ts.map +1 -0
  60. package/lib/internals/types.js +3 -0
  61. package/lib/internals/types.js.map +1 -0
  62. package/lib/types.d.ts +146 -0
  63. package/lib/types.d.ts.map +1 -0
  64. package/lib/types.js +3 -0
  65. package/lib/types.js.map +1 -0
  66. package/lib/vectra-cli.d.ts +2 -0
  67. package/lib/vectra-cli.js +323 -0
  68. package/lib/vectra-cli.js.map +1 -0
  69. package/package.json +3 -1
  70. package/src/LocalIndex.spec.ts +265 -8
  71. package/src/LocalIndex.ts +1 -0
  72. package/src/TextSplitter.spec.ts +87 -0
  73. package/src/TextSplitter.ts +459 -531
@@ -0,0 +1,328 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.LocalDocumentResult = void 0;
13
+ const LocalDocument_1 = require("./LocalDocument");
14
+ /**
15
+ * Represents a search result for a document stored on disk.
16
+ */
17
+ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
18
+ /**
19
+ * @private
20
+ * Internal constructor for `LocalDocumentResult` instances.
21
+ */
22
+ constructor(index, id, uri, chunks, tokenizer) {
23
+ super(index, id, uri);
24
+ this._chunks = chunks;
25
+ this._tokenizer = tokenizer;
26
+ // Compute average score
27
+ let score = 0;
28
+ this._chunks.forEach(chunk => score += chunk.score);
29
+ this._score = score / this._chunks.length;
30
+ }
31
+ /**
32
+ * Returns the chunks of the document that matched the query.
33
+ */
34
+ get chunks() {
35
+ return this._chunks;
36
+ }
37
+ /**
38
+ * Returns the average score of the document result.
39
+ */
40
+ get score() {
41
+ return this._score;
42
+ }
43
+ /**
44
+ * Renders all of the results chunks as spans of text (sections.)
45
+ * @remarks
46
+ * The returned sections will be sorted by document order and limited to maxTokens in length.
47
+ * @param maxTokens Maximum number of tokens per section.
48
+ * @returns Array of rendered text sections.
49
+ */
50
+ renderAllSections(maxTokens) {
51
+ return __awaiter(this, void 0, void 0, function* () {
52
+ // Load text from disk
53
+ const text = yield this.loadText();
54
+ // Add chunks to a temp array and split any chunks that are longer than maxTokens.
55
+ const chunks = [];
56
+ for (let i = 0; i < this._chunks.length; i++) {
57
+ const chunk = this._chunks[i];
58
+ const startPos = chunk.item.metadata.startPos;
59
+ const endPos = chunk.item.metadata.endPos;
60
+ const chunkText = text.substring(startPos, endPos + 1);
61
+ const tokens = this._tokenizer.encode(chunkText);
62
+ let offset = 0;
63
+ while (offset < tokens.length) {
64
+ const chunkLength = Math.min(maxTokens, tokens.length - offset);
65
+ chunks.push({
66
+ text: this._tokenizer.decode(tokens.slice(offset, offset + chunkLength)),
67
+ startPos: startPos + offset,
68
+ endPos: startPos + offset + chunkLength - 1,
69
+ score: chunk.score,
70
+ tokenCount: chunkLength,
71
+ isBm25: false
72
+ });
73
+ offset += chunkLength;
74
+ }
75
+ }
76
+ // Sort chunks by startPos
77
+ const sorted = chunks.sort((a, b) => a.startPos - b.startPos);
78
+ // Generate sections
79
+ const sections = [];
80
+ for (let i = 0; i < sorted.length; i++) {
81
+ const chunk = sorted[i];
82
+ let section = sections[sections.length - 1];
83
+ if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
84
+ section = {
85
+ chunks: [],
86
+ score: 0,
87
+ tokenCount: 0
88
+ };
89
+ sections.push(section);
90
+ }
91
+ section.chunks.push(chunk);
92
+ section.score += chunk.score;
93
+ section.tokenCount += chunk.tokenCount;
94
+ }
95
+ // Normalize section scores
96
+ sections.forEach(section => section.score /= section.chunks.length);
97
+ // Return final rendered sections
98
+ return sections.map(section => {
99
+ let text = '';
100
+ section.chunks.forEach(chunk => text += chunk.text);
101
+ return {
102
+ text: text,
103
+ tokenCount: section.tokenCount,
104
+ score: section.score,
105
+ isBm25: false,
106
+ };
107
+ });
108
+ });
109
+ }
110
+ /**
111
+ * Renders the top spans of text (sections) of the document based on the query result.
112
+ * @remarks
113
+ * The returned sections will be sorted by relevance and limited to the top `maxSections`.
114
+ * @param maxTokens Maximum number of tokens per section.
115
+ * @param maxSections Maximum number of sections to return.
116
+ * @param overlappingChunks Optional. If true, overlapping chunks of text will be added to each section until the maxTokens is reached.
117
+ * @returns Array of rendered text sections.
118
+ */
119
+ renderSections(maxTokens_1, maxSections_1) {
120
+ return __awaiter(this, arguments, void 0, function* (maxTokens, maxSections, overlappingChunks = true) {
121
+ // Load text from disk
122
+ const text = yield this.loadText();
123
+ // First check to see if the entire document is shorter than maxTokens
124
+ const length = yield this.getLength();
125
+ if (length <= maxTokens) {
126
+ return [{
127
+ text,
128
+ tokenCount: length,
129
+ score: 1.0,
130
+ isBm25: false,
131
+ }];
132
+ }
133
+ // Otherwise, we need to split the document into sections
134
+ // - Add each chunk to a temp array and filter out any chunk that's longer then maxTokens.
135
+ // - Sort the array by startPos to arrange chunks in document order.
136
+ // - Generate a new array of sections by combining chunks until the maxTokens is reached for each section.
137
+ // - Generate an aggregate score for each section by averaging the score of each chunk in the section.
138
+ // - Sort the sections by score and limit to maxSections.
139
+ // - For each remaining section combine adjacent chunks of text.
140
+ // - Dynamically add overlapping chunks of text to each section until the maxTokens is reached.
141
+ const chunks = this._chunks.map(chunk => {
142
+ const startPos = chunk.item.metadata.startPos;
143
+ const endPos = chunk.item.metadata.endPos;
144
+ const chunkText = text.substring(startPos, endPos + 1);
145
+ return {
146
+ text: chunkText,
147
+ startPos,
148
+ endPos,
149
+ score: chunk.score,
150
+ tokenCount: this._tokenizer.encode(chunkText).length,
151
+ isBm25: Boolean(chunk.item.metadata.isBm25),
152
+ };
153
+ }).filter(chunk => chunk.tokenCount <= maxTokens).sort((a, b) => a.startPos - b.startPos);
154
+ // Check for no chunks
155
+ if (chunks.length === 0) {
156
+ // Take the top chunk and return a subset of its text
157
+ const topChunk = this._chunks[0];
158
+ const startPos = topChunk.item.metadata.startPos;
159
+ const endPos = topChunk.item.metadata.endPos;
160
+ const chunkText = text.substring(startPos, endPos + 1);
161
+ const tokens = this._tokenizer.encode(chunkText);
162
+ return [{
163
+ text: this._tokenizer.decode(tokens.slice(0, maxTokens)),
164
+ tokenCount: maxTokens,
165
+ score: topChunk.score,
166
+ isBm25: false,
167
+ }];
168
+ }
169
+ // Generate semantic sections
170
+ const sections = [];
171
+ for (let i = 0; i < chunks.length; i++) {
172
+ const chunk = chunks[i];
173
+ let section = sections[sections.length - 1];
174
+ if (!chunk.isBm25) {
175
+ if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
176
+ section = {
177
+ chunks: [],
178
+ score: 0,
179
+ tokenCount: 0
180
+ };
181
+ sections.push(section);
182
+ }
183
+ section.chunks.push(chunk);
184
+ section.score += chunk.score;
185
+ section.tokenCount += chunk.tokenCount;
186
+ }
187
+ }
188
+ // Generate bm25 sections
189
+ const bm25Sections = [];
190
+ for (let i = 0; i < chunks.length; i++) {
191
+ const chunk = chunks[i];
192
+ let section = bm25Sections[bm25Sections.length - 1];
193
+ if (chunk.isBm25) {
194
+ if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
195
+ section = {
196
+ chunks: [],
197
+ score: 0,
198
+ tokenCount: 0
199
+ };
200
+ bm25Sections.push(section);
201
+ }
202
+ section.chunks.push(chunk);
203
+ section.score += chunk.score;
204
+ section.tokenCount += chunk.tokenCount;
205
+ }
206
+ }
207
+ // Normalize section scores
208
+ sections.forEach(section => section.score /= section.chunks.length);
209
+ bm25Sections.forEach(section => section.score /= section.chunks.length);
210
+ // Sort sections by score and limit to maxSections
211
+ sections.sort((a, b) => b.score - a.score);
212
+ bm25Sections.sort((a, b) => b.score - a.score);
213
+ if (sections.length > maxSections) {
214
+ sections.splice(maxSections, sections.length - maxSections);
215
+ }
216
+ if (bm25Sections.length > maxSections) {
217
+ bm25Sections.splice(maxSections, bm25Sections.length - maxSections);
218
+ }
219
+ // Combine adjacent chunks of text
220
+ sections.forEach(section => {
221
+ for (let i = 0; i < section.chunks.length - 1; i++) {
222
+ const chunk = section.chunks[i];
223
+ const nextChunk = section.chunks[i + 1];
224
+ if (chunk.endPos + 1 === nextChunk.startPos) {
225
+ chunk.text += nextChunk.text;
226
+ chunk.endPos = nextChunk.endPos;
227
+ chunk.tokenCount += nextChunk.tokenCount;
228
+ section.chunks.splice(i + 1, 1);
229
+ i--;
230
+ }
231
+ }
232
+ });
233
+ // Add overlapping chunks of text to each section until the maxTokens is reached
234
+ if (overlappingChunks) {
235
+ const connector = {
236
+ text: '\n\n...\n\n',
237
+ startPos: -1,
238
+ endPos: -1,
239
+ score: 0,
240
+ tokenCount: this._tokenizer.encode('\n\n...\n\n').length,
241
+ isBm25: false,
242
+ };
243
+ sections.forEach(section => {
244
+ // Insert connectors between chunks
245
+ if (section.chunks.length > 1) {
246
+ for (let i = 0; i < section.chunks.length - 1; i++) {
247
+ section.chunks.splice(i + 1, 0, connector);
248
+ section.tokenCount += connector.tokenCount;
249
+ i++;
250
+ }
251
+ }
252
+ // Add chunks to beginning and end of the section until maxTokens is reached
253
+ let budget = maxTokens - section.tokenCount;
254
+ if (budget > 40) {
255
+ const sectionStart = section.chunks[0].startPos;
256
+ const sectionEnd = section.chunks[section.chunks.length - 1].endPos;
257
+ if (sectionStart > 0) {
258
+ const beforeTex = text.substring(0, section.chunks[0].startPos);
259
+ const beforeTokens = this.encodeBeforeText(beforeTex, Math.ceil(budget / 2));
260
+ const beforeBudget = sectionEnd < text.length - 1 ? Math.min(beforeTokens.length, Math.ceil(budget / 2)) : Math.min(beforeTokens.length, budget);
261
+ const chunk = {
262
+ text: this._tokenizer.decode(beforeTokens.slice(-beforeBudget)),
263
+ startPos: sectionStart - beforeBudget,
264
+ endPos: sectionStart - 1,
265
+ score: 0,
266
+ tokenCount: beforeBudget,
267
+ isBm25: false,
268
+ };
269
+ section.chunks.unshift(chunk);
270
+ section.tokenCount += chunk.tokenCount;
271
+ budget -= chunk.tokenCount;
272
+ }
273
+ if (sectionEnd < text.length - 1) {
274
+ const afterText = text.substring(sectionEnd + 1);
275
+ const afterTokens = this.encodeAfterText(afterText, budget);
276
+ const afterBudget = Math.min(afterTokens.length, budget);
277
+ const chunk = {
278
+ text: this._tokenizer.decode(afterTokens.slice(0, afterBudget)),
279
+ startPos: sectionEnd + 1,
280
+ endPos: sectionEnd + afterBudget,
281
+ score: 0,
282
+ tokenCount: afterBudget,
283
+ isBm25: false,
284
+ };
285
+ section.chunks.push(chunk);
286
+ section.tokenCount += chunk.tokenCount;
287
+ budget -= chunk.tokenCount;
288
+ }
289
+ }
290
+ });
291
+ }
292
+ const semanticDocTextSections = sections.map(section => {
293
+ let text = '';
294
+ section.chunks.forEach(chunk => text += chunk.text);
295
+ return {
296
+ text: text,
297
+ tokenCount: section.tokenCount,
298
+ score: section.score,
299
+ isBm25: false,
300
+ };
301
+ });
302
+ const bm25DocTextSections = bm25Sections.map(section => {
303
+ let text = '';
304
+ section.chunks.forEach(chunk => text += chunk.text);
305
+ return {
306
+ text: text,
307
+ tokenCount: section.tokenCount,
308
+ score: section.score,
309
+ isBm25: true,
310
+ };
311
+ });
312
+ // Return final rendered sections
313
+ return [...semanticDocTextSections, ...bm25DocTextSections];
314
+ });
315
+ }
316
+ encodeBeforeText(text, budget) {
317
+ const maxLength = budget * 8;
318
+ const substr = text.length <= maxLength ? text : text.substring(text.length - maxLength);
319
+ return this._tokenizer.encode(substr);
320
+ }
321
+ encodeAfterText(text, budget) {
322
+ const maxLength = budget * 8;
323
+ const substr = text.length <= maxLength ? text : text.substring(0, maxLength);
324
+ return this._tokenizer.encode(substr);
325
+ }
326
+ }
327
+ exports.LocalDocumentResult = LocalDocumentResult;
328
+ //# sourceMappingURL=LocalDocumentResult.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;AAIhD;;GAEG;AACH,MAAa,mBAAoB,SAAQ,6BAAa;IAKlD;;;OAGG;IACH,YAAmB,KAAyB,EAAE,EAAU,EAAE,GAAW,EAAE,MAA4C,EAAE,SAAoB;QACrI,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,wBAAwB;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACU,iBAAiB,CAAC,SAAiB;;YAC5C,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,kFAAkF;YAClF,MAAM,MAAM,GAAoB,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;oBAChE,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;wBACxE,QAAQ,EAAE,QAAQ,GAAG,MAAM;wBAC3B,MAAM,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,CAAC;wBAC3C,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,UAAU,EAAE,WAAW;wBACvB,MAAM,EAAE,KAAK;qBAChB,CAAC,CAAC;oBACH,MAAM,IAAI,WAAW,CAAC;gBAC1B,CAAC;YACL,CAAC;YAED,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE9D,oBAAoB;YACpB,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;oBAChE,OAAO,GAAG;wBACN,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC;oBACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC3B,CAAC;gBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;YAC3C,CAAC;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,KAAK;iBAChB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAED;;;;;;;;OAQG;IACU,cAAc;6DAAC,SAAiB,EAAE,WAAmB,EAAE,iBAAiB,GAAG,IAAI;YACxF,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;gBACtB,OAAO,CAAC;wBACJ,IAAI;wBACJ,UAAU,EAAE,MAAM;wBAClB,KAAK,EAAE,GAAG;wBACV,MAAM,EAAE,KAAK;qBAChB,CAAC,CAAC;YACP,CAAC;YAED,yDAAyD;YACzD,0FAA0F;YAC1F,oEAAoE;YACpE,0GAA0G;YAC1G,sGAAsG;YACtG,yDAAyD;YACzD,gEAAgE;YAChE,+FAA+F;YAC/F,MAAM,MAAM,GAAmB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gBACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,OAAO;oBACH,IAAI,EAAE,SAAS;oBACf,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM;oBACpD,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;iBAC9C,CAAC;YACN,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE1F,sBAAsB;YACtB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,qDAAqD;gBACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,OAAO,CAAC;wBACJ,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxD,UAAU,EAAE,SAAS;wBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;wBACrB,MAAM,EAAE,KAAK;qBAChB,CAAC,CAAC;YACP,CAAC;YAED,6BAA6B;YAC7B,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;oBAChB,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;wBAChE,OAAO,GAAG;4BACN,MAAM,EAAE,EAAE;4BACV,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,CAAC;yBAChB,CAAC;wBACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3B,CAAC;oBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;oBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC3C,CAAC;YACL,CAAC;YAED,yBAAyB;YACzB,MAAM,YAAY,GAAc,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACpD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;oBACf,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;wBAChE,OAAO,GAAG;4BACN,MAAM,EAAE,EAAE;4BACV,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,CAAC;yBAChB,CAAC;wBACF,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC/B,CAAC;oBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;oBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC3C,CAAC;YACL,CAAC;YACD,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACpE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAExE,kDAAkD;YAClD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3C,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/C,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;gBAChC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;gBACpC,YAAY,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;YACxE,CAAC;YAED,kCAAkC;YAClC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBACjD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE,CAAC;wBAC1C,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC;wBAC7B,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;wBAChC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBACzC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChC,CAAC,EAAE,CAAC;oBACR,CAAC;gBACL,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,gFAAgF;YAChF,IAAI,iBAAiB,EAAE,CAAC;gBACpB,MAAM,SAAS,GAAiB;oBAC5B,IAAI,EAAE,aAAa;oBACnB,QAAQ,EAAE,CAAC,CAAC;oBACZ,MAAM,EAAE,CAAC,CAAC;oBACV,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM;oBACxD,MAAM,EAAE,KAAK;iBAChB,CAAC;gBACF,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;oBACvB,mCAAmC;oBACnC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;4BACjD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;4BAC3C,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;4BAC3C,CAAC,EAAE,CAAC;wBACR,CAAC;oBACL,CAAC;oBAED,4EAA4E;oBAC5E,IAAI,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;oBAC5C,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;wBACd,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;wBAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;wBACpE,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;4BACnB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;4BAChE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC;4BAC3E,MAAM,YAAY,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;4BAC/I,MAAM,KAAK,GAAiB;gCACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;gCAC/D,QAAQ,EAAE,YAAY,GAAG,YAAY;gCACrC,MAAM,EAAE,YAAY,GAAG,CAAC;gCACxB,KAAK,EAAE,CAAC;gCACR,UAAU,EAAE,YAAY;gCACxB,MAAM,EAAE,KAAK;6BAChB,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;4BAC9B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;4BACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;wBAC/B,CAAC;wBAED,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;4BACjD,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;4BAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;4BACzD,MAAM,KAAK,GAAiB;gCACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;gCAC/D,QAAQ,EAAE,UAAU,GAAG,CAAC;gCACxB,MAAM,EAAE,UAAU,GAAG,WAAW;gCAChC,KAAK,EAAE,CAAC;gCACR,UAAU,EAAE,WAAW;gCACvB,MAAM,EAAE,KAAK;6BAChB,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;4BAC3B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;4BACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;wBAC/B,CAAC;oBACL,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;YAED,MAAM,uBAAuB,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBACnD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,KAAK;iBAChB,CAAC;YACN,CAAC,CAAC,CAAC;YACH,MAAM,mBAAmB,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBACnD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,MAAM,EAAE,IAAI;iBACf,CAAC;YACN,CAAC,CAAC,CAAC;YAEH,iCAAiC;YACjC,OAAO,CAAC,GAAG,uBAAuB,EAAE,GAAG,mBAAmB,CAAC,CAAC;QAChE,CAAC;KAAA;IAEO,gBAAgB,CAAC,IAAY,EAAE,MAAc;QACjD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QACzF,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe,CAAC,IAAY,EAAE,MAAc;QAChD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;CAEJ;AA9UD,kDA8UC"}
@@ -0,0 +1,150 @@
1
+ import { IndexItem, IndexStats, MetadataFilter, MetadataTypes, QueryResult } from './types';
2
+ export interface CreateIndexConfig {
3
+ version: number;
4
+ deleteIfExists?: boolean;
5
+ metadata_config?: {
6
+ indexed?: string[];
7
+ };
8
+ }
9
+ /**
10
+ * Local vector index instance.
11
+ * @remarks
12
+ * This class is used to create, update, and query a local vector index.
13
+ * Each index is a folder on disk containing an index.json file and an optional set of metadata files.
14
+ */
15
+ export declare class LocalIndex<TMetadata extends Record<string, MetadataTypes> = Record<string, MetadataTypes>> {
16
+ private readonly _folderPath;
17
+ private readonly _indexName;
18
+ private _data?;
19
+ private _update?;
20
+ private _bm25Engine;
21
+ /**
22
+ * Creates a new instance of LocalIndex.
23
+ * @param folderPath Path to the index folder.
24
+ * @param indexName Optional name of the index file. Defaults to index.json.
25
+ */
26
+ constructor(folderPath: string, indexName?: string);
27
+ /**
28
+ * Path to the index folder.
29
+ */
30
+ get folderPath(): string;
31
+ /**
32
+ * Optional name of the index file.
33
+ */
34
+ get indexName(): string;
35
+ /**
36
+ * Begins an update to the index.
37
+ * @remarks
38
+ * This method loads the index into memory and prepares it for updates.
39
+ */
40
+ beginUpdate(): Promise<void>;
41
+ /**
42
+ * Cancels an update to the index.
43
+ * @remarks
44
+ * This method discards any changes made to the index since the update began.
45
+ */
46
+ cancelUpdate(): void;
47
+ /**
48
+ * Creates a new index.
49
+ * @remarks
50
+ * This method creates a new folder on disk containing an index.json file.
51
+ * @param config Index configuration.
52
+ */
53
+ createIndex(config?: CreateIndexConfig): Promise<void>;
54
+ /**
55
+ * Deletes the index.
56
+ * @remarks
57
+ * This method deletes the index folder from disk.
58
+ */
59
+ deleteIndex(): Promise<void>;
60
+ /**
61
+ * Deletes an item from the index.
62
+ * @param id ID of item to delete.
63
+ */
64
+ deleteItem(id: string): Promise<void>;
65
+ /**
66
+ * Ends an update to the index.
67
+ * @remarks
68
+ * This method saves the index to disk.
69
+ */
70
+ endUpdate(): Promise<void>;
71
+ /**
72
+ * Loads an index from disk and returns its stats.
73
+ * @returns Index stats.
74
+ */
75
+ getIndexStats(): Promise<IndexStats>;
76
+ /**
77
+ * Returns an item from the index given its ID.
78
+ * @param id ID of the item to retrieve.
79
+ * @returns Item or undefined if not found.
80
+ */
81
+ getItem<TItemMetadata extends TMetadata = TMetadata>(id: string): Promise<IndexItem<TItemMetadata> | undefined>;
82
+ /**
83
+ * Adds an item to the index.
84
+ * @remarks
85
+ * A new update is started if one is not already in progress. If an item with the same ID
86
+ * already exists, an error will be thrown.
87
+ * @param item Item to insert.
88
+ * @returns Inserted item.
89
+ */
90
+ insertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>>;
91
+ /**
92
+ * Adds a batch of items to the index.
93
+ * @remarks
94
+ * Batch update requires no update to be in progress. This is necessary so that if any one
95
+ * insert operation fails, the entire update can be safely cancelled. This prevents partial
96
+ * updates from being applied to the local index.
97
+ * @param items Items to insert.
98
+ * @returns Inserted items.
99
+ */
100
+ batchInsertItems<TItemMetadata extends TMetadata = TMetadata>(items: Partial<IndexItem<TItemMetadata>>[]): Promise<IndexItem[]>;
101
+ /**
102
+ * Returns true if the index exists.
103
+ */
104
+ isIndexCreated(): Promise<boolean>;
105
+ /**
106
+ * Returns all items in the index.
107
+ * @remarks
108
+ * This method loads the index into memory and returns all its items. A copy of the items
109
+ * array is returned so no modifications should be made to the array.
110
+ * @returns Array of all items in the index.
111
+ */
112
+ listItems<TItemMetadata extends TMetadata = TMetadata>(): Promise<IndexItem<TItemMetadata>[]>;
113
+ /**
114
+ * Returns all items in the index matching the filter.
115
+ * @remarks
116
+ * This method loads the index into memory and returns all its items matching the filter.
117
+ * @param filter Filter to apply.
118
+ * @returns Array of items matching the filter.
119
+ */
120
+ listItemsByMetadata<TItemMetadata extends TMetadata = TMetadata>(filter: MetadataFilter): Promise<IndexItem<TItemMetadata>[]>;
121
+ /**
122
+ * Finds the top k items in the index that are most similar to the vector.
123
+ * @remarks
124
+ * This method loads the index into memory and returns the top k items that are most similar.
125
+ * An optional filter can be applied to the metadata of the items.
126
+ * @param vector Vector to query against.
127
+ * @param query Query text used for BM25 keyword search.
128
+ * @param topK Number of items to return.
129
+ * @param filter Optional. Filter to apply.
130
+ * @returns Similar items to the vector that matche the supplied filter.
131
+ */
132
+ queryItems<TItemMetadata extends TMetadata = TMetadata>(vector: number[], query: string, topK: number, filter?: MetadataFilter, isBm25?: boolean): Promise<QueryResult<TItemMetadata>[]>;
133
+ /**
134
+ * Adds or replaces an item in the index.
135
+ * @remarks
136
+ * A new update is started if one is not already in progress. If an item with the same ID
137
+ * already exists, it will be replaced.
138
+ * @param item Item to insert or replace.
139
+ * @returns Upserted item.
140
+ */
141
+ upsertItem<TItemMetadata extends TMetadata = TMetadata>(item: Partial<IndexItem<TItemMetadata>>): Promise<IndexItem<TItemMetadata>>;
142
+ /**
143
+ * Ensures that the index has been loaded into memory.
144
+ */
145
+ protected loadIndexData(): Promise<void>;
146
+ private addItemToUpdate;
147
+ private setupbm25;
148
+ private bm25Search;
149
+ }
150
+ //# sourceMappingURL=LocalIndex.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM5F,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED;;;;;GAKG;AACH,qBAAa,UAAU,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B,OAAO,CAAC,WAAW,CAAM;IAEzB;;;;OAIG;gBACgB,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAKzD;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,MAAM,CAE7B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,SAAS,CAAC;IAK5H;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;;;;;;;OAQG;IACU,gBAAgB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAiB5I;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1G;;;;;;OAMG;IACU,mBAAmB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1I;;;;;;;;;OASG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;IAuErM;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;OAEG;cACa,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAahC,eAAe;YA+Df,SAAS;YAsBT,UAAU;CAQ3B"}
1
+ {"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAM5F,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED;;;;;GAKG;AACH,qBAAa,UAAU,CAAC,SAAS,SAAS,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjG,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B,OAAO,CAAC,WAAW,CAAM;IAEzB;;;;OAIG;gBACgB,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAKzD;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,MAAM,CAE7B;IAED;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,SAAS,CAAC;IAK5H;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;;;;;;;OAQG;IACU,gBAAgB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAiB5I;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1G;;;;;;OAMG;IACU,mBAAmB,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC;IAK1I;;;;;;;;;;OAUG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,CAAC;IAuErM;;;;;;;OAOG;IACU,UAAU,CAAC,aAAa,SAAS,SAAS,GAAG,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAWhJ;;OAEG;cACa,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;YAahC,eAAe;YA+Df,SAAS;YAsBT,UAAU;CAQ3B"}