vectra 0.5.5 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ItemSelector.d.ts.map +1 -1
- package/lib/ItemSelector.js +7 -1
- package/lib/ItemSelector.js.map +1 -1
- package/lib/LocalDocument.d.ts +40 -2
- package/lib/LocalDocument.d.ts.map +1 -1
- package/lib/LocalDocument.js +50 -3
- package/lib/LocalDocument.js.map +1 -1
- package/lib/LocalDocumentIndex.d.ts +79 -2
- package/lib/LocalDocumentIndex.d.ts.map +1 -1
- package/lib/LocalDocumentIndex.js +90 -19
- package/lib/LocalDocumentIndex.js.map +1 -1
- package/lib/LocalDocumentResult.d.ts +32 -1
- package/lib/LocalDocumentResult.d.ts.map +1 -1
- package/lib/LocalDocumentResult.js +105 -23
- package/lib/LocalDocumentResult.js.map +1 -1
- package/lib/LocalIndex.d.ts +18 -18
- package/lib/LocalIndex.js +18 -18
- package/lib/TextSplitter.d.ts.map +1 -1
- package/lib/TextSplitter.js +1 -0
- package/lib/TextSplitter.js.map +1 -1
- package/package.json +1 -1
- package/src/ItemSelector.ts +6 -2
- package/src/LocalDocument.ts +50 -5
- package/src/LocalDocumentIndex.ts +124 -16
- package/src/LocalDocumentResult.ts +110 -23
- package/src/LocalIndex.ts +18 -18
- package/src/TextSplitter.ts +1 -0
|
@@ -11,9 +11,16 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.LocalDocumentResult = void 0;
|
|
13
13
|
const LocalDocument_1 = require("./LocalDocument");
|
|
14
|
+
/**
|
|
15
|
+
* Represents a search result for a document stored on disk.
|
|
16
|
+
*/
|
|
14
17
|
class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
15
|
-
|
|
16
|
-
|
|
18
|
+
/**
|
|
19
|
+
* @private
|
|
20
|
+
* Internal constructor for `LocalDocumentResult` instances.
|
|
21
|
+
*/
|
|
22
|
+
constructor(index, id, uri, chunks, tokenizer) {
|
|
23
|
+
super(index, id, uri);
|
|
17
24
|
this._chunks = chunks;
|
|
18
25
|
this._tokenizer = tokenizer;
|
|
19
26
|
// Compute average score
|
|
@@ -21,26 +28,104 @@ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
|
21
28
|
this._chunks.forEach(chunk => score += chunk.score);
|
|
22
29
|
this._score = score / this._chunks.length;
|
|
23
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Returns the chunks of the document that matched the query.
|
|
33
|
+
*/
|
|
24
34
|
get chunks() {
|
|
25
35
|
return this._chunks;
|
|
26
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Returns the average score of the document result.
|
|
39
|
+
*/
|
|
27
40
|
get score() {
|
|
28
41
|
return this._score;
|
|
29
42
|
}
|
|
30
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Renders all of the results chunks as spans of text (sections.)
|
|
45
|
+
* @remarks
|
|
46
|
+
* The returned sections will be sorted by document order and limited to maxTokens in length.
|
|
47
|
+
* @param maxTokens Maximum number of tokens per section.
|
|
48
|
+
* @returns Array of rendered text sections.
|
|
49
|
+
*/
|
|
50
|
+
renderAllSections(maxTokens) {
|
|
31
51
|
return __awaiter(this, void 0, void 0, function* () {
|
|
32
52
|
// Load text from disk
|
|
33
53
|
const text = yield this.loadText();
|
|
34
|
-
//
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
54
|
+
// Add chunks to a temp array and split any chunks that are longer than maxTokens.
|
|
55
|
+
const chunks = [];
|
|
56
|
+
for (let i = 0; i < this._chunks.length; i++) {
|
|
57
|
+
const chunk = this._chunks[i];
|
|
58
|
+
const startPos = chunk.item.metadata.startPos;
|
|
59
|
+
const endPos = chunk.item.metadata.endPos;
|
|
60
|
+
const chunkText = text.substring(startPos, endPos + 1);
|
|
61
|
+
const tokens = this._tokenizer.encode(chunkText);
|
|
62
|
+
let offset = 0;
|
|
63
|
+
while (offset < tokens.length) {
|
|
64
|
+
const chunkLength = Math.min(maxTokens, tokens.length - offset);
|
|
65
|
+
chunks.push({
|
|
66
|
+
text: this._tokenizer.decode(tokens.slice(offset, offset + chunkLength)),
|
|
67
|
+
startPos: startPos + offset,
|
|
68
|
+
endPos: startPos + offset + chunkLength - 1,
|
|
69
|
+
score: chunk.score,
|
|
70
|
+
tokenCount: chunkLength
|
|
71
|
+
});
|
|
72
|
+
offset += chunkLength;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// Sort chunks by startPos
|
|
76
|
+
const sorted = chunks.sort((a, b) => a.startPos - b.startPos);
|
|
77
|
+
// Generate sections
|
|
78
|
+
const sections = [];
|
|
79
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
80
|
+
const chunk = sorted[i];
|
|
81
|
+
let section = sections[sections.length - 1];
|
|
82
|
+
if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
|
|
83
|
+
section = {
|
|
84
|
+
chunks: [],
|
|
85
|
+
score: 0,
|
|
86
|
+
tokenCount: 0
|
|
87
|
+
};
|
|
88
|
+
sections.push(section);
|
|
43
89
|
}
|
|
90
|
+
section.chunks.push(chunk);
|
|
91
|
+
section.score += chunk.score;
|
|
92
|
+
section.tokenCount += chunk.tokenCount;
|
|
93
|
+
}
|
|
94
|
+
// Normalize section scores
|
|
95
|
+
sections.forEach(section => section.score /= section.chunks.length);
|
|
96
|
+
// Return final rendered sections
|
|
97
|
+
return sections.map(section => {
|
|
98
|
+
let text = '';
|
|
99
|
+
section.chunks.forEach(chunk => text += chunk.text);
|
|
100
|
+
return {
|
|
101
|
+
text: text,
|
|
102
|
+
tokenCount: section.tokenCount,
|
|
103
|
+
score: section.score
|
|
104
|
+
};
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Renders the top spans of text (sections) of the document based on the query result.
|
|
110
|
+
* @remarks
|
|
111
|
+
* The returned sections will be sorted by relevance and limited to the top `maxSections`.
|
|
112
|
+
* @param maxTokens Maximum number of tokens per section.
|
|
113
|
+
* @param maxSections Maximum number of sections to return.
|
|
114
|
+
* @param overlappingChunks Optional. If true, overlapping chunks of text will be added to each section until the maxTokens is reached.
|
|
115
|
+
* @returns Array of rendered text sections.
|
|
116
|
+
*/
|
|
117
|
+
renderSections(maxTokens, maxSections, overlappingChunks = true) {
|
|
118
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
119
|
+
// Load text from disk
|
|
120
|
+
const text = yield this.loadText();
|
|
121
|
+
// First check to see if the entire document is shorter than maxTokens
|
|
122
|
+
const length = yield this.getLength();
|
|
123
|
+
if (length <= maxTokens) {
|
|
124
|
+
return [{
|
|
125
|
+
text,
|
|
126
|
+
tokenCount: length,
|
|
127
|
+
score: 1.0
|
|
128
|
+
}];
|
|
44
129
|
}
|
|
45
130
|
// Otherwise, we need to split the document into sections
|
|
46
131
|
// - Add each chunk to a temp array and filter out any chunk that's longer then maxTokens.
|
|
@@ -77,24 +162,21 @@ class LocalDocumentResult extends LocalDocument_1.LocalDocument {
|
|
|
77
162
|
}];
|
|
78
163
|
}
|
|
79
164
|
// Generate sections
|
|
80
|
-
const sections = [
|
|
81
|
-
chunks: [],
|
|
82
|
-
score: 0,
|
|
83
|
-
tokenCount: 0
|
|
84
|
-
}];
|
|
165
|
+
const sections = [];
|
|
85
166
|
for (let i = 0; i < chunks.length; i++) {
|
|
86
167
|
const chunk = chunks[i];
|
|
87
168
|
let section = sections[sections.length - 1];
|
|
88
|
-
if (section.tokenCount + chunk.tokenCount > maxTokens) {
|
|
89
|
-
|
|
169
|
+
if (!section || section.tokenCount + chunk.tokenCount > maxTokens) {
|
|
170
|
+
section = {
|
|
90
171
|
chunks: [],
|
|
91
172
|
score: 0,
|
|
92
173
|
tokenCount: 0
|
|
93
|
-
}
|
|
174
|
+
};
|
|
175
|
+
sections.push(section);
|
|
94
176
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
177
|
+
section.chunks.push(chunk);
|
|
178
|
+
section.score += chunk.score;
|
|
179
|
+
section.tokenCount += chunk.tokenCount;
|
|
98
180
|
}
|
|
99
181
|
// Normalize section scores
|
|
100
182
|
sections.forEach(section => section.score /= section.chunks.length);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;
|
|
1
|
+
{"version":3,"file":"LocalDocumentResult.js","sourceRoot":"","sources":["../src/LocalDocumentResult.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mDAAgD;AAIhD;;GAEG;AACH,MAAa,mBAAoB,SAAQ,6BAAa;IAKlD;;;OAGG;IACH,YAAmB,KAAyB,EAAE,EAAU,EAAE,GAAW,EAAE,MAA4C,EAAE,SAAoB;QACrI,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,wBAAwB;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;OAMG;IACU,iBAAiB,CAAC,SAAiB;;YAC5C,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,kFAAkF;YAClF,MAAM,MAAM,GAAoB,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,IAAI,MAAM,GAAG,CAAC,CAAC;gBACf,OAAO,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE;oBAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;oBAChE,MAAM,CAAC,IAAI,CAAC;wBACR,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;wBACxE,QAAQ,EAAE,QAAQ,GAAG,MAAM;wBAC3B,MAAM,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,GAAG,CAAC;wBAC3C,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,UAAU,EAAE,WAAW;qBAC1B,CAAC,CAAC;oBACH,MAAM,IAAI,WAAW,CAAC;iBACzB;aACJ;YAED,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE9D,oBAAoB;YACpB,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE;oBAC/D,OAAO,GAAG;wBACN,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC;oBACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC1B;gBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;aAC1C;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACvB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAED;;;;;;;;OAQG;IACU,cAAc,CAAC,SAAiB,EAAE,WAAmB,EAAE,iBAAiB,GAAG,IAAI;;YACxF,sBAAsB;YACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YAEnC,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,OAAO,CAAC;wBACJ,IAAI;wBACJ,UAAU,EAAE,MAAM;wBAClB,KAAK,EAAE,GAAG;qBACb,CAAC,CAAC;aACN;YAED,yDAAyD;YACzD,0FAA0F;YAC1F,oEAAoE;YACpE,0GAA0G;YAC1G,sGAAsG;YACtG,yDAAyD;YACzD,gEAAgE;YAChE,+FAA+F;YAC/F,MAAM,MAAM,GAAmB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;gBACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,OAAO;oBACH,IAAI,EAAE,SAAS;oBACf,QAAQ;oBACR,MAAM;oBACN,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM;iBACvD,CAAC;YACN,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAE1F,sBAAsB;YACtB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBACrB,qDAAqD;gBACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjD,OAAO,CAAC;wBACJ,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;wBACxD,UAAU,EAAE,SAAS;wBACrB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACxB,CAAC,CAAC;aACN;YAED,oBAAoB;YACpB,MAAM,QAAQ,GAAc,EAAE,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC5C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,GAAG,SAAS,EAAE;oBAC/D,OAAO,GAAG;wBACN,MAAM,EAAE,EAAE;wBACV,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;qBAChB,CAAC;oBACF,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC1B;gBACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;gBAC7B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;aAC1C;YAED,2BAA2B;YAC3B,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEpE,kDAAkD;YAClD,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,QAAQ,CAAC,MAAM,GAAG,WAAW,EAAE;gBAC/B,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;aAC/D;YAED,kCAAkC;YAClC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;oBAChD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBAChC,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,KAAK,SAAS,CAAC,QAAQ,EAAE;wBACzC,KAAK,CAAC,IAAI,IAAI,SAAS,CAAC,IAAI,CAAC;wBAC7B,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;wBAChC,KAAK,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;wBACzC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;wBAChC,CAAC,EAAE,CAAC;qBACP;iBACJ;YACL,CAAC,CAAC,CAAC;YAEH,gFAAgF;YAChF,IAAI,iBAAiB,EAAE;gBACnB,MAAM,SAAS,GAAiB;oBAC5B,IAAI,EAAE,aAAa;oBACnB,QAAQ,EAAE,CAAC,CAAC;oBACZ,MAAM,EAAE,CAAC,CAAC;oBACV,KAAK,EAAE,CAAC;oBACR,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM;iBAC3D,CAAC;gBACF,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;oBACvB,mCAAmC;oBACnC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;wBAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;4BAChD,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;4BAC3C,OAAO,CAAC,UAAU,IAAI,SAAS,CAAC,UAAU,CAAC;4BAC3C,CAAC,EAAE,CAAC;yBACP;qBACJ;oBAED,4EAA4E;oBAC5E,IAAI,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC;oBAC5C,IAAI,MAAM,GAAG,EAAE,EAAE;wBACb,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;wBAChD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;wBACpE,IAAI,YAAY,GAAG,CAAC,EAAE;4BAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;4BAChE,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC;4BAC3E,MAAM,YAAY,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;4BAC/I,MAAM,KAAK,GAAiB;gCACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;gCAC/D,QAAQ,EAAE,YAAY,GAAG,YAAY;gCACrC,MAAM,EAAE,YAAY,GAAG,CAAC;gCACxB,KAAK,EAAE,CAAC;gCACR,UAAU,EAAE,YAAY;6BAC3B,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;4BAC9B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;4BACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;yBAC9B;wBAED,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;4BAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;4BACjD,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;4BAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;4BACzD,MAAM,KAAK,GAAiB;gCACxB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;gCAC/D,QAAQ,EAAE,UAAU,GAAG,CAAC;gCACxB,MAAM,EAAE,UAAU,GAAG,WAAW;gCAChC,KAAK,EAAE,CAAC;gCACR,UAAU,EAAE,WAAW;6BAC1B,CAAC;4BACF,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;4BAC3B,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;4BACvC,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;yBAC9B;qBACJ;gBACL,CAAC,CAAC,CAAC;aACN;YAED,iCAAiC;YACjC,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;gBAC1B,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,OAAO;oBACH,IAAI,EAAE,IAAI;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACvB,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC;KAAA;IAEO,gBAAgB,CAAC,IAAY,EAAE,MAAc;QACjD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QACzF,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe,CAAC,IAAY,EAAE,MAAc;QAChD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;CAEJ;AA/RD,kDA+RC"}
|
package/lib/LocalIndex.d.ts
CHANGED
|
@@ -19,8 +19,8 @@ export declare class LocalIndex {
|
|
|
19
19
|
private _update?;
|
|
20
20
|
/**
|
|
21
21
|
* Creates a new instance of LocalIndex.
|
|
22
|
-
* @param folderPath
|
|
23
|
-
* @param indexName
|
|
22
|
+
* @param folderPath Path to the index folder.
|
|
23
|
+
* @param indexName Optional name of the index file. Defaults to index.json.
|
|
24
24
|
*/
|
|
25
25
|
constructor(folderPath: string, indexName?: string);
|
|
26
26
|
/**
|
|
@@ -47,7 +47,7 @@ export declare class LocalIndex {
|
|
|
47
47
|
* Creates a new index.
|
|
48
48
|
* @remarks
|
|
49
49
|
* This method creates a new folder on disk containing an index.json file.
|
|
50
|
-
* @param config
|
|
50
|
+
* @param config Index configuration.
|
|
51
51
|
*/
|
|
52
52
|
createIndex(config?: CreateIndexConfig): Promise<void>;
|
|
53
53
|
/**
|
|
@@ -58,7 +58,7 @@ export declare class LocalIndex {
|
|
|
58
58
|
deleteIndex(): Promise<void>;
|
|
59
59
|
/**
|
|
60
60
|
* Deletes an item from the index.
|
|
61
|
-
* @param id
|
|
61
|
+
* @param id ID of item to delete.
|
|
62
62
|
*/
|
|
63
63
|
deleteItem(id: string): Promise<void>;
|
|
64
64
|
/**
|
|
@@ -69,13 +69,13 @@ export declare class LocalIndex {
|
|
|
69
69
|
endUpdate(): Promise<void>;
|
|
70
70
|
/**
|
|
71
71
|
* Loads an index from disk and returns its stats.
|
|
72
|
-
* @returns Index stats
|
|
72
|
+
* @returns Index stats.
|
|
73
73
|
*/
|
|
74
74
|
getIndexStats(): Promise<IndexStats>;
|
|
75
75
|
/**
|
|
76
76
|
* Returns an item from the index given its ID.
|
|
77
|
-
* @param id
|
|
78
|
-
* @returns Item or undefined if not found
|
|
77
|
+
* @param id ID of the item to retrieve.
|
|
78
|
+
* @returns Item or undefined if not found.
|
|
79
79
|
*/
|
|
80
80
|
getItem<TMetadata = Record<string, MetadataTypes>>(id: string): Promise<IndexItem<TMetadata> | undefined>;
|
|
81
81
|
/**
|
|
@@ -83,8 +83,8 @@ export declare class LocalIndex {
|
|
|
83
83
|
* @remarks
|
|
84
84
|
* A new update is started if one is not already in progress. If an item with the same ID
|
|
85
85
|
* already exists, an error will be thrown.
|
|
86
|
-
* @param item Item to insert
|
|
87
|
-
* @returns Inserted item
|
|
86
|
+
* @param item Item to insert.
|
|
87
|
+
* @returns Inserted item.
|
|
88
88
|
*/
|
|
89
89
|
insertItem<TMetadata = Record<string, MetadataTypes>>(item: Partial<IndexItem<TMetadata>>): Promise<IndexItem<TMetadata>>;
|
|
90
90
|
/**
|
|
@@ -96,15 +96,15 @@ export declare class LocalIndex {
|
|
|
96
96
|
* @remarks
|
|
97
97
|
* This method loads the index into memory and returns all its items. A copy of the items
|
|
98
98
|
* array is returned so no modifications should be made to the array.
|
|
99
|
-
* @returns
|
|
99
|
+
* @returns Array of all items in the index.
|
|
100
100
|
*/
|
|
101
101
|
listItems<TMetadata = Record<string, MetadataTypes>>(): Promise<IndexItem<TMetadata>[]>;
|
|
102
102
|
/**
|
|
103
103
|
* Returns all items in the index matching the filter.
|
|
104
104
|
* @remarks
|
|
105
105
|
* This method loads the index into memory and returns all its items matching the filter.
|
|
106
|
-
* @param filter Filter to apply
|
|
107
|
-
* @returns
|
|
106
|
+
* @param filter Filter to apply.
|
|
107
|
+
* @returns Array of items matching the filter.
|
|
108
108
|
*/
|
|
109
109
|
listItemsByMetadata<TMetadata = Record<string, MetadataTypes>>(filter: MetadataFilter): Promise<IndexItem<TMetadata>[]>;
|
|
110
110
|
/**
|
|
@@ -112,10 +112,10 @@ export declare class LocalIndex {
|
|
|
112
112
|
* @remarks
|
|
113
113
|
* This method loads the index into memory and returns the top k items that are most similar.
|
|
114
114
|
* An optional filter can be applied to the metadata of the items.
|
|
115
|
-
* @param vector Vector to query against
|
|
116
|
-
* @param topK Number of items to return
|
|
117
|
-
* @param filter Optional
|
|
118
|
-
* @returns Similar items to the vector that
|
|
115
|
+
* @param vector Vector to query against.
|
|
116
|
+
* @param topK Number of items to return.
|
|
117
|
+
* @param filter Optional. Filter to apply.
|
|
118
|
+
* @returns Similar items to the vector that matche the supplied filter.
|
|
119
119
|
*/
|
|
120
120
|
queryItems<TMetadata = Record<string, MetadataTypes>>(vector: number[], topK: number, filter?: MetadataFilter): Promise<QueryResult<TMetadata>[]>;
|
|
121
121
|
/**
|
|
@@ -123,8 +123,8 @@ export declare class LocalIndex {
|
|
|
123
123
|
* @remarks
|
|
124
124
|
* A new update is started if one is not already in progress. If an item with the same ID
|
|
125
125
|
* already exists, it will be replaced.
|
|
126
|
-
* @param item Item to insert or replace
|
|
127
|
-
* @returns Upserted item
|
|
126
|
+
* @param item Item to insert or replace.
|
|
127
|
+
* @returns Upserted item.
|
|
128
128
|
*/
|
|
129
129
|
upsertItem<TMetadata = Record<string, MetadataTypes>>(item: Partial<IndexItem<TMetadata>>): Promise<IndexItem<TMetadata>>;
|
|
130
130
|
/**
|
package/lib/LocalIndex.js
CHANGED
|
@@ -46,8 +46,8 @@ const ItemSelector_1 = require("./ItemSelector");
|
|
|
46
46
|
class LocalIndex {
|
|
47
47
|
/**
|
|
48
48
|
* Creates a new instance of LocalIndex.
|
|
49
|
-
* @param folderPath
|
|
50
|
-
* @param indexName
|
|
49
|
+
* @param folderPath Path to the index folder.
|
|
50
|
+
* @param indexName Optional name of the index file. Defaults to index.json.
|
|
51
51
|
*/
|
|
52
52
|
constructor(folderPath, indexName) {
|
|
53
53
|
this._folderPath = folderPath;
|
|
@@ -91,7 +91,7 @@ class LocalIndex {
|
|
|
91
91
|
* Creates a new index.
|
|
92
92
|
* @remarks
|
|
93
93
|
* This method creates a new folder on disk containing an index.json file.
|
|
94
|
-
* @param config
|
|
94
|
+
* @param config Index configuration.
|
|
95
95
|
*/
|
|
96
96
|
createIndex(config = { version: 1 }) {
|
|
97
97
|
var _a;
|
|
@@ -136,7 +136,7 @@ class LocalIndex {
|
|
|
136
136
|
}
|
|
137
137
|
/**
|
|
138
138
|
* Deletes an item from the index.
|
|
139
|
-
* @param id
|
|
139
|
+
* @param id ID of item to delete.
|
|
140
140
|
*/
|
|
141
141
|
deleteItem(id) {
|
|
142
142
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -179,7 +179,7 @@ class LocalIndex {
|
|
|
179
179
|
}
|
|
180
180
|
/**
|
|
181
181
|
* Loads an index from disk and returns its stats.
|
|
182
|
-
* @returns Index stats
|
|
182
|
+
* @returns Index stats.
|
|
183
183
|
*/
|
|
184
184
|
getIndexStats() {
|
|
185
185
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -193,8 +193,8 @@ class LocalIndex {
|
|
|
193
193
|
}
|
|
194
194
|
/**
|
|
195
195
|
* Returns an item from the index given its ID.
|
|
196
|
-
* @param id
|
|
197
|
-
* @returns Item or undefined if not found
|
|
196
|
+
* @param id ID of the item to retrieve.
|
|
197
|
+
* @returns Item or undefined if not found.
|
|
198
198
|
*/
|
|
199
199
|
getItem(id) {
|
|
200
200
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -207,8 +207,8 @@ class LocalIndex {
|
|
|
207
207
|
* @remarks
|
|
208
208
|
* A new update is started if one is not already in progress. If an item with the same ID
|
|
209
209
|
* already exists, an error will be thrown.
|
|
210
|
-
* @param item Item to insert
|
|
211
|
-
* @returns Inserted item
|
|
210
|
+
* @param item Item to insert.
|
|
211
|
+
* @returns Inserted item.
|
|
212
212
|
*/
|
|
213
213
|
insertItem(item) {
|
|
214
214
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -242,7 +242,7 @@ class LocalIndex {
|
|
|
242
242
|
* @remarks
|
|
243
243
|
* This method loads the index into memory and returns all its items. A copy of the items
|
|
244
244
|
* array is returned so no modifications should be made to the array.
|
|
245
|
-
* @returns
|
|
245
|
+
* @returns Array of all items in the index.
|
|
246
246
|
*/
|
|
247
247
|
listItems() {
|
|
248
248
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -254,8 +254,8 @@ class LocalIndex {
|
|
|
254
254
|
* Returns all items in the index matching the filter.
|
|
255
255
|
* @remarks
|
|
256
256
|
* This method loads the index into memory and returns all its items matching the filter.
|
|
257
|
-
* @param filter Filter to apply
|
|
258
|
-
* @returns
|
|
257
|
+
* @param filter Filter to apply.
|
|
258
|
+
* @returns Array of items matching the filter.
|
|
259
259
|
*/
|
|
260
260
|
listItemsByMetadata(filter) {
|
|
261
261
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -268,10 +268,10 @@ class LocalIndex {
|
|
|
268
268
|
* @remarks
|
|
269
269
|
* This method loads the index into memory and returns the top k items that are most similar.
|
|
270
270
|
* An optional filter can be applied to the metadata of the items.
|
|
271
|
-
* @param vector Vector to query against
|
|
272
|
-
* @param topK Number of items to return
|
|
273
|
-
* @param filter Optional
|
|
274
|
-
* @returns Similar items to the vector that
|
|
271
|
+
* @param vector Vector to query against.
|
|
272
|
+
* @param topK Number of items to return.
|
|
273
|
+
* @param filter Optional. Filter to apply.
|
|
274
|
+
* @returns Similar items to the vector that matche the supplied filter.
|
|
275
275
|
*/
|
|
276
276
|
queryItems(vector, topK, filter) {
|
|
277
277
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -314,8 +314,8 @@ class LocalIndex {
|
|
|
314
314
|
* @remarks
|
|
315
315
|
* A new update is started if one is not already in progress. If an item with the same ID
|
|
316
316
|
* already exists, it will be replaced.
|
|
317
|
-
* @param item Item to insert or replace
|
|
318
|
-
* @returns Upserted item
|
|
317
|
+
* @param item Item to insert or replace.
|
|
318
|
+
* @returns Upserted item.
|
|
319
319
|
*/
|
|
320
320
|
upsertItem(item) {
|
|
321
321
|
return __awaiter(this, void 0, void 0, function* () {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAI/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA2BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,aAAa;
|
|
1
|
+
{"version":3,"file":"TextSplitter.d.ts","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAI/C,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,cAAc,EAAE,OAAO,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IA2BhD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE;IA4BvC,OAAO,CAAC,cAAc;IAoEtB,OAAO,CAAC,aAAa;IA8BrB,OAAO,CAAC,oBAAoB;IAS5B,OAAO,CAAC,aAAa;IAsBrB,OAAO,CAAC,aAAa;CA4WxB"}
|
package/lib/TextSplitter.js
CHANGED
package/lib/TextSplitter.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAGhD,MAAM,kBAAkB,GAAG,gEAAgE,CAAC;AAW5F,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC/B;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;YACjB,wBAAwB;YACxB,IAAI,KAAe,CAAC;YACpB,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,qBAAqB;gBACrB,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,KAAK,GAAG,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;aAC/E;iBAAM;gBACH,mBAAmB;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzC,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;aAC3D;YAED,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,6BAA6B;gBAC7B,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;oBACnC,SAAS;iBACZ;gBAED,qDAAqD;gBACrD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBAC3C,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,oBAAoB;oBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;wBACxC,qCAAqC;wBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;wBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;qBAC7B;yBAAM;wBACH,yBAAyB;wBACzB,MAAM,CAAC,IAAI,CAAC;4BACR,IAAI,EAAE,KAAK;4BACX,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,QAAQ;4BAClB,MAAM,EAAE,MAAM;4BACd,YAAY,EAAE,EAAE;4BAChB,UAAU,EAAE,EAAE;yBACjB,CAAC,CAAC;qBACN;iBAEJ;gBAGD,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,MAAmB;QACrC,MAAM,cAAc,GAAgB,EAAE,CAAC;QACvC,IAAI,YAAiC,CAAC;QACtC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,YAAY,EAAE;gBACd,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBAChE,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACjC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAClC,YAAY,GAAG,KAAK,CAAC;oBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACvC;qBAAM;oBACH,YAAY,CAAC,IAAI,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;oBAC5C,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBAC1C,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACxC;aACJ;iBAAM;gBACH,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;aACvC;SACJ;QACD,IAAI,YAAY,EAAE;YACd,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SACrC;QACD,OAAO,cAAc,CAAC;IAC1B,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;gBACtC,OAAO,IAAI,CAAC;aACf;SACJ;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,IAAY;QAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAClB,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACvF,IAAI,IAAI,GAAG,GAAG,QAAQ,CAAC;iBAC1B;qBAAM;oBACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjB,IAAI,GAAG,QAAQ,CAAC;iBACnB;aACJ;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpB;aAAM;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpB;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,gBAAgB;oBAChB,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,UAAU;oBACV,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAtiBD,oCAsiBC"}
|
|
1
|
+
{"version":3,"file":"TextSplitter.js","sourceRoot":"","sources":["../src/TextSplitter.ts"],"names":[],"mappings":";;;AAAA,mDAAgD;AAGhD,MAAM,kBAAkB,GAAG,gEAAgE,CAAC;AAW5F,MAAa,YAAY;IAGrB,YAAmB,MAAoC;QACnD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC;YACzB,cAAc,EAAE,KAAK;YACrB,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,EAAE;SACG,EAAE,MAAM,CAAC,CAAC;QAEjC,iDAAiD;QACjD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YACzB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,6BAAa,EAAE,CAAC;SAChD;QAED,8CAA8C;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE;YAClE,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;SACtE;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5B,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;SAC7C;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;SAChD;aAAM,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC3D,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;SACxD;IACL,CAAC;IAEM,KAAK,CAAC,IAAY;QACrB,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAErE,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,SAAS,gBAAgB,CAAC,MAAiB;YACvC,IAAI,MAAM,IAAI,SAAS,EAAE;gBACrB,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClG,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;aAC/B;iBAAM;gBACH,OAAO,EAAE,CAAC;aACb;QACL,CAAC;QAED,iEAAiE;QACjE,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACpC,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACxB,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACpE,KAAK,CAAC,YAAY,GAAG,gBAAgB,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChF,KAAK,CAAC,UAAU,GAAG,gBAAgB,CAAC,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAE,MAAM,CAAC,CAAC;aAC1D;SACJ;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,cAAc,CAAC,IAAY,EAAE,UAAoB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;YACjB,wBAAwB;YACxB,IAAI,KAAe,CAAC;YACpB,IAAI,SAAS,GAAG,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;gBACvB,qBAAqB;gBACrB,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC1B,KAAK,GAAG,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;aAC/E;iBAAM;gBACH,mBAAmB;gBACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACzC,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;aAC3D;YAED,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE3C,4BAA4B;gBAC5B,IAAI,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,CAAC,QAAQ,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACpF,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK,IAAI,SAAS,CAAC;iBACtB;gBAED,6BAA6B;gBAC7B,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;oBACnC,SAAS;iBACZ;gBAED,qDAAqD;gBACrD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBAC3C,qCAAqC;oBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;iBAC7B;qBAAM;oBACH,oBAAoB;oBACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACpD,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;wBACxC,qCAAqC;wBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;wBACvE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;qBAC7B;yBAAM;wBACH,yBAAyB;wBACzB,MAAM,CAAC,IAAI,CAAC;4BACR,IAAI,EAAE,KAAK;4BACX,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,QAAQ;4BAClB,MAAM,EAAE,MAAM;4BACd,YAAY,EAAE,EAAE;4BAChB,UAAU,EAAE,EAAE;yBACjB,CAAC,CAAC;qBACN;iBAEJ;gBAGD,kBAAkB;gBAClB,QAAQ,GAAG,MAAM,GAAG,CAAC,CAAC;aACzB;SACJ;QAED,OAAO,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,MAAmB;QACrC,MAAM,cAAc,GAAgB,EAAE,CAAC;QACvC,IAAI,YAAiC,CAAC;QACtC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,YAAY,EAAE;gBACd,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;gBAChE,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACjC,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAClC,YAAY,GAAG,KAAK,CAAC;oBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACvC;qBAAM;oBACH,YAAY,CAAC,IAAI,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;oBAC5C,YAAY,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;oBACnC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;oBAC1C,aAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;iBACxC;aACJ;iBAAM;gBACH,YAAY,GAAG,KAAK,CAAC;gBACrB,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;aACvC;SACJ;QACD,IAAI,YAAY,EAAE;YACd,cAAc,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SACrC;QACD,OAAO,cAAc,CAAC;IAC1B,CAAC;IAEO,oBAAoB,CAAC,IAAY;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;gBACtC,OAAO,IAAI,CAAC;aACf;SACJ;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,IAAY;QAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAClB,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;oBACvF,IAAI,IAAI,GAAG,GAAG,QAAQ,CAAC;iBAC1B;qBAAM;oBACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACjB,IAAI,GAAG,QAAQ,CAAC;iBACnB;aACJ;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpB;aAAM;YACH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACpB;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,aAAa,CAAC,OAAgB;QAClC,QAAQ,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,EAAE;YACnB,KAAK,KAAK;gBACN,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI;gBACL,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,QAAQ;oBACR,UAAU;oBACV,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI,CAAC;YACV,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,iCAAiC;oBACjC,WAAW;oBACX,cAAc;oBACd,YAAY;oBACZ,WAAW;oBACX,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,KAAK,CAAC;YACX,KAAK,YAAY;gBACb,OAAO;oBACH,sBAAsB;oBACtB,eAAe;oBACf,eAAe;oBACf,gBAAgB;oBAChB,gCAAgC;oBAChC,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,UAAU;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,YAAY;oBACZ,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,mCAAmC;oBACnC,aAAa;oBACb,gCAAgC;oBAChC,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,YAAY;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,kCAAkC;oBAClC,YAAY;oBACZ,kCAAkC;oBAClC,YAAY;oBACZ,+BAA+B;oBAC/B,SAAS;oBACT,iCAAiC;oBACjC,WAAW;oBACX,gCAAgC;oBAChC,WAAW;oBACX,kCAAkC;oBAClC,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,QAAQ,CAAC;YACd,KAAK,IAAI;gBACL,OAAO;oBACH,8CAA8C;oBAC9C,UAAU;oBACV,QAAQ;oBACR,UAAU;oBACV,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,6BAA6B;oBAC7B,SAAS;oBACT,SAAS;oBACT,SAAS;oBACT,gCAAgC;oBAChC,OAAO;oBACP,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,iCAAiC;oBACjC,QAAQ;oBACR,UAAU;oBACV,sCAAsC;oBACtC,OAAO;oBACP,WAAW;oBACX,UAAU;oBACV,QAAQ;oBACR,OAAO;oBACP,UAAU;oBACV,WAAW;oBACX,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,mCAAmC;oBACnC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,iCAAiC;oBACjC,QAAQ;oBACR,QAAQ;oBACR,QAAQ;oBACR,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,UAAU;oBACV,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,mCAAmC;oBACnC,SAAS;oBACT,gCAAgC;oBAChC,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,IAAI,CAAC;YACV,KAAK,UAAU;gBACX,OAAO;oBACH,sEAAsE;oBACtE,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,WAAW;oBACX,uEAAuE;oBACvE,kBAAkB;oBAClB,kBAAkB;oBAClB,oBAAoB;oBACpB,SAAS;oBACT,mBAAmB;oBACnB,aAAa;oBACb,aAAa;oBACb,aAAa;oBACb,kEAAkE;oBAClE,kEAAkE;oBAClE,gBAAgB;oBAChB,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,UAAU;oBACV,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,OAAO;gBACR,OAAO;oBACH,2CAA2C;oBAC3C,cAAc;oBACd,cAAc;oBACd,iBAAiB;oBACjB,oBAAoB;oBAEpB,4BAA4B;oBAC5B,sBAAsB;oBACtB,oBAAoB;oBACpB,wBAAwB;oBACxB,iBAAiB;oBACjB,kBAAkB;oBAClB,sBAAsB;oBACtB,kBAAkB;oBAClB,qBAAqB;oBAErB,iCAAiC;oBACjC,kBAAkB;oBAClB,IAAI;oBACJ,GAAG;oBAEH,wCAAwC;oBACxC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN,KAAK,MAAM;gBACP,OAAO;oBACH,sCAAsC;oBACtC,QAAQ;oBACR,OAAO;oBACP,KAAK;oBACL,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,MAAM;oBACN,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,OAAO;oBACP,QAAQ;oBACR,SAAS;oBACT,UAAU;oBACV,QAAQ;oBACR,SAAS;oBACT,uBAAuB;oBACvB,GAAG;iBACN,CAAC;YACN,KAAK,KAAK;gBACN,OAAO;oBACH,gDAAgD;oBAChD,WAAW;oBACX,UAAU;oBACV,mCAAmC;oBACnC,aAAa;oBACb,cAAc;oBACd,YAAY;oBACZ,iCAAiC;oBACjC,gBAAgB;oBAChB,SAAS;oBACT,aAAa;oBACb,UAAU;oBACV,aAAa;oBACb,UAAU;oBACV,WAAW;oBACX,SAAS;oBACT,sCAAsC;oBACtC,OAAO;oBACP,QAAQ;oBACR,UAAU;oBACV,aAAa;oBACb,aAAa;oBACb,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;iBACN,CAAC;YACN;gBACI,OAAO;oBACH,oCAAoC;oBACpC,MAAM;oBACN,IAAI;oBACJ,GAAG;oBACH,EAAE;iBACL,CAAC;SACT;IACL,CAAC;CACJ;AAviBD,oCAuiBC"}
|
package/package.json
CHANGED
package/src/ItemSelector.ts
CHANGED
|
@@ -140,8 +140,12 @@ export class ItemSelector {
|
|
|
140
140
|
}
|
|
141
141
|
break;
|
|
142
142
|
case '$in':
|
|
143
|
-
if (typeof value == 'boolean'
|
|
143
|
+
if (typeof value == 'boolean') {
|
|
144
144
|
return false;
|
|
145
|
+
} else if(typeof value == 'string' && !filter[key]!.includes(value)){
|
|
146
|
+
return false
|
|
147
|
+
} else if(!filter[key]!.some(val => typeof val == 'string' && val.includes(value as string))){
|
|
148
|
+
return false
|
|
145
149
|
}
|
|
146
150
|
break;
|
|
147
151
|
case '$nin':
|
|
@@ -155,4 +159,4 @@ export class ItemSelector {
|
|
|
155
159
|
}
|
|
156
160
|
return true;
|
|
157
161
|
}
|
|
158
|
-
}
|
|
162
|
+
}
|
package/src/LocalDocument.ts
CHANGED
|
@@ -1,32 +1,70 @@
|
|
|
1
1
|
import * as fs from 'fs/promises';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import { MetadataTypes } from './types';
|
|
4
|
+
import { LocalDocumentIndex } from './LocalDocumentIndex';
|
|
4
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Represents an indexed document stored on disk.
|
|
8
|
+
*/
|
|
5
9
|
export class LocalDocument {
|
|
6
|
-
private readonly
|
|
10
|
+
private readonly _index: LocalDocumentIndex;
|
|
7
11
|
private readonly _id: string;
|
|
8
12
|
private readonly _uri: string;
|
|
9
13
|
private _metadata: Record<string,MetadataTypes>|undefined;
|
|
10
14
|
private _text: string|undefined;
|
|
11
15
|
|
|
12
|
-
|
|
13
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Creates a new `LocalDocument` instance.
|
|
18
|
+
* @param index Parent index that contains the document.
|
|
19
|
+
* @param id ID of the document.
|
|
20
|
+
* @param uri URI of the document.
|
|
21
|
+
*/
|
|
22
|
+
public constructor(index: LocalDocumentIndex, id: string, uri: string) {
|
|
23
|
+
this._index = index;
|
|
14
24
|
this._id = id;
|
|
15
25
|
this._uri = uri;
|
|
16
26
|
}
|
|
17
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Returns the folder path where the document is stored.
|
|
30
|
+
*/
|
|
18
31
|
public get folderPath(): string {
|
|
19
|
-
return this.
|
|
32
|
+
return this._index.folderPath;
|
|
20
33
|
}
|
|
21
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Returns the ID of the document.
|
|
37
|
+
*/
|
|
22
38
|
public get id(): string {
|
|
23
39
|
return this._id;
|
|
24
40
|
}
|
|
25
41
|
|
|
42
|
+
/**
|
|
43
|
+
* Returns the URI of the document.
|
|
44
|
+
*/
|
|
26
45
|
public get uri(): string {
|
|
27
46
|
return this._uri;
|
|
28
47
|
}
|
|
29
48
|
|
|
49
|
+
/**
|
|
50
|
+
* Returns the length of the document in tokens.
|
|
51
|
+
* @remarks
|
|
52
|
+
* This value will be estimated for documents longer then 40k bytes.
|
|
53
|
+
* @returns Length of the document in tokens.
|
|
54
|
+
*/
|
|
55
|
+
public async getLength(): Promise<number> {
|
|
56
|
+
const text = await this.loadText();
|
|
57
|
+
if (text.length <= 40000) {
|
|
58
|
+
return this._index.tokenizer.encode(text).length;
|
|
59
|
+
} else {
|
|
60
|
+
return Math.ceil(text.length / 4);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Determines if the document has additional metadata storred on disk.
|
|
66
|
+
* @returns True if the document has metadata; otherwise, false.
|
|
67
|
+
*/
|
|
30
68
|
public async hasMetadata(): Promise<boolean> {
|
|
31
69
|
try {
|
|
32
70
|
await fs.access(path.join(this.folderPath, `${this.id}.json`));
|
|
@@ -36,6 +74,10 @@ export class LocalDocument {
|
|
|
36
74
|
}
|
|
37
75
|
}
|
|
38
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Loads the metadata for the document from disk.
|
|
79
|
+
* @returns Metadata for the document.
|
|
80
|
+
*/
|
|
39
81
|
public async loadMetadata(): Promise<Record<string,MetadataTypes>> {
|
|
40
82
|
if (this._metadata == undefined) {
|
|
41
83
|
let json: string;
|
|
@@ -55,6 +97,10 @@ export class LocalDocument {
|
|
|
55
97
|
return this._metadata!;
|
|
56
98
|
}
|
|
57
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Loads the text for the document from disk.
|
|
102
|
+
* @returns Text for the document.
|
|
103
|
+
*/
|
|
58
104
|
public async loadText(): Promise<string> {
|
|
59
105
|
if (this._text == undefined) {
|
|
60
106
|
try {
|
|
@@ -66,5 +112,4 @@ export class LocalDocument {
|
|
|
66
112
|
|
|
67
113
|
return this._text;
|
|
68
114
|
}
|
|
69
|
-
|
|
70
115
|
}
|