vectra 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +10 -0
- package/lib/ItemSelector.d.ts +41 -0
- package/lib/ItemSelector.d.ts.map +1 -0
- package/lib/ItemSelector.js +156 -0
- package/lib/ItemSelector.js.map +1 -0
- package/lib/LocalIndex.d.ts +184 -0
- package/lib/LocalIndex.d.ts.map +1 -0
- package/lib/LocalIndex.js +392 -0
- package/lib/LocalIndex.js.map +1 -0
- package/lib/index.d.ts +3 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +15 -0
- package/lib/index.js.map +1 -0
- package/lib/utilities.d.ts +5 -0
- package/lib/utilities.d.ts.map +1 -0
- package/lib/utilities.js +38 -0
- package/lib/utilities.js.map +1 -0
- package/package.json +51 -0
- package/src/ItemSelector.ts +158 -0
- package/src/LocalIndex.ts +450 -0
- package/src/index.ts +2 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Steven Ickman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# vectra
|
|
2
|
+
Vectra is a local vector database for Node.js with features similar to [Pinecone](https://www.pinecone.io/) or [Qdrant](https://qdrant.tech/) but built using local files. Each Vectra index is a folder on disk. There's an `index.json` file in the folder that contains all the vectors for the index along with any indexed metadata. When you create an index you can specify which metadata properties to index and only those fields will be stored in the `index.json` file. All of the other metadata for an item will be stored on disk in a separate file keyed by a GUID.
|
|
3
|
+
|
|
4
|
+
When queryng Vectra you'll be able to use the same subset of [Mongo DB query operators](https://www.mongodb.com/docs/manual/reference/operator/query/) that Pinecone supports and the results will be returned sorted by simularity. Every item in the index will first be filtered by metadata and then ranked for simularity. Even though every item is evaluated its all in memory so it should by nearly instantanious. Likely 1ms - 2ms for even a rather large index. Smaller indexes should be <1ms.
|
|
5
|
+
|
|
6
|
+
Keep in mind that your entire Vectra index is loaded into memory so it's not well suited for scenarios like long term chat bot memory. Use a really vector DB for that. Vectra is intended to be used in scenarios where you have a small corpus of mostly static data you'd like to include in your prompt. Infinite few shot examples would be a great use case for Vectra or even just a single document you want to ask questions over.
|
|
7
|
+
|
|
8
|
+
Pinecone style namespaces aren't directly supported but you could easily mimic them by creating a separate Vectra index (and folder) for each namespace.
|
|
9
|
+
|
|
10
|
+
Sample code and the actual code coming the week. GPT and I are still writting it :)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { MetadataFilter, MetadataTypes } from './LocalIndex';
|
|
2
|
+
export declare class ItemSelector {
|
|
3
|
+
/**
|
|
4
|
+
* Returns the similarity between two vectors using the cosine similarity.
|
|
5
|
+
* @param vector1 Vector 1
|
|
6
|
+
* @param vector2 Vector 2
|
|
7
|
+
* @returns Similarity between the two vectors
|
|
8
|
+
*/
|
|
9
|
+
static cosineSimilarity(vector1: number[], vector2: number[]): number;
|
|
10
|
+
/**
|
|
11
|
+
* Normalizes a vector.
|
|
12
|
+
* @remarks
|
|
13
|
+
* The norm of a vector is the square root of the sum of the squares of the elements.
|
|
14
|
+
* The LocalIndex pre-normalizes all vectors to improve performance.
|
|
15
|
+
* @param vector Vector to normalize
|
|
16
|
+
* @returns Normalized vector
|
|
17
|
+
*/
|
|
18
|
+
static normalize(vector: number[]): number;
|
|
19
|
+
/**
|
|
20
|
+
* Returns the similarity between two vectors using cosine similarity.
|
|
21
|
+
* @remarks
|
|
22
|
+
* The LocalIndex pre-normalizes all vectors to improve performance.
|
|
23
|
+
* This method uses the pre-calculated norms to improve performance.
|
|
24
|
+
* @param vector1 Vector 1
|
|
25
|
+
* @param norm1 Norm of vector 1
|
|
26
|
+
* @param vector2 Vector 2
|
|
27
|
+
* @param norm2 Norm of vector 2
|
|
28
|
+
* @returns Similarity between the two vectors
|
|
29
|
+
*/
|
|
30
|
+
static normalizedCosineSimilarity(vector1: number[], norm1: number, vector2: number[], norm2: number): number;
|
|
31
|
+
/**
|
|
32
|
+
* Applies a filter to the metadata of an item.
|
|
33
|
+
* @param metadata Metadata of the item
|
|
34
|
+
* @param filter Filter to apply
|
|
35
|
+
* @returns True if the item matches the filter, false otherwise
|
|
36
|
+
*/
|
|
37
|
+
static select(metadata: Record<string, MetadataTypes>, filter: MetadataFilter): boolean;
|
|
38
|
+
private static dotProduct;
|
|
39
|
+
private static metadataFilter;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=ItemSelector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ItemSelector.d.ts","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7D,qBAAa,YAAY;IACrB;;;;;OAKG;WACW,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE;IAKnE;;;;;;;OAOG;WACW,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE;IAYxC;;;;;;;;;;OAUG;WACW,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM;IAK3G;;;;;OAKG;WACW,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO;IAoC9F,OAAO,CAAC,MAAM,CAAC,UAAU;IAYzB,OAAO,CAAC,MAAM,CAAC,cAAc;CAqDhC"}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ItemSelector = void 0;
|
|
4
|
+
class ItemSelector {
|
|
5
|
+
/**
|
|
6
|
+
* Returns the similarity between two vectors using the cosine similarity.
|
|
7
|
+
* @param vector1 Vector 1
|
|
8
|
+
* @param vector2 Vector 2
|
|
9
|
+
* @returns Similarity between the two vectors
|
|
10
|
+
*/
|
|
11
|
+
static cosineSimilarity(vector1, vector2) {
|
|
12
|
+
// Return the quotient of the dot product and the product of the norms
|
|
13
|
+
return this.dotProduct(vector1, vector2) / (this.normalize(vector1) * this.normalize(vector2));
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Normalizes a vector.
|
|
17
|
+
* @remarks
|
|
18
|
+
* The norm of a vector is the square root of the sum of the squares of the elements.
|
|
19
|
+
* The LocalIndex pre-normalizes all vectors to improve performance.
|
|
20
|
+
* @param vector Vector to normalize
|
|
21
|
+
* @returns Normalized vector
|
|
22
|
+
*/
|
|
23
|
+
static normalize(vector) {
|
|
24
|
+
// Initialize a variable to store the sum of the squares
|
|
25
|
+
let sum = 0;
|
|
26
|
+
// Loop through the elements of the array
|
|
27
|
+
for (let i = 0; i < vector.length; i++) {
|
|
28
|
+
// Square the element and add it to the sum
|
|
29
|
+
sum += vector[i] * vector[i];
|
|
30
|
+
}
|
|
31
|
+
// Return the square root of the sum
|
|
32
|
+
return Math.sqrt(sum);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Returns the similarity between two vectors using cosine similarity.
|
|
36
|
+
* @remarks
|
|
37
|
+
* The LocalIndex pre-normalizes all vectors to improve performance.
|
|
38
|
+
* This method uses the pre-calculated norms to improve performance.
|
|
39
|
+
* @param vector1 Vector 1
|
|
40
|
+
* @param norm1 Norm of vector 1
|
|
41
|
+
* @param vector2 Vector 2
|
|
42
|
+
* @param norm2 Norm of vector 2
|
|
43
|
+
* @returns Similarity between the two vectors
|
|
44
|
+
*/
|
|
45
|
+
static normalizedCosineSimilarity(vector1, norm1, vector2, norm2) {
|
|
46
|
+
// Return the quotient of the dot product and the product of the norms
|
|
47
|
+
return this.dotProduct(vector1, vector2) / (norm1 * norm2);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Applies a filter to the metadata of an item.
|
|
51
|
+
* @param metadata Metadata of the item
|
|
52
|
+
* @param filter Filter to apply
|
|
53
|
+
* @returns True if the item matches the filter, false otherwise
|
|
54
|
+
*/
|
|
55
|
+
static select(metadata, filter) {
|
|
56
|
+
if (filter === undefined || filter === null) {
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
for (const key in filter) {
|
|
60
|
+
switch (key) {
|
|
61
|
+
case '$and':
|
|
62
|
+
if (!filter[key].every((f) => this.select(metadata, f))) {
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
break;
|
|
66
|
+
case '$or':
|
|
67
|
+
if (!filter[key].some((f) => this.select(metadata, f))) {
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
break;
|
|
71
|
+
default:
|
|
72
|
+
const value = filter[key];
|
|
73
|
+
if (value === undefined || value === null) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
else if (typeof value == 'object') {
|
|
77
|
+
if (!this.metadataFilter(metadata[key], value)) {
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
if (metadata[key] !== value) {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
break;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
static dotProduct(arr1, arr2) {
|
|
92
|
+
// Initialize a variable to store the sum of the products
|
|
93
|
+
let sum = 0;
|
|
94
|
+
// Loop through the elements of the arrays
|
|
95
|
+
for (let i = 0; i < arr1.length; i++) {
|
|
96
|
+
// Multiply the corresponding elements and add them to the sum
|
|
97
|
+
sum += arr1[i] * arr2[i];
|
|
98
|
+
}
|
|
99
|
+
// Return the sum
|
|
100
|
+
return sum;
|
|
101
|
+
}
|
|
102
|
+
static metadataFilter(value, filter) {
|
|
103
|
+
if (value === undefined || value === null) {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
for (const key in filter) {
|
|
107
|
+
switch (key) {
|
|
108
|
+
case '$eq':
|
|
109
|
+
if (value !== filter[key]) {
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
break;
|
|
113
|
+
case '$ne':
|
|
114
|
+
if (value === filter[key]) {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
break;
|
|
118
|
+
case '$gt':
|
|
119
|
+
if (typeof value != 'number' || value <= filter[key]) {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
break;
|
|
123
|
+
case '$gte':
|
|
124
|
+
if (typeof value != 'number' || value < filter[key]) {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
break;
|
|
128
|
+
case '$lt':
|
|
129
|
+
if (typeof value != 'number' || value >= filter[key]) {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
break;
|
|
133
|
+
case '$lte':
|
|
134
|
+
if (typeof value != 'number' || value > filter[key]) {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
break;
|
|
138
|
+
case '$in':
|
|
139
|
+
if (typeof value == 'boolean' || !filter[key].includes(value)) {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
break;
|
|
143
|
+
case '$nin':
|
|
144
|
+
if (typeof value == 'boolean' || filter[key].includes(value)) {
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
break;
|
|
148
|
+
default:
|
|
149
|
+
return value === filter[key];
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
exports.ItemSelector = ItemSelector;
|
|
156
|
+
//# sourceMappingURL=ItemSelector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ItemSelector.js","sourceRoot":"","sources":["../src/ItemSelector.ts"],"names":[],"mappings":";;;AAEA,MAAa,YAAY;IACrB;;;;;OAKG;IACI,MAAM,CAAC,gBAAgB,CAAC,OAAiB,EAAE,OAAiB;QAC/D,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;;;;OAOG;IACI,MAAM,CAAC,SAAS,CAAC,MAAgB;QACpC,wDAAwD;QACxD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,2CAA2C;YAC3C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SAChC;QACD,oCAAoC;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;OAUG;IACI,MAAM,CAAC,0BAA0B,CAAC,OAAiB,EAAE,KAAa,EAAE,OAAiB,EAAE,KAAa;QACvG,sEAAsE;QACtE,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;IAC/D,CAAC;IAED;;;;;OAKG;IACI,MAAM,CAAC,MAAM,CAAC,QAAuC,EAAE,MAAsB;QAChF,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAE;YACzC,OAAO,IAAI,CAAC;SACf;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,MAAM;oBACP,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACrD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;wBACpD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;wBACvC,OAAO,KAAK,CAAC;qBAChB;yBAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE;wBACjC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,KAAuB,CAAC,EAAE;4BAC9D,OAAO,KAAK,CAAC;yBAChB;qBACJ;yBAAM;wBACH,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE;4BACzB,OAAO,KAAK,CAAC;yBAChB;qBACJ;oBACD,MAAM;aACb;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,MAAM,CAAC,UAAU,CAAC,IAAc,EAAE,IAAc;QACpD,yDAAyD;QACzD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,0CAA0C;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAClC,8DAA8D;YAC9D,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;SAC5B;QACD,iBAAiB;QACjB,OAAO,GAAG,CAAC;IACf,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAoB,EAAE,MAAsB;QACtE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE;YACvC,OAAO,KAAK,CAAC;SAChB;QAED,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;YACtB,QAAQ,GAAG,EAAE;gBACT,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE;wBACvB,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE;wBACjD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;wBAClD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE;wBACjD,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,KAAK;oBACN,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC3D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV,KAAK,MAAM;oBACP,IAAI,OAAO,KAAK,IAAI,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;wBAC1D,OAAO,KAAK,CAAC;qBAChB;oBACD,MAAM;gBACV;oBACI,OAAO,KAAK,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;aACpC;SACJ;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ;AA3JD,oCA2JC"}
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
export interface CreateIndexConfig {
|
|
2
|
+
version: number;
|
|
3
|
+
deleteIfExists?: boolean;
|
|
4
|
+
metadata_config?: {
|
|
5
|
+
indexed?: string[];
|
|
6
|
+
};
|
|
7
|
+
}
|
|
8
|
+
export interface IndexStats {
|
|
9
|
+
version: number;
|
|
10
|
+
metadata_config: {
|
|
11
|
+
indexed?: string[];
|
|
12
|
+
};
|
|
13
|
+
items: number;
|
|
14
|
+
}
|
|
15
|
+
export interface IndexItem<TMetadata = Record<string, MetadataTypes>> {
|
|
16
|
+
id: string;
|
|
17
|
+
metadata: TMetadata;
|
|
18
|
+
vector: number[];
|
|
19
|
+
norm: number;
|
|
20
|
+
metadataFile?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface QueryResult<TMetadata = Record<string, MetadataTypes>> {
|
|
23
|
+
item: IndexItem<TMetadata>;
|
|
24
|
+
score: number;
|
|
25
|
+
}
|
|
26
|
+
export interface MetadataFilter {
|
|
27
|
+
[key: string]: MetadataTypes | MetadataFilter | (number | string)[] | MetadataFilter[];
|
|
28
|
+
/**
|
|
29
|
+
* Equal to (number, string, boolean)
|
|
30
|
+
*/
|
|
31
|
+
'$eq': number | string | boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Not equal to (number, string, boolean)
|
|
34
|
+
*/
|
|
35
|
+
'$ne': number | string | boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Greater than (number)
|
|
38
|
+
*/
|
|
39
|
+
'$gt': number;
|
|
40
|
+
/**
|
|
41
|
+
* Greater than or equal to (number)
|
|
42
|
+
*/
|
|
43
|
+
'$gte': number;
|
|
44
|
+
/**
|
|
45
|
+
* Less than (number)
|
|
46
|
+
*/
|
|
47
|
+
'$lt': number;
|
|
48
|
+
/**
|
|
49
|
+
* Less than or equal to (number)
|
|
50
|
+
*/
|
|
51
|
+
'$lte': number;
|
|
52
|
+
/**
|
|
53
|
+
* In array (string or number)
|
|
54
|
+
*/
|
|
55
|
+
'$in': (number | string)[];
|
|
56
|
+
/**
|
|
57
|
+
* Not in array (string or number)
|
|
58
|
+
*/
|
|
59
|
+
'$nin': (number | string)[];
|
|
60
|
+
/**
|
|
61
|
+
* AND (MetadataFilter[])
|
|
62
|
+
*/
|
|
63
|
+
'$and': MetadataFilter[];
|
|
64
|
+
/**
|
|
65
|
+
* OR (MetadataFilter[])
|
|
66
|
+
*/
|
|
67
|
+
'$or': MetadataFilter[];
|
|
68
|
+
}
|
|
69
|
+
export declare type MetadataTypes = number | string | boolean;
|
|
70
|
+
/**
|
|
71
|
+
* Local vector index instance.
|
|
72
|
+
* @remarks
|
|
73
|
+
* This class is used to create, update, and query a local vector index.
|
|
74
|
+
* Each index is a folder on disk containing an index.json file and an optional set of metadata files.
|
|
75
|
+
*/
|
|
76
|
+
export declare class LocalIndex {
|
|
77
|
+
private readonly _folderPath;
|
|
78
|
+
private _data?;
|
|
79
|
+
private _update?;
|
|
80
|
+
/**
|
|
81
|
+
* Creates a new instance of LocalIndex.
|
|
82
|
+
* @param folderPath - Path to the index folder
|
|
83
|
+
*/
|
|
84
|
+
constructor(folderPath: string);
|
|
85
|
+
/**
|
|
86
|
+
* Begins an update to the index.
|
|
87
|
+
* @remarks
|
|
88
|
+
* This method loads the index into memory and prepares it for updates.
|
|
89
|
+
*/
|
|
90
|
+
beginUpdate(): Promise<void>;
|
|
91
|
+
/**
|
|
92
|
+
* Cancels an update to the index.
|
|
93
|
+
* @remarks
|
|
94
|
+
* This method discards any changes made to the index since the update began.
|
|
95
|
+
*/
|
|
96
|
+
cancelUpdate(): void;
|
|
97
|
+
/**
|
|
98
|
+
* Creates a new index.
|
|
99
|
+
* @remarks
|
|
100
|
+
* This method creates a new folder on disk containing an index.json file.
|
|
101
|
+
* @param config - Index configuration
|
|
102
|
+
*/
|
|
103
|
+
createIndex(config?: CreateIndexConfig): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Deletes the index.
|
|
106
|
+
* @remarks
|
|
107
|
+
* This method deletes the index folder from disk.
|
|
108
|
+
*/
|
|
109
|
+
deleteIndex(): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Deletes an item from the index.
|
|
112
|
+
* @param id - Item id
|
|
113
|
+
*/
|
|
114
|
+
deleteItem(id: string): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Ends an update to the index.
|
|
117
|
+
* @remarks
|
|
118
|
+
* This method saves the index to disk.
|
|
119
|
+
*/
|
|
120
|
+
endUpdate(): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Loads an index from disk and returns its stats.
|
|
123
|
+
* @returns Index stats
|
|
124
|
+
*/
|
|
125
|
+
getIndexStats(): Promise<IndexStats>;
|
|
126
|
+
/**
|
|
127
|
+
* Returns an item from the index given its ID.
|
|
128
|
+
* @param id Item id
|
|
129
|
+
* @returns Item or undefined if not found
|
|
130
|
+
*/
|
|
131
|
+
getItem<TMetadata = Record<string, MetadataTypes>>(id: string): Promise<IndexItem<TMetadata> | undefined>;
|
|
132
|
+
/**
|
|
133
|
+
* Adds an item to the index.
|
|
134
|
+
* @remarks
|
|
135
|
+
* A new update is started if one is not already in progress. If an item with the same ID
|
|
136
|
+
* already exists, an error will be thrown.
|
|
137
|
+
* @param item Item to insert
|
|
138
|
+
* @returns Inserted item
|
|
139
|
+
*/
|
|
140
|
+
insertItem<TMetadata = Record<string, MetadataTypes>>(item: Partial<IndexItem<TMetadata>>): Promise<IndexItem<TMetadata>>;
|
|
141
|
+
/**
|
|
142
|
+
* Returns true if the index exists.
|
|
143
|
+
*/
|
|
144
|
+
isIndexCreated(): Promise<boolean>;
|
|
145
|
+
/**
|
|
146
|
+
* Returns all items in the index.
|
|
147
|
+
* @remarks
|
|
148
|
+
* This method loads the index into memory and returns all its items. A copy of the items
|
|
149
|
+
* array is returned so no modifications should be made to the array.
|
|
150
|
+
* @returns All items in the index
|
|
151
|
+
*/
|
|
152
|
+
listItems<TMetadata = Record<string, MetadataTypes>>(): Promise<IndexItem<TMetadata>[]>;
|
|
153
|
+
/**
|
|
154
|
+
* Returns all items in the index matching the filter.
|
|
155
|
+
* @remarks
|
|
156
|
+
* This method loads the index into memory and returns all its items matching the filter.
|
|
157
|
+
* @param filter Filter to apply
|
|
158
|
+
* @returns Items matching the filter
|
|
159
|
+
*/
|
|
160
|
+
listItemsByMetadata<TMetadata = Record<string, MetadataTypes>>(filter: MetadataFilter): Promise<IndexItem<TMetadata>[]>;
|
|
161
|
+
/**
|
|
162
|
+
* Finds the top k items in the index that are most similar to the vector.
|
|
163
|
+
* @remarks
|
|
164
|
+
* This method loads the index into memory and returns the top k items that are most similar.
|
|
165
|
+
* An optional filter can be applied to the metadata of the items.
|
|
166
|
+
* @param vector Vector to query against
|
|
167
|
+
* @param topK Number of items to return
|
|
168
|
+
* @param filter Optional filter to apply
|
|
169
|
+
* @returns Similar items to the vector that matches the filter
|
|
170
|
+
*/
|
|
171
|
+
queryItems<TMetadata = Record<string, MetadataTypes>>(vector: number[], topK: number, filter?: MetadataFilter): Promise<QueryResult<TMetadata>[]>;
|
|
172
|
+
/**
|
|
173
|
+
* Adds or replaces an item in the index.
|
|
174
|
+
* @remarks
|
|
175
|
+
* A new update is started if one is not already in progress. If an item with the same ID
|
|
176
|
+
* already exists, it will be replaced.
|
|
177
|
+
* @param item Item to insert or replace
|
|
178
|
+
* @returns Upserted item
|
|
179
|
+
*/
|
|
180
|
+
upsertItem<TMetadata = Record<string, MetadataTypes>>(item: Partial<IndexItem<TMetadata>>): Promise<IndexItem<TMetadata>>;
|
|
181
|
+
private loadIndexData;
|
|
182
|
+
private addItemToUpdate;
|
|
183
|
+
}
|
|
184
|
+
//# sourceMappingURL=LocalIndex.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"LocalIndex.d.ts","sourceRoot":"","sources":["../src/LocalIndex.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,eAAe,CAAC,EAAE;QACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACL;AAED,MAAM,WAAW,UAAU;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,EAAE;QACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,SAAS,CAAC;IACpB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC;IACjE,IAAI,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAC,cAAc,GAAC,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,GAAC,cAAc,EAAE,CAAC;IAE/E;;OAEG;IACH,KAAK,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE7B;;OAEG;IACH,KAAK,EAAE,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;IAE7B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,KAAK,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAEzB;;OAEG;IACH,MAAM,EAAE,CAAC,MAAM,GAAC,MAAM,CAAC,EAAE,CAAC;IAE1B;;OAEG;IACH,MAAM,EAAE,cAAc,EAAE,CAAC;IAEzB;;OAEG;IACH,KAAK,EAAE,cAAc,EAAE,CAAC;CAC3B;AAED,oBAAY,aAAa,GAAG,MAAM,GAAC,MAAM,GAAC,OAAO,CAAC;AAElD;;;;;GAKG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,KAAK,CAAC,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAY;IAE5B;;;OAGG;gBACgB,UAAU,EAAE,MAAM;IAIrC;;;;OAIG;IACU,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IASzC;;;;OAIG;IACI,YAAY,IAAI,IAAI;IAI3B;;;;;OAKG;IACU,WAAW,CAAC,MAAM,GAAE,iBAAgC,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BjF;;;;OAIG;IACI,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQnC;;;OAGG;IACU,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlD;;;;OAIG;IACU,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC;;;OAGG;IACU,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IASjD;;;;OAIG;IACU,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC;IAKrH;;;;;;;OAOG;IACU,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAWrI;;OAEG;IACU,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAS/C;;;;;;OAMG;IACU,SAAS,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,KAAK,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;IAKnG;;;;;;OAMG;IACU,mBAAmB,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;IAKnI;;;;;;;;;OASG;IACU,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;IAyC7J;;;;;;;OAOG;IACU,UAAU,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,EAAC,aAAa,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAWvH,aAAa;YAab,eAAe;CA8DhC"}
|