verso-db 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +13 -7
- package/dist/BinaryHeap.d.ts +11 -1
- package/dist/BinaryHeap.d.ts.map +1 -1
- package/dist/BinaryHeap.js +138 -0
- package/dist/BinaryHeap.js.map +1 -0
- package/dist/Collection.d.ts +30 -4
- package/dist/Collection.d.ts.map +1 -1
- package/dist/Collection.js +1186 -0
- package/dist/Collection.js.map +1 -0
- package/dist/HNSWIndex.d.ts +59 -0
- package/dist/HNSWIndex.d.ts.map +1 -1
- package/dist/HNSWIndex.js +2818 -0
- package/dist/HNSWIndex.js.map +1 -0
- package/dist/MaxBinaryHeap.d.ts +2 -64
- package/dist/MaxBinaryHeap.d.ts.map +1 -1
- package/dist/MaxBinaryHeap.js +5 -0
- package/dist/MaxBinaryHeap.js.map +1 -0
- package/dist/SearchWorker.d.ts +57 -4
- package/dist/SearchWorker.d.ts.map +1 -1
- package/dist/SearchWorker.js +573 -0
- package/dist/SearchWorker.js.map +1 -0
- package/dist/VectorDB.d.ts.map +1 -1
- package/dist/VectorDB.js +246 -0
- package/dist/VectorDB.js.map +1 -0
- package/dist/WorkerPool.d.ts +32 -2
- package/dist/WorkerPool.d.ts.map +1 -1
- package/dist/WorkerPool.js +266 -0
- package/dist/WorkerPool.js.map +1 -0
- package/dist/backends/JsDistanceBackend.d.ts.map +1 -1
- package/dist/backends/JsDistanceBackend.js +163 -0
- package/dist/backends/JsDistanceBackend.js.map +1 -0
- package/dist/encoding/DeltaEncoder.d.ts +2 -2
- package/dist/encoding/DeltaEncoder.d.ts.map +1 -1
- package/dist/encoding/DeltaEncoder.js +199 -0
- package/dist/encoding/DeltaEncoder.js.map +1 -0
- package/dist/errors.js +97 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +61 -42
- package/dist/index.js.map +1 -9
- package/dist/presets.js +205 -0
- package/dist/presets.js.map +1 -0
- package/dist/quantization/ScalarQuantizer.d.ts +0 -34
- package/dist/quantization/ScalarQuantizer.d.ts.map +1 -1
- package/dist/quantization/ScalarQuantizer.js +346 -0
- package/dist/quantization/ScalarQuantizer.js.map +1 -0
- package/dist/storage/BatchWriter.js +351 -0
- package/dist/storage/BatchWriter.js.map +1 -0
- package/dist/storage/BunStorageBackend.d.ts +7 -3
- package/dist/storage/BunStorageBackend.d.ts.map +1 -1
- package/dist/storage/BunStorageBackend.js +182 -0
- package/dist/storage/BunStorageBackend.js.map +1 -0
- package/dist/storage/MemoryBackend.js +109 -0
- package/dist/storage/MemoryBackend.js.map +1 -0
- package/dist/storage/OPFSBackend.d.ts.map +1 -1
- package/dist/storage/OPFSBackend.js +325 -0
- package/dist/storage/OPFSBackend.js.map +1 -0
- package/dist/storage/StorageBackend.js +12 -0
- package/dist/storage/StorageBackend.js.map +1 -0
- package/dist/storage/WriteAheadLog.js +321 -0
- package/dist/storage/WriteAheadLog.js.map +1 -0
- package/dist/storage/createStorageBackend.d.ts +4 -0
- package/dist/storage/createStorageBackend.d.ts.map +1 -1
- package/dist/storage/createStorageBackend.js +119 -0
- package/dist/storage/createStorageBackend.js.map +1 -0
- package/{src/storage/index.ts → dist/storage/index.js} +7 -27
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/nodeFsRuntime.d.ts +14 -0
- package/dist/storage/nodeFsRuntime.d.ts.map +1 -0
- package/dist/storage/nodeFsRuntime.js +105 -0
- package/dist/storage/nodeFsRuntime.js.map +1 -0
- package/package.json +9 -7
- package/src/BinaryHeap.ts +0 -136
- package/src/Collection.ts +0 -1262
- package/src/HNSWIndex.ts +0 -2894
- package/src/MaxBinaryHeap.ts +0 -181
- package/src/SearchWorker.ts +0 -264
- package/src/VectorDB.ts +0 -319
- package/src/WorkerPool.ts +0 -222
- package/src/backends/JsDistanceBackend.ts +0 -171
- package/src/encoding/DeltaEncoder.ts +0 -236
- package/src/errors.ts +0 -110
- package/src/index.ts +0 -106
- package/src/presets.ts +0 -229
- package/src/quantization/ScalarQuantizer.ts +0 -487
- package/src/storage/BatchWriter.ts +0 -420
- package/src/storage/BunStorageBackend.ts +0 -199
- package/src/storage/MemoryBackend.ts +0 -122
- package/src/storage/OPFSBackend.ts +0 -348
- package/src/storage/StorageBackend.ts +0 -74
- package/src/storage/WriteAheadLog.ts +0 -379
- package/src/storage/createStorageBackend.ts +0 -137
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optimized JavaScript distance functions with SIMD-style unrolling.
|
|
3
|
+
* These provide ~1.5-2x speedup over naive loops by reducing loop overhead
|
|
4
|
+
* and enabling better CPU pipelining.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Fast inline distance functions for single-vector comparisons.
|
|
8
|
+
* Uses 8-wide unrolling with separate accumulators for instruction-level parallelism.
|
|
9
|
+
*/
|
|
10
|
+
import { DimensionMismatchError } from '../errors.js';
|
|
11
|
+
function assertMatchingDimensions(a, b, context) {
|
|
12
|
+
if (a.length !== b.length) {
|
|
13
|
+
throw new DimensionMismatchError(a.length, b.length, context);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export function dotProductFast(a, b) {
|
|
17
|
+
assertMatchingDimensions(a, b, 'dotProductFast');
|
|
18
|
+
const len = a.length;
|
|
19
|
+
// Use 8 accumulators for better ILP (instruction-level parallelism)
|
|
20
|
+
let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
|
|
21
|
+
let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
|
|
22
|
+
let i = 0;
|
|
23
|
+
// 8-wide unrolling for high-dimensional vectors (768D, 1536D)
|
|
24
|
+
const limit8 = len - 7;
|
|
25
|
+
for (; i < limit8; i += 8) {
|
|
26
|
+
sum0 += a[i] * b[i];
|
|
27
|
+
sum1 += a[i + 1] * b[i + 1];
|
|
28
|
+
sum2 += a[i + 2] * b[i + 2];
|
|
29
|
+
sum3 += a[i + 3] * b[i + 3];
|
|
30
|
+
sum4 += a[i + 4] * b[i + 4];
|
|
31
|
+
sum5 += a[i + 5] * b[i + 5];
|
|
32
|
+
sum6 += a[i + 6] * b[i + 6];
|
|
33
|
+
sum7 += a[i + 7] * b[i + 7];
|
|
34
|
+
}
|
|
35
|
+
// Handle remaining elements
|
|
36
|
+
for (; i < len; i++) {
|
|
37
|
+
sum0 += a[i] * b[i];
|
|
38
|
+
}
|
|
39
|
+
return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
|
|
40
|
+
}
|
|
41
|
+
export function l2SquaredFast(a, b) {
|
|
42
|
+
assertMatchingDimensions(a, b, 'l2SquaredFast');
|
|
43
|
+
const len = a.length;
|
|
44
|
+
// Use 8 accumulators for better ILP
|
|
45
|
+
let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
|
|
46
|
+
let sum4 = 0, sum5 = 0, sum6 = 0, sum7 = 0;
|
|
47
|
+
let i = 0;
|
|
48
|
+
// 8-wide unrolling for high-dimensional vectors
|
|
49
|
+
const limit8 = len - 7;
|
|
50
|
+
for (; i < limit8; i += 8) {
|
|
51
|
+
const d0 = a[i] - b[i];
|
|
52
|
+
const d1 = a[i + 1] - b[i + 1];
|
|
53
|
+
const d2 = a[i + 2] - b[i + 2];
|
|
54
|
+
const d3 = a[i + 3] - b[i + 3];
|
|
55
|
+
const d4 = a[i + 4] - b[i + 4];
|
|
56
|
+
const d5 = a[i + 5] - b[i + 5];
|
|
57
|
+
const d6 = a[i + 6] - b[i + 6];
|
|
58
|
+
const d7 = a[i + 7] - b[i + 7];
|
|
59
|
+
sum0 += d0 * d0;
|
|
60
|
+
sum1 += d1 * d1;
|
|
61
|
+
sum2 += d2 * d2;
|
|
62
|
+
sum3 += d3 * d3;
|
|
63
|
+
sum4 += d4 * d4;
|
|
64
|
+
sum5 += d5 * d5;
|
|
65
|
+
sum6 += d6 * d6;
|
|
66
|
+
sum7 += d7 * d7;
|
|
67
|
+
}
|
|
68
|
+
// Handle remaining elements
|
|
69
|
+
for (; i < len; i++) {
|
|
70
|
+
const d = a[i] - b[i];
|
|
71
|
+
sum0 += d * d;
|
|
72
|
+
}
|
|
73
|
+
return sum0 + sum1 + sum2 + sum3 + sum4 + sum5 + sum6 + sum7;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Normalize a vector in place.
|
|
77
|
+
* Uses 8-wide unrolling for better ILP.
|
|
78
|
+
* Returns the original norm of the vector.
|
|
79
|
+
*/
|
|
80
|
+
export function normalizeInPlace(v) {
|
|
81
|
+
const len = v.length;
|
|
82
|
+
let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
|
|
83
|
+
let s4 = 0, s5 = 0, s6 = 0, s7 = 0;
|
|
84
|
+
let i = 0;
|
|
85
|
+
const limit8 = len - 7;
|
|
86
|
+
for (; i < limit8; i += 8) {
|
|
87
|
+
s0 += v[i] * v[i];
|
|
88
|
+
s1 += v[i + 1] * v[i + 1];
|
|
89
|
+
s2 += v[i + 2] * v[i + 2];
|
|
90
|
+
s3 += v[i + 3] * v[i + 3];
|
|
91
|
+
s4 += v[i + 4] * v[i + 4];
|
|
92
|
+
s5 += v[i + 5] * v[i + 5];
|
|
93
|
+
s6 += v[i + 6] * v[i + 6];
|
|
94
|
+
s7 += v[i + 7] * v[i + 7];
|
|
95
|
+
}
|
|
96
|
+
for (; i < len; i++) {
|
|
97
|
+
s0 += v[i] * v[i];
|
|
98
|
+
}
|
|
99
|
+
const norm = Math.sqrt(s0 + s1 + s2 + s3 + s4 + s5 + s6 + s7);
|
|
100
|
+
if (norm < 1e-10) {
|
|
101
|
+
// Zero or near-zero vector: leave as-is to avoid Infinity/NaN from division
|
|
102
|
+
return norm;
|
|
103
|
+
}
|
|
104
|
+
const invNorm = 1 / norm;
|
|
105
|
+
// 8-wide unrolling for the division loop matches the norm computation above
|
|
106
|
+
const divLimit8 = len - 7;
|
|
107
|
+
let j = 0;
|
|
108
|
+
for (; j < divLimit8; j += 8) {
|
|
109
|
+
v[j] *= invNorm;
|
|
110
|
+
v[j + 1] *= invNorm;
|
|
111
|
+
v[j + 2] *= invNorm;
|
|
112
|
+
v[j + 3] *= invNorm;
|
|
113
|
+
v[j + 4] *= invNorm;
|
|
114
|
+
v[j + 5] *= invNorm;
|
|
115
|
+
v[j + 6] *= invNorm;
|
|
116
|
+
v[j + 7] *= invNorm;
|
|
117
|
+
}
|
|
118
|
+
for (; j < len; j++)
|
|
119
|
+
v[j] *= invNorm;
|
|
120
|
+
return norm;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Compute cosine distance between two vectors.
|
|
124
|
+
* For pre-normalized vectors, this is simply 1 - dot(a, b).
|
|
125
|
+
* Uses 8-wide unrolling for norm computation.
|
|
126
|
+
*/
|
|
127
|
+
export function cosineDistanceFast(a, b, aIsNormalized = false, bIsNormalized = false) {
|
|
128
|
+
assertMatchingDimensions(a, b, 'cosineDistanceFast');
|
|
129
|
+
const dot = dotProductFast(a, b);
|
|
130
|
+
if (aIsNormalized && bIsNormalized) {
|
|
131
|
+
// Both vectors are normalized, cosine distance = 1 - dot
|
|
132
|
+
const dist = 1 - dot;
|
|
133
|
+
return dist < 1e-10 ? 0 : dist;
|
|
134
|
+
}
|
|
135
|
+
// Need to compute norms with 8-wide unrolling
|
|
136
|
+
const len = a.length;
|
|
137
|
+
let nA0 = 0, nA1 = 0, nA2 = 0, nA3 = 0;
|
|
138
|
+
let nB0 = 0, nB1 = 0, nB2 = 0, nB3 = 0;
|
|
139
|
+
let i = 0;
|
|
140
|
+
const limit8 = len - 7;
|
|
141
|
+
for (; i < limit8; i += 8) {
|
|
142
|
+
nA0 += a[i] * a[i] + a[i + 4] * a[i + 4];
|
|
143
|
+
nA1 += a[i + 1] * a[i + 1] + a[i + 5] * a[i + 5];
|
|
144
|
+
nA2 += a[i + 2] * a[i + 2] + a[i + 6] * a[i + 6];
|
|
145
|
+
nA3 += a[i + 3] * a[i + 3] + a[i + 7] * a[i + 7];
|
|
146
|
+
nB0 += b[i] * b[i] + b[i + 4] * b[i + 4];
|
|
147
|
+
nB1 += b[i + 1] * b[i + 1] + b[i + 5] * b[i + 5];
|
|
148
|
+
nB2 += b[i + 2] * b[i + 2] + b[i + 6] * b[i + 6];
|
|
149
|
+
nB3 += b[i + 3] * b[i + 3] + b[i + 7] * b[i + 7];
|
|
150
|
+
}
|
|
151
|
+
for (; i < len; i++) {
|
|
152
|
+
nA0 += a[i] * a[i];
|
|
153
|
+
nB0 += b[i] * b[i];
|
|
154
|
+
}
|
|
155
|
+
const normA = nA0 + nA1 + nA2 + nA3;
|
|
156
|
+
const normB = nB0 + nB1 + nB2 + nB3;
|
|
157
|
+
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
|
|
158
|
+
if (magnitude === 0)
|
|
159
|
+
return 1;
|
|
160
|
+
const dist = 1 - (dot / magnitude);
|
|
161
|
+
return dist < 1e-10 ? 0 : dist;
|
|
162
|
+
}
|
|
163
|
+
//# sourceMappingURL=JsDistanceBackend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"JsDistanceBackend.js","sourceRoot":"","sources":["../../src/backends/JsDistanceBackend.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;GAGG;AACH,OAAO,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAEnD,SAAS,wBAAwB,CAAC,CAAe,EAAE,CAAe,EAAE,OAAe;IACjF,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,sBAAsB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAe,EAAE,CAAe;IAC7D,wBAAwB,CAAC,CAAC,EAAE,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IACrB,oEAAoE;IACpE,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC;IAC3C,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,8DAA8D;IAC9D,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED,4BAA4B;IAC5B,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACpB,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,CAAe,EAAE,CAAe;IAC5D,wBAAwB,CAAC,CAAC,EAAE,CAAC,EAAE,eAAe,CAAC,CAAC;IAChD,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IACrB,oCAAoC;IACpC,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC;IAC3C,IAAI,CAAC,GAAG,CAAC,CAAC;IAEV,gDAAgD;IAChD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;QAChB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAC;IAClB,CAAC;IAED,4BAA4B;IAC5B,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IAED,OAAO,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAC/D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAe;IAC9C,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IACrB,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC;IAEvB,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1B,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACpB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9D,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;QACjB,4EAA4E;QAC5E,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC;IACzB,4EAA4E;IAC5E,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,SAAS,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC;QAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAC/E,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;QAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC;IACrF,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC;IACrC,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,CAAe,EAAE,CAAe,EAAE,aAAa,GAAG,KAAK,EAAE,aAAa,GAAG,KAAK;IAC/G,wBAAwB,CAAC,CAAC,EAAE,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACrD,MAAM,GAAG,GAAG,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEjC,IAAI,aAAa,IAAI,aAAa,EAAE,CAAC;QACnC,yDAAyD;QACzD,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC;QACrB,OAAO,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACjC,CAAC;IAED,8CAA8C;IAC9C,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IACrB,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;IACvC,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC;IAEvB,OAAO,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACpB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;IACpC,MAAM,KAAK,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;IACpC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,IAAI,SAAS,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAE9B,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IACnC,OAAO,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjC,CAAC"}
|
|
@@ -33,7 +33,7 @@ export declare function varintSize(value: number): number;
|
|
|
33
33
|
* Delta-encode a sorted array of neighbor IDs
|
|
34
34
|
* Returns the encoded buffer
|
|
35
35
|
*/
|
|
36
|
-
export declare function deltaEncodeNeighbors(neighbors: number[]): Uint8Array;
|
|
36
|
+
export declare function deltaEncodeNeighbors(neighbors: number[], preSorted?: boolean): Uint8Array;
|
|
37
37
|
/**
|
|
38
38
|
* Decode a delta-encoded neighbor list
|
|
39
39
|
* Returns the original neighbor IDs (sorted)
|
|
@@ -43,7 +43,7 @@ export declare function deltaDecodeNeighbors(buffer: Uint8Array, count: number):
|
|
|
43
43
|
* Calculate the encoded size for a neighbor list without actually encoding
|
|
44
44
|
* Useful for calculating total buffer size before serialization
|
|
45
45
|
*/
|
|
46
|
-
export declare function deltaEncodedSize(neighbors: number[]): number;
|
|
46
|
+
export declare function deltaEncodedSize(neighbors: number[], preSorted?: boolean): number;
|
|
47
47
|
/**
|
|
48
48
|
* Batch encode multiple neighbor lists efficiently
|
|
49
49
|
* Returns a single buffer with all encoded lists concatenated
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DeltaEncoder.d.ts","sourceRoot":"","sources":["../../src/encoding/DeltaEncoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"DeltaEncoder.d.ts","sourceRoot":"","sources":["../../src/encoding/DeltaEncoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH;;;GAGG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAWtF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CA0BjF;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAQhD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,SAAS,UAAQ,GAAG,UAAU,CAyCvF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAoBhF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,SAAS,UAAQ,GAAG,MAAM,CAuB/E;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,aAAa,EAAE,MAAM,EAAE,EAAE,GAAG;IAC3D,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CA0BA;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,MAAM,EAAE,EACjB,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,EAAE,MAAM,EAAE,GACf,MAAM,EAAE,EAAE,CAaZ"}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Delta Encoding with Varint for Neighbor Lists
|
|
3
|
+
*
|
|
4
|
+
* Implements delta-encoded neighbor lists as used by Qdrant for ~38% storage reduction.
|
|
5
|
+
* Neighbor IDs are sorted, then stored as deltas with variable-length encoding.
|
|
6
|
+
*
|
|
7
|
+
* Format:
|
|
8
|
+
* - First ID stored as full uint32
|
|
9
|
+
* - Subsequent IDs stored as varint deltas from previous ID
|
|
10
|
+
*
|
|
11
|
+
* Varint encoding (like Protocol Buffers):
|
|
12
|
+
* - Values 0-127: 1 byte
|
|
13
|
+
* - Values 128-16383: 2 bytes
|
|
14
|
+
* - Values 16384-2097151: 3 bytes
|
|
15
|
+
* - Values 2097152-268435455: 4 bytes
|
|
16
|
+
* - Larger: 5 bytes
|
|
17
|
+
*/
|
|
18
|
+
import { VectorDBError } from '../errors.js';
|
|
19
|
+
/**
|
|
20
|
+
* Encode an unsigned integer as a varint
|
|
21
|
+
* Returns the number of bytes written
|
|
22
|
+
*/
|
|
23
|
+
export function encodeVarint(value, buffer, offset) {
|
|
24
|
+
let v = value >>> 0; // Ensure unsigned
|
|
25
|
+
let bytesWritten = 0;
|
|
26
|
+
while (v >= 0x80) {
|
|
27
|
+
buffer[offset + bytesWritten] = (v & 0x7f) | 0x80;
|
|
28
|
+
v >>>= 7;
|
|
29
|
+
bytesWritten++;
|
|
30
|
+
}
|
|
31
|
+
buffer[offset + bytesWritten] = v;
|
|
32
|
+
return bytesWritten + 1;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Decode a varint from buffer
|
|
36
|
+
* Returns [value, bytesRead]
|
|
37
|
+
*/
|
|
38
|
+
export function decodeVarint(buffer, offset) {
|
|
39
|
+
let result = 0;
|
|
40
|
+
let shift = 0;
|
|
41
|
+
let bytesRead = 0;
|
|
42
|
+
while (offset + bytesRead < buffer.length) {
|
|
43
|
+
const byte = buffer[offset + bytesRead];
|
|
44
|
+
if (shift < 28) {
|
|
45
|
+
result |= (byte & 0x7f) << shift;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
// At shift >= 28, use multiplication to avoid sign-bit corruption from << on int32
|
|
49
|
+
result = (result + (byte & 0x7f) * (2 ** shift)) >>> 0;
|
|
50
|
+
}
|
|
51
|
+
bytesRead++;
|
|
52
|
+
if ((byte & 0x80) === 0) {
|
|
53
|
+
return [result >>> 0, bytesRead];
|
|
54
|
+
}
|
|
55
|
+
shift += 7;
|
|
56
|
+
if (shift > 35) {
|
|
57
|
+
throw new VectorDBError('Varint too long', 'CORRUPT_INDEX');
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
throw new VectorDBError('Unexpected end of buffer', 'CORRUPT_INDEX');
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Calculate the number of bytes needed to encode a varint
|
|
64
|
+
*/
|
|
65
|
+
export function varintSize(value) {
|
|
66
|
+
let v = value >>> 0;
|
|
67
|
+
let size = 1;
|
|
68
|
+
while (v >= 0x80) {
|
|
69
|
+
v >>>= 7;
|
|
70
|
+
size++;
|
|
71
|
+
}
|
|
72
|
+
return size;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Delta-encode a sorted array of neighbor IDs
|
|
76
|
+
* Returns the encoded buffer
|
|
77
|
+
*/
|
|
78
|
+
export function deltaEncodeNeighbors(neighbors, preSorted = false) {
|
|
79
|
+
if (neighbors.length === 0) {
|
|
80
|
+
return new Uint8Array(0);
|
|
81
|
+
}
|
|
82
|
+
for (let i = 0; i < neighbors.length; i++) {
|
|
83
|
+
const id = neighbors[i];
|
|
84
|
+
if (!Number.isInteger(id) || id < 0 || id > 0xFFFFFFFF) {
|
|
85
|
+
throw new VectorDBError(`Invalid neighbor ID at index ${i}: ${id}`, 'VALIDATION_ERROR');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Sort neighbors for optimal delta encoding
|
|
89
|
+
const sorted = preSorted ? neighbors : neighbors.slice().sort((a, b) => a - b);
|
|
90
|
+
// Calculate required buffer size
|
|
91
|
+
let size = 4; // First ID as uint32
|
|
92
|
+
let prev = sorted[0];
|
|
93
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
94
|
+
const delta = sorted[i] - prev;
|
|
95
|
+
size += varintSize(delta);
|
|
96
|
+
prev = sorted[i];
|
|
97
|
+
}
|
|
98
|
+
// Encode
|
|
99
|
+
const buffer = new Uint8Array(size);
|
|
100
|
+
const view = new DataView(buffer.buffer);
|
|
101
|
+
// First ID as full uint32 (little-endian)
|
|
102
|
+
view.setUint32(0, sorted[0], true);
|
|
103
|
+
let offset = 4;
|
|
104
|
+
// Remaining as deltas
|
|
105
|
+
prev = sorted[0];
|
|
106
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
107
|
+
const delta = sorted[i] - prev;
|
|
108
|
+
offset += encodeVarint(delta, buffer, offset);
|
|
109
|
+
prev = sorted[i];
|
|
110
|
+
}
|
|
111
|
+
return buffer;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Decode a delta-encoded neighbor list
|
|
115
|
+
* Returns the original neighbor IDs (sorted)
|
|
116
|
+
*/
|
|
117
|
+
export function deltaDecodeNeighbors(buffer, count) {
|
|
118
|
+
if (count === 0 || buffer.length === 0) {
|
|
119
|
+
return [];
|
|
120
|
+
}
|
|
121
|
+
const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
|
122
|
+
const neighbors = new Array(count);
|
|
123
|
+
// First ID as full uint32
|
|
124
|
+
neighbors[0] = view.getUint32(0, true);
|
|
125
|
+
let offset = 4;
|
|
126
|
+
// Remaining as deltas
|
|
127
|
+
for (let i = 1; i < count; i++) {
|
|
128
|
+
const [delta, bytesRead] = decodeVarint(buffer, offset);
|
|
129
|
+
neighbors[i] = neighbors[i - 1] + delta;
|
|
130
|
+
offset += bytesRead;
|
|
131
|
+
}
|
|
132
|
+
return neighbors;
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Calculate the encoded size for a neighbor list without actually encoding
|
|
136
|
+
* Useful for calculating total buffer size before serialization
|
|
137
|
+
*/
|
|
138
|
+
export function deltaEncodedSize(neighbors, preSorted = false) {
|
|
139
|
+
if (neighbors.length === 0) {
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
for (let i = 0; i < neighbors.length; i++) {
|
|
143
|
+
const id = neighbors[i];
|
|
144
|
+
if (!Number.isInteger(id) || id < 0 || id > 0xFFFFFFFF) {
|
|
145
|
+
throw new VectorDBError(`Invalid neighbor ID at index ${i}: ${id}`, 'VALIDATION_ERROR');
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const sorted = preSorted ? neighbors : neighbors.slice().sort((a, b) => a - b);
|
|
149
|
+
let size = 4; // First ID as uint32
|
|
150
|
+
let prev = sorted[0];
|
|
151
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
152
|
+
const delta = sorted[i] - prev;
|
|
153
|
+
size += varintSize(delta);
|
|
154
|
+
prev = sorted[i];
|
|
155
|
+
}
|
|
156
|
+
return size;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Batch encode multiple neighbor lists efficiently
|
|
160
|
+
* Returns a single buffer with all encoded lists concatenated
|
|
161
|
+
* Also returns offsets for each list
|
|
162
|
+
*/
|
|
163
|
+
export function deltaEncodeBatch(neighborLists) {
|
|
164
|
+
// Sort each list once, then pass pre-sorted to both size calculation and encoding
|
|
165
|
+
const sortedLists = neighborLists.map(list => list.length > 0 ? list.slice().sort((a, b) => a - b) : list);
|
|
166
|
+
// Calculate total size and individual sizes
|
|
167
|
+
const sizes = sortedLists.map(list => deltaEncodedSize(list, true));
|
|
168
|
+
const totalSize = sizes.reduce((a, b) => a + b, 0);
|
|
169
|
+
// Allocate single buffer
|
|
170
|
+
const buffer = new Uint8Array(totalSize);
|
|
171
|
+
const offsets = [];
|
|
172
|
+
let currentOffset = 0;
|
|
173
|
+
for (let i = 0; i < sortedLists.length; i++) {
|
|
174
|
+
offsets.push(currentOffset);
|
|
175
|
+
if (sortedLists[i].length > 0) {
|
|
176
|
+
const encoded = deltaEncodeNeighbors(sortedLists[i], true);
|
|
177
|
+
buffer.set(encoded, currentOffset);
|
|
178
|
+
currentOffset += encoded.length;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return { buffer, offsets, sizes };
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Decode a batch of neighbor lists from a single buffer
|
|
185
|
+
*/
|
|
186
|
+
export function deltaDecodeBatch(buffer, offsets, sizes, counts) {
|
|
187
|
+
const results = [];
|
|
188
|
+
for (let i = 0; i < offsets.length; i++) {
|
|
189
|
+
if (counts[i] === 0 || sizes[i] === 0) {
|
|
190
|
+
results.push([]);
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
const slice = buffer.subarray(offsets[i], offsets[i] + sizes[i]);
|
|
194
|
+
results.push(deltaDecodeNeighbors(slice, counts[i]));
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return results;
|
|
198
|
+
}
|
|
199
|
+
//# sourceMappingURL=DeltaEncoder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DeltaEncoder.js","sourceRoot":"","sources":["../../src/encoding/DeltaEncoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,MAAkB,EAAE,MAAc;IAC5E,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,kBAAkB;IACvC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QACjB,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAClD,CAAC,MAAM,CAAC,CAAC;QACT,YAAY,EAAE,CAAC;IACjB,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO,YAAY,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB,EAAE,MAAc;IAC7D,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,OAAO,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,KAAK,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,mFAAmF;YACnF,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC;QACD,SAAS,EAAE,CAAC;QAEZ,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;QACnC,CAAC;QAED,KAAK,IAAI,CAAC,CAAC;QACX,IAAI,KAAK,GAAG,EAAE,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,IAAI,aAAa,CAAC,0BAA0B,EAAE,eAAe,CAAC,CAAC;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC;IACpB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QACjB,CAAC,MAAM,CAAC,CAAC;QACT,IAAI,EAAE,CAAC;IACT,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAmB,EAAE,SAAS,GAAG,KAAK;IACzE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,UAAU,EAAE,CAAC;YACvD,MAAM,IAAI,aAAa,CAAC,gCAAgC,CAAC,KAAK,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE/E,iCAAiC;IACjC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,qBAAqB;IACnC,IAAI,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAC/B,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,SAAS;IACT,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAEzC,0CAA0C;IAC1C,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,sBAAsB;IACtB,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAC/B,MAAM,IAAI,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC9C,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAkB,EAAE,KAAa;IACpE,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,IAAI,KAAK,CAAS,KAAK,CAAC,CAAC;IAE3C,0BAA0B;IAC1B,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,sBAAsB;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACxD,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC;QACxC,MAAM,IAAI,SAAS,CAAC;IACtB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAmB,EAAE,SAAS,GAAG,KAAK;IACrE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,UAAU,EAAE,CAAC;YACvD,MAAM,IAAI,aAAa,CAAC,gCAAgC,CAAC,KAAK,EAAE,EAAE,EAAE,kBAAkB,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE/E,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,qBAAqB;IACnC,IAAI,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAC/B,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,aAAyB;IAKxD,kFAAkF;IAClF,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAC3C,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC5D,CAAC;IAEF,4CAA4C;IAC5C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACpE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAEnD,yBAAyB;IACzB,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE5B,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YACnC,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAkB,EAClB,OAAiB,EACjB,KAAe,EACf,MAAgB;IAEhB,MAAM,OAAO,GAAe,EAAE,CAAC;IAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Base error class for all VectorDB errors
|
|
3
|
+
*/
|
|
4
|
+
export class VectorDBError extends Error {
|
|
5
|
+
code;
|
|
6
|
+
constructor(message, code) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = 'VectorDBError';
|
|
9
|
+
this.code = code;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Thrown when vector dimensions don't match expected dimensions
|
|
14
|
+
*/
|
|
15
|
+
export class DimensionMismatchError extends VectorDBError {
|
|
16
|
+
expected;
|
|
17
|
+
actual;
|
|
18
|
+
constructor(expected, actual, context) {
|
|
19
|
+
const message = context
|
|
20
|
+
? `${context}: expected dimension ${expected}, got ${actual}`
|
|
21
|
+
: `Dimension mismatch: expected ${expected}, got ${actual}`;
|
|
22
|
+
super(message, 'DIMENSION_MISMATCH');
|
|
23
|
+
this.name = 'DimensionMismatchError';
|
|
24
|
+
this.expected = expected;
|
|
25
|
+
this.actual = actual;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Thrown when attempting to add a vector with an ID that already exists
|
|
30
|
+
*/
|
|
31
|
+
export class DuplicateVectorError extends VectorDBError {
|
|
32
|
+
ids;
|
|
33
|
+
constructor(ids) {
|
|
34
|
+
const message = ids.length === 1
|
|
35
|
+
? `Vector with ID '${ids[0]}' already exists`
|
|
36
|
+
: `Vectors with IDs already exist: ${ids.join(', ')}`;
|
|
37
|
+
super(message, 'DUPLICATE_VECTOR');
|
|
38
|
+
this.name = 'DuplicateVectorError';
|
|
39
|
+
this.ids = ids;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Thrown when a requested collection does not exist
|
|
44
|
+
*/
|
|
45
|
+
export class CollectionNotFoundError extends VectorDBError {
|
|
46
|
+
collectionName;
|
|
47
|
+
constructor(collectionName) {
|
|
48
|
+
super(`Collection '${collectionName}' does not exist`, 'COLLECTION_NOT_FOUND');
|
|
49
|
+
this.name = 'CollectionNotFoundError';
|
|
50
|
+
this.collectionName = collectionName;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Thrown when attempting to create a collection that already exists
|
|
55
|
+
*/
|
|
56
|
+
export class CollectionExistsError extends VectorDBError {
|
|
57
|
+
collectionName;
|
|
58
|
+
constructor(collectionName) {
|
|
59
|
+
super(`Collection '${collectionName}' already exists`, 'COLLECTION_EXISTS');
|
|
60
|
+
this.name = 'CollectionExistsError';
|
|
61
|
+
this.collectionName = collectionName;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Thrown when a storage operation fails
|
|
66
|
+
*/
|
|
67
|
+
export class StorageError extends VectorDBError {
|
|
68
|
+
operation;
|
|
69
|
+
path;
|
|
70
|
+
constructor(operation, message, path) {
|
|
71
|
+
super(`Storage ${operation} failed: ${message}`, 'STORAGE_ERROR');
|
|
72
|
+
this.name = 'StorageError';
|
|
73
|
+
this.operation = operation;
|
|
74
|
+
this.path = path;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Thrown when quantization operations fail
|
|
79
|
+
*/
|
|
80
|
+
export class QuantizationError extends VectorDBError {
|
|
81
|
+
constructor(message) {
|
|
82
|
+
super(message, 'QUANTIZATION_ERROR');
|
|
83
|
+
this.name = 'QuantizationError';
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Thrown when a vector is not found in the index
|
|
88
|
+
*/
|
|
89
|
+
export class VectorNotFoundError extends VectorDBError {
|
|
90
|
+
vectorId;
|
|
91
|
+
constructor(vectorId) {
|
|
92
|
+
super(`Vector '${vectorId}' not found`, 'VECTOR_NOT_FOUND');
|
|
93
|
+
this.name = 'VectorNotFoundError';
|
|
94
|
+
this.vectorId = vectorId;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC7B,IAAI,CAAS;IAEtB,YAAY,OAAe,EAAE,IAAY;QACvC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;QAC5B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,aAAa;IAC9C,QAAQ,CAAS;IACjB,MAAM,CAAS;IAExB,YAAY,QAAgB,EAAE,MAAc,EAAE,OAAgB;QAC5D,MAAM,OAAO,GAAG,OAAO;YACrB,CAAC,CAAC,GAAG,OAAO,wBAAwB,QAAQ,SAAS,MAAM,EAAE;YAC7D,CAAC,CAAC,gCAAgC,QAAQ,SAAS,MAAM,EAAE,CAAC;QAC9D,KAAK,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;QACrC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,aAAa;IAC5C,GAAG,CAAW;IAEvB,YAAY,GAAa;QACvB,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC;YAC9B,CAAC,CAAC,mBAAmB,GAAG,CAAC,CAAC,CAAC,kBAAkB;YAC7C,CAAC,CAAC,mCAAmC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxD,KAAK,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,uBAAwB,SAAQ,aAAa;IAC/C,cAAc,CAAS;IAEhC,YAAY,cAAsB;QAChC,KAAK,CAAC,eAAe,cAAc,kBAAkB,EAAE,sBAAsB,CAAC,CAAC;QAC/E,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,qBAAsB,SAAQ,aAAa;IAC7C,cAAc,CAAS;IAEhC,YAAY,cAAsB;QAChC,KAAK,CAAC,eAAe,cAAc,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;QAC5E,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;QACpC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAa,SAAQ,aAAa;IACpC,SAAS,CAAS;IAClB,IAAI,CAAU;IAEvB,YAAY,SAAiB,EAAE,OAAe,EAAE,IAAa;QAC3D,KAAK,CAAC,WAAW,SAAS,YAAY,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,GAAG,cAAc,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,aAAa;IAClD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,mBAAoB,SAAQ,aAAa;IAC3C,QAAQ,CAAkB;IAEnC,YAAY,QAAyB;QACnC,KAAK,CAAC,WAAW,QAAQ,aAAa,EAAE,kBAAkB,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* - Int8 scalar quantization for 4x memory reduction
|
|
8
8
|
* - Batch query support for improved throughput
|
|
9
9
|
* - Parameter presets for different use cases
|
|
10
|
-
* - Multi-platform: Bun (file system) and Browser (OPFS)
|
|
10
|
+
* - Multi-platform: Bun/Node.js (file system) and Browser (OPFS)
|
|
11
11
|
*
|
|
12
12
|
* @example
|
|
13
13
|
* ```typescript
|
|
@@ -41,12 +41,12 @@
|
|
|
41
41
|
export { VectorDB } from './VectorDB';
|
|
42
42
|
export type { VectorDBConfig, CollectionConfig } from './VectorDB';
|
|
43
43
|
export { Collection } from './Collection';
|
|
44
|
-
export type { AddConfig, QueryConfig, QueryResult } from './Collection';
|
|
44
|
+
export type { AddConfig, QueryConfig, QueryResult, CollectionOptions } from './Collection';
|
|
45
45
|
export { HNSWIndex } from './HNSWIndex';
|
|
46
46
|
export type { DistanceMetric } from './HNSWIndex';
|
|
47
47
|
export type { HNSWPreset } from './presets';
|
|
48
48
|
export { PRESET_LOW_DIM, PRESET_MEDIUM_DIM, PRESET_HIGH_DIM, PRESET_VERY_HIGH_DIM, PRESET_SMALL_DATASET, PRESET_LARGE_DATASET, PRESET_MAX_RECALL, PRESET_LOW_LATENCY, PRESETS, getRecommendedPreset, getPreset, getRAGPreset } from './presets';
|
|
49
|
-
export { ScalarQuantizer
|
|
49
|
+
export { ScalarQuantizer } from './quantization/ScalarQuantizer';
|
|
50
50
|
export type { QuantizationParams } from './quantization/ScalarQuantizer';
|
|
51
51
|
export type { StorageBackend, StorageOptions } from './storage/StorageBackend';
|
|
52
52
|
export { BunStorageBackend } from './storage/BunStorageBackend';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAKH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,YAAY,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAKH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,YAAY,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAC3F,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,YAAY,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAGlD,YAAY,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,OAAO,EACP,oBAAoB,EACpB,SAAS,EACT,YAAY,EACb,MAAM,WAAW,CAAC;AAGnB,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAGzE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,sBAAsB,EACtB,KAAK,WAAW,EAChB,KAAK,oBAAoB,GAC1B,MAAM,gCAAgC,CAAC;AAGxC,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC1E,YAAY,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACvE,YAAY,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAGhE,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGnD,OAAO,EACL,aAAa,EACb,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,EACrB,YAAY,EACZ,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,UAAU,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,42 +1,61 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
1
|
+
/**
|
|
2
|
+
* verso-db - High-performance vector search with HNSW indexing
|
|
3
|
+
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - HNSW algorithm for approximate nearest neighbor search
|
|
6
|
+
* - Multiple distance metrics: cosine, euclidean, dot product
|
|
7
|
+
* - Int8 scalar quantization for 4x memory reduction
|
|
8
|
+
* - Batch query support for improved throughput
|
|
9
|
+
* - Parameter presets for different use cases
|
|
10
|
+
* - Multi-platform: Bun/Node.js (file system) and Browser (OPFS)
|
|
11
|
+
*
|
|
12
|
+
* @example
|
|
13
|
+
* ```typescript
|
|
14
|
+
* import { VectorDB, getRecommendedPreset } from 'verso-db';
|
|
15
|
+
*
|
|
16
|
+
* const db = new VectorDB({ storagePath: './vectors' });
|
|
17
|
+
* const preset = getRecommendedPreset(768);
|
|
18
|
+
*
|
|
19
|
+
* const collection = await db.createCollection('my-vectors', {
|
|
20
|
+
* dimension: 768,
|
|
21
|
+
* metric: 'cosine',
|
|
22
|
+
* M: preset.M,
|
|
23
|
+
* efConstruction: preset.efConstruction
|
|
24
|
+
* });
|
|
25
|
+
*
|
|
26
|
+
* await collection.add({
|
|
27
|
+
* ids: ['doc1', 'doc2'],
|
|
28
|
+
* vectors: [new Float32Array(768), new Float32Array(768)]
|
|
29
|
+
* });
|
|
30
|
+
*
|
|
31
|
+
* const results = await collection.query({
|
|
32
|
+
* queryVector: new Float32Array(768),
|
|
33
|
+
* k: 10,
|
|
34
|
+
* efSearch: preset.efSearch
|
|
35
|
+
* });
|
|
36
|
+
* ```
|
|
37
|
+
*
|
|
38
|
+
* @packageDocumentation
|
|
39
|
+
* @module verso-db
|
|
40
|
+
*/
|
|
41
|
+
// ─── Public API ──────────────────────────────────────────────────────────────
|
|
42
|
+
// Core
|
|
43
|
+
export { VectorDB } from './VectorDB.js';
|
|
44
|
+
export { Collection } from './Collection.js';
|
|
45
|
+
export { HNSWIndex } from './HNSWIndex.js';
|
|
46
|
+
export { PRESET_LOW_DIM, PRESET_MEDIUM_DIM, PRESET_HIGH_DIM, PRESET_VERY_HIGH_DIM, PRESET_SMALL_DATASET, PRESET_LARGE_DATASET, PRESET_MAX_RECALL, PRESET_LOW_LATENCY, PRESETS, getRecommendedPreset, getPreset, getRAGPreset } from './presets.js';
|
|
47
|
+
// Quantization
|
|
48
|
+
export { ScalarQuantizer } from './quantization/ScalarQuantizer.js';
|
|
49
|
+
export { BunStorageBackend } from './storage/BunStorageBackend.js';
|
|
50
|
+
export { MemoryBackend } from './storage/MemoryBackend.js';
|
|
51
|
+
export { OPFSBackend } from './storage/OPFSBackend.js';
|
|
52
|
+
export { createStorageBackend, getRecommendedStorageType, isStorageTypeAvailable, } from './storage/createStorageBackend.js';
|
|
53
|
+
// Write-ahead log and batch writer
|
|
54
|
+
export { WriteAheadLog, WALOperationType } from './storage/WriteAheadLog.js';
|
|
55
|
+
export { BatchWriter, createBatchWriter } from './storage/BatchWriter.js';
|
|
56
|
+
// Parallel query processing
|
|
57
|
+
export { WorkerPool } from './WorkerPool.js';
|
|
58
|
+
export { WorkerSearchState } from './SearchWorker.js';
|
|
59
|
+
// Error classes
|
|
60
|
+
export { VectorDBError, DimensionMismatchError, DuplicateVectorError, CollectionNotFoundError, CollectionExistsError, StorageError, QuantizationError, VectorNotFoundError, } from './errors.js';
|
|
61
|
+
//# sourceMappingURL=index.js.map
|