usearch 2.25.0 → 2.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -46
- package/c/README.md +1 -1
- package/c/test.c +41 -0
- package/include/usearch/index.hpp +1 -1
- package/javascript/README.md +1 -1
- package/javascript/usearch.test.js +117 -94
- package/package.json +1 -1
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
package/README.md
CHANGED
|
@@ -51,7 +51,7 @@ Linux • macOS • Windows • iOS • Android • WebAssembly •
|
|
|
51
51
|
- ✅ Simple and extensible [single C++11 header][usearch-header] __library__.
|
|
52
52
|
- ✅ [Trusted](#integrations) by giants like Google and DBs like [ClickHouse][clickhouse-docs] & [DuckDB][duckdb-docs].
|
|
53
53
|
- ✅ [SIMD][simd]-optimized and [user-defined metrics](#user-defined-functions) with JIT compilation.
|
|
54
|
-
- ✅ Hardware-agnostic `
|
|
54
|
+
- ✅ Hardware-agnostic `bf16`, `e5m2`, & `i8` - [half-precision & quarter-precision support](#memory-efficiency-downcasting-and-quantization).
|
|
55
55
|
- ✅ [View large indexes from disk](#serialization--serving-index-from-disk) without loading into RAM.
|
|
56
56
|
- ✅ Heterogeneous lookups, renaming/relabeling, and on-the-fly deletions.
|
|
57
57
|
- ✅ Binary Tanimoto and Sorensen coefficients for [Genomics and Chemistry applications](#usearch--rdkit--molecular-search).
|
|
@@ -138,7 +138,7 @@ The default storage/quantization level is hardware-dependant for efficiency, but
|
|
|
138
138
|
index = Index(
|
|
139
139
|
ndim=3, # Define the number of dimensions in input vectors
|
|
140
140
|
metric='cos', # Choose 'l2sq', 'ip', 'haversine' or other metric, default = 'cos'
|
|
141
|
-
dtype='bf16', # Store as 'f64', 'f32', 'f16', 'i8', 'b1'..., default = None
|
|
141
|
+
dtype='bf16', # Store as 'f64', 'f32', 'bf16', 'f16', 'e5m2', 'e4m3', 'e3m2', 'e2m3', 'u8', 'i8', 'b1'..., default = None
|
|
142
142
|
connectivity=16, # Optional: Limit number of neighbors per graph node
|
|
143
143
|
expansion_add=128, # Optional: Control the recall of indexing
|
|
144
144
|
expansion_search=64, # Optional: Control the quality of the search
|
|
@@ -251,7 +251,7 @@ assert!(
|
|
|
251
251
|
Training a quantization model and dimension-reduction is a common approach to accelerate vector search.
|
|
252
252
|
Those, however, are only sometimes reliable, can significantly affect the statistical properties of your data, and require regular adjustments if your distribution shifts.
|
|
253
253
|
Instead, we have focused on high-precision arithmetic over low-precision downcasted vectors.
|
|
254
|
-
The same index, and `add` and `search` operations will automatically down-cast or up-cast between `f64_t`, `f32_t`, `f16_t`, `i8_t`, and single-bit `b1x8_t` representations.
|
|
254
|
+
The same index, and `add` and `search` operations will automatically down-cast or up-cast between `f64_t`, `f32_t`, `bf16_t`, `f16_t`, `e5m2_t`, `e4m3_t`, `e3m2_t`, `e2m3_t`, `u8_t`, `i8_t`, and single-bit `b1x8_t` representations.
|
|
255
255
|
You can use the following command to check, if hardware acceleration is enabled:
|
|
256
256
|
|
|
257
257
|
```sh
|
|
@@ -261,7 +261,9 @@ $ python -c 'from usearch.index import Index; print(Index(ndim=166, metric="tani
|
|
|
261
261
|
> ice
|
|
262
262
|
```
|
|
263
263
|
|
|
264
|
-
In most cases,
|
|
264
|
+
In most cases, `bf16` is recommended for modern CPUs.
|
|
265
|
+
For even smaller footprints, USearch supports IEEE & MX-compatible Float8 (`e5m2` and `e4m3`) and Float6 (`e3m2` and `e2m3`) formats.
|
|
266
|
+
You can pass pre-quantized buffers from [NumKong](https://github.com/ashvardanian/numkong) with the explicit `dtype=` parameter on `add` and `search`, or let USearch handle the quantization internally from higher-precision inputs.
|
|
265
267
|
When quantization is enabled, the "get"-like functions won't be able to recover the original data, so you may want to replicate the original vectors elsewhere.
|
|
266
268
|
When quantizing to `i8_t` integers, note that it's only valid for cosine-like metrics.
|
|
267
269
|
As part of the quantization process, the vectors are normalized to unit length and later scaled to [-127, 127] range to occupy the full 8-bit range.
|
|
@@ -479,46 +481,41 @@ The Haversine distance is available out of the box, but you can also define more
|
|
|
479
481
|
from numba import cfunc, types, carray
|
|
480
482
|
import math
|
|
481
483
|
|
|
482
|
-
# Define the dimension as 2 for latitude and longitude
|
|
483
484
|
ndim = 2
|
|
485
|
+
semi_major, flattening = 6378137.0, 1 / 298.257223563
|
|
486
|
+
semi_minor = (1 - flattening) * semi_major
|
|
487
|
+
|
|
488
|
+
def vincenty_distance(first_ptr, second_ptr):
|
|
489
|
+
first, second = carray(first_ptr, ndim), carray(second_ptr, ndim)
|
|
490
|
+
lat1, lon1, lat2, lon2 = first[0], first[1], second[0], second[1]
|
|
491
|
+
diff_lon = lon2 - lon1
|
|
492
|
+
rlat1, rlat2 = math.atan((1 - flattening) * math.tan(lat1)), math.atan((1 - flattening) * math.tan(lat2))
|
|
493
|
+
sin_rlat1, cos_rlat1 = math.sin(rlat1), math.cos(rlat1)
|
|
494
|
+
sin_rlat2, cos_rlat2 = math.sin(rlat2), math.cos(rlat2)
|
|
495
|
+
lon_on_sphere = diff_lon
|
|
496
|
+
for _ in range(100):
|
|
497
|
+
sin_lon, cos_lon = math.sin(lon_on_sphere), math.cos(lon_on_sphere)
|
|
498
|
+
sin_ang = math.sqrt((cos_rlat2 * sin_lon) ** 2 + (cos_rlat1 * sin_rlat2 - sin_rlat1 * cos_rlat2 * cos_lon) ** 2)
|
|
499
|
+
if sin_ang == 0: return 0.0
|
|
500
|
+
cos_ang = sin_rlat1 * sin_rlat2 + cos_rlat1 * cos_rlat2 * cos_lon
|
|
501
|
+
ang = math.atan2(sin_ang, cos_ang)
|
|
502
|
+
sin_az = cos_rlat1 * cos_rlat2 * sin_lon / sin_ang
|
|
503
|
+
cos2_az = 1 - sin_az ** 2
|
|
504
|
+
cos2_mid = cos_ang - 2 * sin_rlat1 * sin_rlat2 / cos2_az if cos2_az != 0 else 0.0
|
|
505
|
+
corr = flattening / 16 * cos2_az * (4 + flattening * (4 - 3 * cos2_az))
|
|
506
|
+
prev = lon_on_sphere
|
|
507
|
+
lon_on_sphere = diff_lon + (1 - corr) * flattening * (
|
|
508
|
+
sin_az * (ang + corr * sin_ang * (cos2_mid + corr * cos_ang * (-1 + 2 * cos2_mid ** 2))))
|
|
509
|
+
if abs(lon_on_sphere - prev) <= 1e-12: break
|
|
510
|
+
else:
|
|
511
|
+
return float('nan')
|
|
512
|
+
u_sq = cos2_az * (semi_major ** 2 - semi_minor ** 2) / (semi_minor ** 2)
|
|
513
|
+
ca = 1 + u_sq / 16384 * (4096 + u_sq * (-768 + u_sq * (320 - 175 * u_sq)))
|
|
514
|
+
cb = u_sq / 1024 * (256 + u_sq * (-128 + u_sq * (74 - 47 * u_sq)))
|
|
515
|
+
delta = cb * sin_ang * (cos2_mid + cb / 4 * (cos_ang * (-1 + 2 * cos2_mid ** 2)
|
|
516
|
+
- cb / 6 * cos2_mid * (-3 + 4 * sin_ang ** 2) * (-3 + 4 * cos2_mid ** 2)))
|
|
517
|
+
return semi_minor * ca * (ang - delta) / 1000.0
|
|
484
518
|
|
|
485
|
-
# Signature for the custom metric
|
|
486
|
-
signature = types.float32(
|
|
487
|
-
types.CPointer(types.float32),
|
|
488
|
-
types.CPointer(types.float32))
|
|
489
|
-
|
|
490
|
-
# WGS-84 ellipsoid parameters
|
|
491
|
-
a = 6378137.0 # major axis in meters
|
|
492
|
-
f = 1 / 298.257223563 # flattening
|
|
493
|
-
b = (1 - f) * a # minor axis
|
|
494
|
-
|
|
495
|
-
def vincenty_distance(a_ptr, b_ptr):
|
|
496
|
-
a_array = carray(a_ptr, ndim)
|
|
497
|
-
b_array = carray(b_ptr, ndim)
|
|
498
|
-
lat1, lon1, lat2, lon2 = a_array[0], a_array[1], b_array[0], b_array[1]
|
|
499
|
-
L, U1, U2 = lon2 - lon1, math.atan((1 - f) * math.tan(lat1)), math.atan((1 - f) * math.tan(lat2))
|
|
500
|
-
sinU1, cosU1, sinU2, cosU2 = math.sin(U1), math.cos(U1), math.sin(U2), math.cos(U2)
|
|
501
|
-
lambda_, iterLimit = L, 100
|
|
502
|
-
while iterLimit > 0:
|
|
503
|
-
iterLimit -= 1
|
|
504
|
-
sinLambda, cosLambda = math.sin(lambda_), math.cos(lambda_)
|
|
505
|
-
sinSigma = math.sqrt((cosU2 * sinLambda) ** 2 + (cosU1 * sinU2 - sinU1 * cosU2 * cosLambda) ** 2)
|
|
506
|
-
if sinSigma == 0: return 0.0 # Co-incident points
|
|
507
|
-
cosSigma, sigma = sinU1 * sinU2 + cosU1 * cosU2 * cosLambda, math.atan2(sinSigma, cosSigma)
|
|
508
|
-
sinAlpha, cos2Alpha = cosU1 * cosU2 * sinLambda / sinSigma, 1 - (cosU1 * cosU2 * sinLambda / sinSigma) ** 2
|
|
509
|
-
cos2SigmaM = cosSigma - 2 * sinU1 * sinU2 / cos2Alpha if not math.isnan(cosSigma - 2 * sinU1 * sinU2 / cos2Alpha) else 0 # Equatorial line
|
|
510
|
-
C = f / 16 * cos2Alpha * (4 + f * (4 - 3 * cos2Alpha))
|
|
511
|
-
lambda_, lambdaP = L + (1 - C) * f * (sinAlpha * (sigma + C * sinSigma * (cos2SigmaM + C * cosSigma * (-1 + 2 * cos2SigmaM ** 2)))), lambda_
|
|
512
|
-
if abs(lambda_ - lambdaP) <= 1e-12: break
|
|
513
|
-
if iterLimit == 0: return float('nan') # formula failed to converge
|
|
514
|
-
u2 = cos2Alpha * (a ** 2 - b ** 2) / (b ** 2)
|
|
515
|
-
A = 1 + u2 / 16384 * (4096 + u2 * (-768 + u2 * (320 - 175 * u2)))
|
|
516
|
-
B = u2 / 1024 * (256 + u2 * (-128 + u2 * (74 - 47 * u2)))
|
|
517
|
-
deltaSigma = B * sinSigma * (cos2SigmaM + B / 4 * (cosSigma * (-1 + 2 * cos2SigmaM ** 2) - B / 6 * cos2SigmaM * (-3 + 4 * sinSigma ** 2) * (-3 + 4 * cos2SigmaM ** 2)))
|
|
518
|
-
s = b * A * (sigma - deltaSigma)
|
|
519
|
-
return s / 1000.0 # Distance in kilometers
|
|
520
|
-
|
|
521
|
-
# Example usage:
|
|
522
519
|
index = Index(ndim=ndim, metric=CompiledMetric(
|
|
523
520
|
pointer=vincenty_distance.address,
|
|
524
521
|
kind=MetricKind.Haversine,
|
|
@@ -547,13 +544,12 @@ index = Index(ndim=ndim, metric=CompiledMetric(
|
|
|
547
544
|
## Citations
|
|
548
545
|
|
|
549
546
|
```bibtex
|
|
550
|
-
@software{
|
|
547
|
+
@software{Vardanian_USearch,
|
|
551
548
|
doi = {10.5281/zenodo.7949416},
|
|
552
549
|
author = {Vardanian, Ash},
|
|
553
550
|
title = {{USearch by Unum Cloud}},
|
|
554
551
|
url = {https://github.com/unum-cloud/USearch},
|
|
555
|
-
version = {2.25.
|
|
556
|
-
year = {
|
|
557
|
-
month = oct,
|
|
552
|
+
version = {2.25.1},
|
|
553
|
+
year = {2026},
|
|
558
554
|
}
|
|
559
555
|
```
|
package/c/README.md
CHANGED
|
@@ -19,7 +19,7 @@ int main() {
|
|
|
19
19
|
usearch_error_t error = NULL;
|
|
20
20
|
usearch_init_options_t opts = {
|
|
21
21
|
.metric_kind = usearch_metric_cos_k,
|
|
22
|
-
.scalar_kind = usearch_scalar_f16_k,
|
|
22
|
+
.scalar_kind = usearch_scalar_f16_k, // or f32_k, bf16_k, e5m2_k, e4m3_k, e3m2_k, e2m3_k, i8_k, u8_k
|
|
23
23
|
.dimensions = dimensions,
|
|
24
24
|
.expansion_add = 0, // for defaults
|
|
25
25
|
.expansion_search = 0 // for defaults
|
package/c/test.c
CHANGED
|
@@ -448,6 +448,46 @@ void test_view(size_t const collection_size, size_t const dimensions) {
|
|
|
448
448
|
printf("Test: View - PASSED\n");
|
|
449
449
|
}
|
|
450
450
|
|
|
451
|
+
void test_mini_float_quantizations(size_t const collection_size, size_t const dimensions) {
|
|
452
|
+
printf("Test: Mini-float quantizations... %zu vectors, %zu dimensions\n", collection_size, dimensions);
|
|
453
|
+
usearch_scalar_kind_t kinds[] = {
|
|
454
|
+
usearch_scalar_e5m2_k,
|
|
455
|
+
usearch_scalar_e4m3_k,
|
|
456
|
+
usearch_scalar_e3m2_k,
|
|
457
|
+
usearch_scalar_e2m3_k,
|
|
458
|
+
};
|
|
459
|
+
float* data = create_vectors(collection_size, dimensions);
|
|
460
|
+
usearch_key_t* keys = (usearch_key_t*)malloc(collection_size * sizeof(usearch_key_t));
|
|
461
|
+
float* distances = (float*)malloc(collection_size * sizeof(float));
|
|
462
|
+
expect(keys && distances, "Failed to allocate memory");
|
|
463
|
+
|
|
464
|
+
for (size_t k = 0; k < sizeof(kinds) / sizeof(kinds[0]); ++k) {
|
|
465
|
+
usearch_error_t error = NULL;
|
|
466
|
+
usearch_init_options_t opts = create_options(dimensions);
|
|
467
|
+
opts.quantization = kinds[k];
|
|
468
|
+
usearch_index_t index = usearch_init(&opts, &error);
|
|
469
|
+
expect(!error, error);
|
|
470
|
+
usearch_reserve(index, collection_size, &error);
|
|
471
|
+
expect(!error, error);
|
|
472
|
+
for (size_t i = 0; i < collection_size; ++i) {
|
|
473
|
+
usearch_add(index, (usearch_key_t)i, data + i * dimensions, usearch_scalar_f32_k, &error);
|
|
474
|
+
expect(!error, error);
|
|
475
|
+
}
|
|
476
|
+
expect_eq(usearch_size(index, &error), collection_size, error);
|
|
477
|
+
for (size_t i = 0; i < collection_size; ++i) {
|
|
478
|
+
size_t found =
|
|
479
|
+
usearch_search(index, data + i * dimensions, usearch_scalar_f32_k, 1, keys, distances, &error);
|
|
480
|
+
expect(!error, error);
|
|
481
|
+
expect(found >= 1, "Vector not found");
|
|
482
|
+
}
|
|
483
|
+
usearch_free(index, &error);
|
|
484
|
+
}
|
|
485
|
+
free(data);
|
|
486
|
+
free(keys);
|
|
487
|
+
free(distances);
|
|
488
|
+
printf("Test: Mini-float quantizations - PASSED\n");
|
|
489
|
+
}
|
|
490
|
+
|
|
451
491
|
int main(int argc, char const* argv[]) {
|
|
452
492
|
install_crash_handlers();
|
|
453
493
|
printf("Running tests...\n");
|
|
@@ -464,6 +504,7 @@ int main(int argc, char const* argv[]) {
|
|
|
464
504
|
test_remove_vector(collection_sizes[index], dimensions[jdx]);
|
|
465
505
|
test_save_load(collection_sizes[index], dimensions[jdx]);
|
|
466
506
|
test_view(collection_sizes[index], dimensions[jdx]);
|
|
507
|
+
test_mini_float_quantizations(collection_sizes[index], dimensions[jdx]);
|
|
467
508
|
}
|
|
468
509
|
}
|
|
469
510
|
|
package/javascript/README.md
CHANGED
|
@@ -48,7 +48,7 @@ Customize your index with additional configuration options:
|
|
|
48
48
|
const index = new usearch.Index({
|
|
49
49
|
dimensions: 128,
|
|
50
50
|
metric: 'ip',
|
|
51
|
-
quantization: 'f32',
|
|
51
|
+
quantization: 'f32', // or 'bf16', 'f16', 'e5m2', 'e4m3', 'e3m2', 'e2m3', 'u8', 'i8', 'b1'
|
|
52
52
|
connectivity: 10,
|
|
53
53
|
expansion_add: 5,
|
|
54
54
|
expansion_search: 3,
|
|
@@ -1,23 +1,23 @@
|
|
|
1
|
-
const nodeTest = require(
|
|
1
|
+
const nodeTest = require("node:test");
|
|
2
2
|
const realTest = nodeTest.test; // the original function
|
|
3
3
|
|
|
4
4
|
function loggedTest(name, options, fn) {
|
|
5
5
|
// The API has two call signatures:
|
|
6
6
|
// test(name, fn)
|
|
7
7
|
// test(name, options, fn)
|
|
8
|
-
if (typeof options ===
|
|
8
|
+
if (typeof options === "function") {
|
|
9
9
|
fn = options;
|
|
10
10
|
options = undefined;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
// Wrap the body so we can log before / after
|
|
14
14
|
const wrapped = async (t) => {
|
|
15
|
-
console.log(
|
|
15
|
+
console.log("▶", name);
|
|
16
16
|
try {
|
|
17
17
|
await fn(t); // run the user’s test
|
|
18
|
-
console.log(
|
|
18
|
+
console.log("✓", name);
|
|
19
19
|
} catch (err) {
|
|
20
|
-
console.log(
|
|
20
|
+
console.log("✖", name);
|
|
21
21
|
throw err; // re-throw so the runner records the failure
|
|
22
22
|
}
|
|
23
23
|
};
|
|
@@ -30,43 +30,43 @@ function loggedTest(name, options, fn) {
|
|
|
30
30
|
global.test = loggedTest;
|
|
31
31
|
module.exports = loggedTest; // for completeness if this file is `require`d
|
|
32
32
|
|
|
33
|
-
const assert = require(
|
|
34
|
-
const fs = require(
|
|
35
|
-
const os = require(
|
|
36
|
-
const path = require(
|
|
37
|
-
const usearch = require(
|
|
33
|
+
const assert = require("node:assert");
|
|
34
|
+
const fs = require("node:fs");
|
|
35
|
+
const os = require("node:os");
|
|
36
|
+
const path = require("node:path");
|
|
37
|
+
const usearch = require("./dist/cjs/usearch.js");
|
|
38
38
|
|
|
39
39
|
function assertAlmostEqual(actual, expected, tolerance = 1e-6) {
|
|
40
40
|
const lowerBound = expected - tolerance;
|
|
41
41
|
const upperBound = expected + tolerance;
|
|
42
42
|
assert(
|
|
43
43
|
actual >= lowerBound && actual <= upperBound,
|
|
44
|
-
`Expected ${actual} to be almost equal to ${expected}
|
|
44
|
+
`Expected ${actual} to be almost equal to ${expected}`,
|
|
45
45
|
);
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
const THREADS = 2;
|
|
49
49
|
|
|
50
|
-
test(
|
|
51
|
-
await t.test(
|
|
52
|
-
const index = new usearch.Index(2,
|
|
50
|
+
test("Single-entry operations", async (t) => {
|
|
51
|
+
await t.test("index info", () => {
|
|
52
|
+
const index = new usearch.Index(2, "l2sq");
|
|
53
53
|
|
|
54
|
-
assert.equal(index.connectivity(), 16,
|
|
55
|
-
assert.equal(index.dimensions(), 2,
|
|
56
|
-
assert.equal(index.size(), 0,
|
|
54
|
+
assert.equal(index.connectivity(), 16, "connectivity should be 16");
|
|
55
|
+
assert.equal(index.dimensions(), 2, "dimensions should be 2");
|
|
56
|
+
assert.equal(index.size(), 0, "initial size should be 0");
|
|
57
57
|
});
|
|
58
58
|
|
|
59
|
-
await t.test(
|
|
60
|
-
const index = new usearch.Index(2,
|
|
59
|
+
await t.test("add", () => {
|
|
60
|
+
const index = new usearch.Index(2, "l2sq");
|
|
61
61
|
|
|
62
62
|
index.add(15n, new Float32Array([10, 20]));
|
|
63
63
|
index.add(16n, new Float32Array([10, 25]));
|
|
64
64
|
|
|
65
|
-
assert.equal(index.size(), 2,
|
|
65
|
+
assert.equal(index.size(), 2, "size after adding elements should be 2");
|
|
66
66
|
assert.equal(
|
|
67
67
|
index.contains(15),
|
|
68
68
|
true,
|
|
69
|
-
|
|
69
|
+
"entry must be present after insertion",
|
|
70
70
|
);
|
|
71
71
|
|
|
72
72
|
const results = index.search(new Float32Array([13, 14]), 2);
|
|
@@ -74,17 +74,17 @@ test('Single-entry operations', async (t) => {
|
|
|
74
74
|
assert.deepEqual(
|
|
75
75
|
results.keys,
|
|
76
76
|
new BigUint64Array([15n, 16n]),
|
|
77
|
-
|
|
77
|
+
"keys should be 15 and 16",
|
|
78
78
|
);
|
|
79
79
|
assert.deepEqual(
|
|
80
80
|
results.distances,
|
|
81
81
|
new Float32Array([45, 130]),
|
|
82
|
-
|
|
82
|
+
"distances should be 45 and 130",
|
|
83
83
|
);
|
|
84
84
|
});
|
|
85
85
|
|
|
86
|
-
await t.test(
|
|
87
|
-
const index = new usearch.Index(2,
|
|
86
|
+
await t.test("remove", () => {
|
|
87
|
+
const index = new usearch.Index(2, "l2sq");
|
|
88
88
|
|
|
89
89
|
index.add(15n, new Float32Array([10, 20]));
|
|
90
90
|
index.add(16n, new Float32Array([10, 25]));
|
|
@@ -93,11 +93,11 @@ test('Single-entry operations', async (t) => {
|
|
|
93
93
|
|
|
94
94
|
assert.equal(index.remove(15n), 1);
|
|
95
95
|
|
|
96
|
-
assert.equal(index.size(), 3,
|
|
96
|
+
assert.equal(index.size(), 3, "size after removing elements should be 3");
|
|
97
97
|
assert.equal(
|
|
98
98
|
index.contains(15),
|
|
99
99
|
false,
|
|
100
|
-
|
|
100
|
+
"entry must be absent after insertion",
|
|
101
101
|
);
|
|
102
102
|
|
|
103
103
|
const results = index.search(new Float32Array([13, 14]), 2);
|
|
@@ -105,37 +105,37 @@ test('Single-entry operations', async (t) => {
|
|
|
105
105
|
assert.deepEqual(
|
|
106
106
|
results.keys,
|
|
107
107
|
new BigUint64Array([16n, 25n]),
|
|
108
|
-
|
|
108
|
+
"keys should not include 15",
|
|
109
109
|
);
|
|
110
110
|
});
|
|
111
111
|
});
|
|
112
112
|
|
|
113
|
-
test(
|
|
114
|
-
await t.test(
|
|
115
|
-
const indexBatch = new usearch.Index(2,
|
|
113
|
+
test("Batch operations", async (t) => {
|
|
114
|
+
await t.test("add", () => {
|
|
115
|
+
const indexBatch = new usearch.Index(2, "l2sq");
|
|
116
116
|
|
|
117
117
|
const keys = [15n, 16n];
|
|
118
118
|
const vectors = [new Float32Array([10, 20]), new Float32Array([10, 25])];
|
|
119
119
|
|
|
120
120
|
indexBatch.add(keys, vectors);
|
|
121
|
-
assert.equal(indexBatch.size(), 2,
|
|
121
|
+
assert.equal(indexBatch.size(), 2, "size after adding batch should be 2");
|
|
122
122
|
|
|
123
123
|
const results = indexBatch.search(new Float32Array([13, 14]), 2);
|
|
124
124
|
|
|
125
125
|
assert.deepEqual(
|
|
126
126
|
results.keys,
|
|
127
127
|
new BigUint64Array([15n, 16n]),
|
|
128
|
-
|
|
128
|
+
"keys should be 15 and 16",
|
|
129
129
|
);
|
|
130
130
|
assert.deepEqual(
|
|
131
131
|
results.distances,
|
|
132
132
|
new Float32Array([45, 130]),
|
|
133
|
-
|
|
133
|
+
"distances should be 45 and 130",
|
|
134
134
|
);
|
|
135
135
|
});
|
|
136
136
|
|
|
137
|
-
await t.test(
|
|
138
|
-
const indexBatch = new usearch.Index(2,
|
|
137
|
+
await t.test("remove", () => {
|
|
138
|
+
const indexBatch = new usearch.Index(2, "l2sq");
|
|
139
139
|
|
|
140
140
|
const keys = [15n, 16n, 25n, 26n];
|
|
141
141
|
const vectors = [
|
|
@@ -147,53 +147,58 @@ test('Batch operations', async (t) => {
|
|
|
147
147
|
indexBatch.add(keys, vectors);
|
|
148
148
|
|
|
149
149
|
assert.deepEqual(indexBatch.remove([15n, 25n]), [1, 1]);
|
|
150
|
-
assert.equal(indexBatch.size(), 2,
|
|
150
|
+
assert.equal(indexBatch.size(), 2, "size after removing batch should be 2");
|
|
151
151
|
|
|
152
152
|
const results = indexBatch.search(new Float32Array([13, 14]), 2);
|
|
153
153
|
|
|
154
154
|
assert.deepEqual(
|
|
155
155
|
results.keys,
|
|
156
156
|
new BigUint64Array([16n, 26n]),
|
|
157
|
-
|
|
157
|
+
"keys should not include 15 and 25",
|
|
158
158
|
);
|
|
159
159
|
});
|
|
160
160
|
|
|
161
|
-
await t.test(
|
|
162
|
-
const indexBatch = new usearch.Index(3,
|
|
161
|
+
await t.test("search", () => {
|
|
162
|
+
const indexBatch = new usearch.Index(3, "l2sq");
|
|
163
163
|
|
|
164
164
|
const keys = [15n, 16n];
|
|
165
|
-
const vectors = [
|
|
165
|
+
const vectors = [
|
|
166
|
+
new Float32Array([10, 20, 30]),
|
|
167
|
+
new Float32Array([10, 25, 35]),
|
|
168
|
+
];
|
|
166
169
|
|
|
167
170
|
indexBatch.add(keys, vectors);
|
|
168
|
-
assert.equal(indexBatch.size(), 2,
|
|
171
|
+
assert.equal(indexBatch.size(), 2, "size after adding batch should be 2");
|
|
172
|
+
|
|
173
|
+
const results = indexBatch.search(
|
|
174
|
+
[new Float32Array([13, 14, 15]), new Float32Array([13, 14, 15])],
|
|
175
|
+
2,
|
|
176
|
+
);
|
|
169
177
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
assert.equal(results.k, 2, 'k should be 2');
|
|
178
|
+
assert.equal(results.k, 2, "k should be 2");
|
|
173
179
|
|
|
174
180
|
assert.deepEqual(
|
|
175
181
|
results.counts,
|
|
176
|
-
new BigUint64Array([
|
|
177
|
-
|
|
182
|
+
new BigUint64Array([2n, 2n]),
|
|
183
|
+
"counts should be 2 for both keys",
|
|
178
184
|
);
|
|
179
185
|
|
|
180
186
|
assert.deepEqual(
|
|
181
187
|
results.keys,
|
|
182
188
|
new BigUint64Array([15n, 16n, 15n, 16n]),
|
|
183
|
-
|
|
189
|
+
"keys should be 15 and 16",
|
|
184
190
|
);
|
|
185
191
|
assert.deepEqual(
|
|
186
192
|
results.distances,
|
|
187
|
-
new Float32Array(
|
|
188
|
-
|
|
193
|
+
new Float32Array([270, 530, 270, 530]),
|
|
194
|
+
"distances should be 270 and 530",
|
|
189
195
|
);
|
|
190
196
|
});
|
|
191
|
-
|
|
192
197
|
});
|
|
193
198
|
|
|
194
|
-
test(
|
|
199
|
+
test("Expected results", () => {
|
|
195
200
|
const index = new usearch.Index({
|
|
196
|
-
metric:
|
|
201
|
+
metric: "l2sq",
|
|
197
202
|
connectivity: 16,
|
|
198
203
|
dimensions: 3,
|
|
199
204
|
});
|
|
@@ -205,9 +210,9 @@ test('Expected results', () => {
|
|
|
205
210
|
assertAlmostEqual(results.distances[0], new Float32Array([0]));
|
|
206
211
|
});
|
|
207
212
|
|
|
208
|
-
test(
|
|
213
|
+
test("Multithread search returns same results", () => {
|
|
209
214
|
const index = new usearch.Index({
|
|
210
|
-
metric:
|
|
215
|
+
metric: "l2sq",
|
|
211
216
|
connectivity: 16,
|
|
212
217
|
dimensions: 3,
|
|
213
218
|
});
|
|
@@ -218,14 +223,14 @@ test('Multithread search returns same results', () => {
|
|
|
218
223
|
const results_2 = index.search(
|
|
219
224
|
new Float32Array([0.2, 0.6, 0.4]),
|
|
220
225
|
10,
|
|
221
|
-
THREADS
|
|
226
|
+
THREADS,
|
|
222
227
|
);
|
|
223
228
|
|
|
224
229
|
assert.deepEqual(results_1.keys, results_2.keys);
|
|
225
230
|
assertAlmostEqual(results_1.distances, results_2.distances);
|
|
226
231
|
});
|
|
227
232
|
|
|
228
|
-
test(
|
|
233
|
+
test("Exact search", async (t) => {
|
|
229
234
|
const dataset = [
|
|
230
235
|
new Float32Array([0.2, 0.6, 0.4]),
|
|
231
236
|
new Float32Array([0.6, 0.6, 0.4]),
|
|
@@ -233,16 +238,16 @@ test('Exact search', async (t) => {
|
|
|
233
238
|
const queries = new Float32Array([0.2, 0.6, 0.4]);
|
|
234
239
|
const dimensions = 3;
|
|
235
240
|
const count = 2;
|
|
236
|
-
const metric =
|
|
241
|
+
const metric = "l2sq";
|
|
237
242
|
|
|
238
|
-
await t.test(
|
|
243
|
+
await t.test("Single core search", () => {
|
|
239
244
|
const result = usearch.exactSearch(
|
|
240
245
|
dataset,
|
|
241
246
|
queries,
|
|
242
247
|
dimensions,
|
|
243
248
|
count,
|
|
244
249
|
metric,
|
|
245
|
-
1 // threads
|
|
250
|
+
1, // threads
|
|
246
251
|
);
|
|
247
252
|
|
|
248
253
|
assert.deepEqual(result.keys, new BigUint64Array([0n, 1n]));
|
|
@@ -250,14 +255,14 @@ test('Exact search', async (t) => {
|
|
|
250
255
|
assertAlmostEqual(new Float32Array([0.16]), result.distances[1]);
|
|
251
256
|
});
|
|
252
257
|
|
|
253
|
-
await t.test(
|
|
258
|
+
await t.test("Multithreaded search", () => {
|
|
254
259
|
const result = usearch.exactSearch(
|
|
255
260
|
dataset,
|
|
256
261
|
queries,
|
|
257
262
|
dimensions,
|
|
258
263
|
count,
|
|
259
264
|
metric,
|
|
260
|
-
THREADS // threads
|
|
265
|
+
THREADS, // threads
|
|
261
266
|
);
|
|
262
267
|
|
|
263
268
|
assert.deepEqual(result.keys, new BigUint64Array([0n, 1n]));
|
|
@@ -266,30 +271,30 @@ test('Exact search', async (t) => {
|
|
|
266
271
|
});
|
|
267
272
|
});
|
|
268
273
|
|
|
269
|
-
test(
|
|
274
|
+
test("Expected count()", async (t) => {
|
|
270
275
|
const index = new usearch.Index({
|
|
271
|
-
metric:
|
|
276
|
+
metric: "l2sq",
|
|
272
277
|
connectivity: 16,
|
|
273
278
|
dimensions: 3,
|
|
274
279
|
});
|
|
275
280
|
index.add(
|
|
276
281
|
[42n, 43n],
|
|
277
|
-
[new Float32Array([0.2, 0.6, 0.4]), new Float32Array([0.2, 0.6, 0.4])]
|
|
282
|
+
[new Float32Array([0.2, 0.6, 0.4]), new Float32Array([0.2, 0.6, 0.4])],
|
|
278
283
|
);
|
|
279
284
|
|
|
280
|
-
await t.test(
|
|
285
|
+
await t.test("Argument is a number", () => {
|
|
281
286
|
assert.equal(1, index.count(43n));
|
|
282
287
|
});
|
|
283
|
-
await t.test(
|
|
288
|
+
await t.test("Argument is a number (does not exist)", () => {
|
|
284
289
|
assert.equal(0, index.count(44n));
|
|
285
290
|
});
|
|
286
|
-
await t.test(
|
|
291
|
+
await t.test("Argument is an array", () => {
|
|
287
292
|
assert.deepEqual([1, 1, 0], index.count([42n, 43n, 44n]));
|
|
288
293
|
});
|
|
289
294
|
});
|
|
290
295
|
|
|
291
|
-
test(
|
|
292
|
-
const indexBatch = new usearch.Index(2,
|
|
296
|
+
test("Operations with invalid values", () => {
|
|
297
|
+
const indexBatch = new usearch.Index(2, "l2sq");
|
|
293
298
|
|
|
294
299
|
const keys = [NaN, 16n];
|
|
295
300
|
const vectors = [new Float32Array([10, 30]), new Float32Array([1, 5])];
|
|
@@ -301,10 +306,10 @@ test('Operations with invalid values', () => {
|
|
|
301
306
|
assert.throws(() => indexBatch.search(NaN, 2));
|
|
302
307
|
});
|
|
303
308
|
|
|
304
|
-
test(
|
|
305
|
-
await t.test(
|
|
309
|
+
test("Invalid operations", async (t) => {
|
|
310
|
+
await t.test("Add the same keys", () => {
|
|
306
311
|
const index = new usearch.Index({
|
|
307
|
-
metric:
|
|
312
|
+
metric: "l2sq",
|
|
308
313
|
connectivity: 16,
|
|
309
314
|
dimensions: 3,
|
|
310
315
|
});
|
|
@@ -312,9 +317,9 @@ test('Invalid operations', async (t) => {
|
|
|
312
317
|
assert.throws(() => index.add(42n, new Float32Array([0.2, 0.6, 0.4])));
|
|
313
318
|
});
|
|
314
319
|
|
|
315
|
-
await t.test(
|
|
320
|
+
await t.test("Batch add containing the same key", () => {
|
|
316
321
|
const index = new usearch.Index({
|
|
317
|
-
metric:
|
|
322
|
+
metric: "l2sq",
|
|
318
323
|
connectivity: 16,
|
|
319
324
|
dimensions: 3,
|
|
320
325
|
});
|
|
@@ -326,18 +331,18 @@ test('Invalid operations', async (t) => {
|
|
|
326
331
|
[0.1, 0.6, 0.4],
|
|
327
332
|
[0.2, 0.6, 0.4],
|
|
328
333
|
[0.3, 0.6, 0.4],
|
|
329
|
-
]
|
|
334
|
+
],
|
|
330
335
|
);
|
|
331
336
|
});
|
|
332
337
|
});
|
|
333
338
|
});
|
|
334
339
|
|
|
335
|
-
test(
|
|
336
|
-
const indexPath = path.join(os.tmpdir(),
|
|
340
|
+
test("Serialization", async (t) => {
|
|
341
|
+
const indexPath = path.join(os.tmpdir(), "usearch.test.index");
|
|
337
342
|
|
|
338
343
|
t.beforeEach(() => {
|
|
339
344
|
const index = new usearch.Index({
|
|
340
|
-
metric:
|
|
345
|
+
metric: "l2sq",
|
|
341
346
|
connectivity: 16,
|
|
342
347
|
dimensions: 3,
|
|
343
348
|
});
|
|
@@ -349,9 +354,9 @@ test('Serialization', async (t) => {
|
|
|
349
354
|
fs.unlinkSync(indexPath);
|
|
350
355
|
});
|
|
351
356
|
|
|
352
|
-
await t.test(
|
|
357
|
+
await t.test("load", () => {
|
|
353
358
|
const index = new usearch.Index({
|
|
354
|
-
metric:
|
|
359
|
+
metric: "l2sq",
|
|
355
360
|
connectivity: 16,
|
|
356
361
|
dimensions: 3,
|
|
357
362
|
});
|
|
@@ -359,7 +364,7 @@ test('Serialization', async (t) => {
|
|
|
359
364
|
const results = index.search(
|
|
360
365
|
new Float32Array([0.2, 0.6, 0.4]),
|
|
361
366
|
10,
|
|
362
|
-
THREADS
|
|
367
|
+
THREADS,
|
|
363
368
|
);
|
|
364
369
|
|
|
365
370
|
assert.equal(index.size(), 1);
|
|
@@ -371,11 +376,11 @@ test('Serialization', async (t) => {
|
|
|
371
376
|
// The following error in afterEach().
|
|
372
377
|
// `error: "EBUSY: resource busy or locked, unlink`
|
|
373
378
|
await t.test(
|
|
374
|
-
|
|
375
|
-
{ skip: process.platform ===
|
|
379
|
+
"view: Read data",
|
|
380
|
+
{ skip: process.platform === "win32" },
|
|
376
381
|
() => {
|
|
377
382
|
const index = new usearch.Index({
|
|
378
|
-
metric:
|
|
383
|
+
metric: "l2sq",
|
|
379
384
|
connectivity: 16,
|
|
380
385
|
dimensions: 3,
|
|
381
386
|
});
|
|
@@ -385,15 +390,15 @@ test('Serialization', async (t) => {
|
|
|
385
390
|
assert.equal(index.size(), 1);
|
|
386
391
|
assert.deepEqual(results.keys, new BigUint64Array([42n]));
|
|
387
392
|
assertAlmostEqual(results.distances[0], new Float32Array([0]));
|
|
388
|
-
}
|
|
393
|
+
},
|
|
389
394
|
);
|
|
390
395
|
|
|
391
396
|
await t.test(
|
|
392
|
-
|
|
393
|
-
{ skip: process.platform ===
|
|
397
|
+
"view: Invalid operations: add",
|
|
398
|
+
{ skip: process.platform === "win32" },
|
|
394
399
|
() => {
|
|
395
400
|
const index = new usearch.Index({
|
|
396
|
-
metric:
|
|
401
|
+
metric: "l2sq",
|
|
397
402
|
connectivity: 16,
|
|
398
403
|
dimensions: 3,
|
|
399
404
|
});
|
|
@@ -401,15 +406,15 @@ test('Serialization', async (t) => {
|
|
|
401
406
|
|
|
402
407
|
// Can't add to an immutable index
|
|
403
408
|
assert.throws(() => index.add(43n, new Float32Array([0.2, 0.6, 0.4])));
|
|
404
|
-
}
|
|
409
|
+
},
|
|
405
410
|
);
|
|
406
411
|
|
|
407
412
|
await t.test(
|
|
408
|
-
|
|
409
|
-
{ skip: process.platform ===
|
|
413
|
+
"view: Invalid operations: remove",
|
|
414
|
+
{ skip: process.platform === "win32" },
|
|
410
415
|
() => {
|
|
411
416
|
const index = new usearch.Index({
|
|
412
|
-
metric:
|
|
417
|
+
metric: "l2sq",
|
|
413
418
|
connectivity: 16,
|
|
414
419
|
dimensions: 3,
|
|
415
420
|
});
|
|
@@ -417,6 +422,24 @@ test('Serialization', async (t) => {
|
|
|
417
422
|
|
|
418
423
|
// Can't remove from an immutable index
|
|
419
424
|
assert.throws(() => index.remove(42n));
|
|
420
|
-
}
|
|
425
|
+
},
|
|
421
426
|
);
|
|
422
427
|
});
|
|
428
|
+
|
|
429
|
+
test("Mini-float quantization types", async (t) => {
|
|
430
|
+
for (const quantization of ["e5m2", "e4m3", "e3m2", "e2m3"]) {
|
|
431
|
+
await t.test(`add and search with ${quantization}`, () => {
|
|
432
|
+
const index = new usearch.Index({
|
|
433
|
+
metric: "cos",
|
|
434
|
+
dimensions: 64,
|
|
435
|
+
quantization: quantization,
|
|
436
|
+
});
|
|
437
|
+
const vector = new Float32Array(64);
|
|
438
|
+
for (let i = 0; i < 64; i++) vector[i] = i * 0.1;
|
|
439
|
+
index.add(1n, vector);
|
|
440
|
+
|
|
441
|
+
const results = index.search(vector, 1);
|
|
442
|
+
assert.equal(results.keys[0], 1n);
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
});
|
package/package.json
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|