usearch 0.18.8 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +1 -1
- package/README.md +187 -40
- package/c/lib.cpp +71 -51
- package/c/test.c +182 -0
- package/c/usearch.h +11 -5
- package/conanfile.py +1 -1
- package/cpp/bench.cpp +6 -6
- package/cpp/test.cpp +11 -9
- package/docs/compilation.md +1 -0
- package/include/usearch/index.hpp +290 -54
- package/include/usearch/index_punned_dense.hpp +125 -108
- package/include/usearch/index_punned_helpers.hpp +2 -12
- package/objc/USearchObjective.mm +4 -0
- package/objc/include/USearchObjective.h +5 -0
- package/package.json +2 -2
- package/python/lib.cpp +234 -199
- package/python/scripts/bench.ipynb +13 -2113
- package/python/scripts/bench.py +45 -29
- package/python/scripts/index_faiss.py +19 -19
- package/python/scripts/test.py +320 -61
- package/python/usearch/eval.py +100 -49
- package/python/usearch/index.py +245 -129
- package/python/usearch/io.py +25 -25
- package/python/usearch/join.py +2 -4
- package/python/usearch/numba.py +22 -18
- package/python/usearch/server.py +38 -26
- package/setup.py +56 -59
- package/simsimd/.vscode/settings.json +2 -1
- package/simsimd/CMakeLists.txt +1 -1
- package/simsimd/README.md +14 -2
- package/simsimd/bench.cxx +44 -27
- package/simsimd/include/simsimd/simsimd.h +2 -1
- package/swift/Test.swift +13 -9
- package/simsimd/include/simsimd/simsimd.hpp +0 -443
package/python/scripts/bench.py
CHANGED
|
@@ -24,15 +24,13 @@ def bench_speed(
|
|
|
24
24
|
jit: bool = False,
|
|
25
25
|
train: bool = False,
|
|
26
26
|
) -> pd.DataFrame:
|
|
27
|
-
|
|
28
27
|
# Build various indexes:
|
|
29
28
|
indexes = []
|
|
30
29
|
jit_options = [False]
|
|
31
30
|
if jit:
|
|
32
31
|
jit_options.append(jit)
|
|
33
|
-
dtype_options = [
|
|
32
|
+
dtype_options = ["f32", "f16", "f8"]
|
|
34
33
|
for jit, dtype in itertools.product(jit_options, dtype_options):
|
|
35
|
-
|
|
36
34
|
index = Index(
|
|
37
35
|
ndim=eval.ndim,
|
|
38
36
|
jit=jit,
|
|
@@ -40,9 +38,9 @@ def bench_speed(
|
|
|
40
38
|
expansion_add=expansion_add,
|
|
41
39
|
expansion_search=expansion_search,
|
|
42
40
|
connectivity=connectivity,
|
|
43
|
-
path=
|
|
41
|
+
path="USearch" + ["", "+JIT"][jit] + ":" + dtype,
|
|
44
42
|
)
|
|
45
|
-
|
|
43
|
+
|
|
46
44
|
# Skip the cases, where JIT-ing is impossible
|
|
47
45
|
if jit and not index.jit:
|
|
48
46
|
continue
|
|
@@ -51,41 +49,60 @@ def bench_speed(
|
|
|
51
49
|
# Add FAISS indexes to the mix:
|
|
52
50
|
try:
|
|
53
51
|
from index_faiss import IndexFAISS, IndexQuantizedFAISS
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
connectivity=connectivity,
|
|
59
|
-
path='FAISS:f32',
|
|
60
|
-
))
|
|
61
|
-
if train:
|
|
62
|
-
indexes.append(IndexQuantizedFAISS(
|
|
63
|
-
train=eval.tasks[0].vectors,
|
|
52
|
+
|
|
53
|
+
indexes.append(
|
|
54
|
+
IndexFAISS(
|
|
55
|
+
ndim=eval.ndim,
|
|
64
56
|
expansion_add=expansion_add,
|
|
65
57
|
expansion_search=expansion_search,
|
|
66
58
|
connectivity=connectivity,
|
|
67
|
-
path=
|
|
68
|
-
)
|
|
59
|
+
path="FAISS:f32",
|
|
60
|
+
)
|
|
61
|
+
)
|
|
62
|
+
if train:
|
|
63
|
+
indexes.append(
|
|
64
|
+
IndexQuantizedFAISS(
|
|
65
|
+
train=eval.tasks[0].vectors,
|
|
66
|
+
expansion_add=expansion_add,
|
|
67
|
+
expansion_search=expansion_search,
|
|
68
|
+
connectivity=connectivity,
|
|
69
|
+
path="FAISS+IVFPQ:f32",
|
|
70
|
+
)
|
|
71
|
+
)
|
|
69
72
|
except (ImportError, ModuleNotFoundError):
|
|
70
73
|
pass
|
|
71
74
|
|
|
72
75
|
# Time to evaluate:
|
|
73
76
|
results = [eval(index) for index in indexes]
|
|
74
|
-
return pd.DataFrame(
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
77
|
+
return pd.DataFrame(
|
|
78
|
+
{
|
|
79
|
+
"names": [i.path for i in indexes],
|
|
80
|
+
"add_per_second": [x["add_per_second"] for x in results],
|
|
81
|
+
"search_per_second": [x["search_per_second"] for x in results],
|
|
82
|
+
"recall_at_one": [x["recall_at_one"] for x in results],
|
|
83
|
+
}
|
|
84
|
+
)
|
|
80
85
|
|
|
81
86
|
|
|
82
87
|
def bench_params(
|
|
83
88
|
count: int = 1_000_000,
|
|
84
89
|
connectivities: int = range(10, 20),
|
|
85
90
|
dimensions: List[int] = [
|
|
86
|
-
2,
|
|
87
|
-
|
|
88
|
-
|
|
91
|
+
2,
|
|
92
|
+
3,
|
|
93
|
+
4,
|
|
94
|
+
8,
|
|
95
|
+
16,
|
|
96
|
+
32,
|
|
97
|
+
96,
|
|
98
|
+
100,
|
|
99
|
+
256,
|
|
100
|
+
384,
|
|
101
|
+
512,
|
|
102
|
+
768,
|
|
103
|
+
1024,
|
|
104
|
+
1536,
|
|
105
|
+
],
|
|
89
106
|
expansion_add: int = DEFAULT_EXPANSION_ADD,
|
|
90
107
|
expansion_search: int = DEFAULT_EXPANSION_SEARCH,
|
|
91
108
|
) -> pd.DataFrame:
|
|
@@ -97,7 +114,6 @@ def bench_params(
|
|
|
97
114
|
|
|
98
115
|
results = []
|
|
99
116
|
for connectivity, ndim in itertools.product(connectivities, dimensions):
|
|
100
|
-
|
|
101
117
|
task = AddTask(
|
|
102
118
|
labels=np.arange(count, dtype=Label),
|
|
103
119
|
vectors=np.random.rand(count, ndim).astype(np.float32),
|
|
@@ -109,8 +125,8 @@ def bench_params(
|
|
|
109
125
|
expansion_search=expansion_search,
|
|
110
126
|
)
|
|
111
127
|
result = asdict(task(index))
|
|
112
|
-
result[
|
|
113
|
-
result[
|
|
128
|
+
result["ndim"] = dimensions
|
|
129
|
+
result["connectivity"] = connectivity
|
|
114
130
|
results.append(result)
|
|
115
131
|
|
|
116
132
|
# return self._execute_tasks(
|
|
@@ -13,7 +13,6 @@ from usearch.index import (
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class IndexFAISS:
|
|
16
|
-
|
|
17
16
|
def __init__(
|
|
18
17
|
self,
|
|
19
18
|
index: IndexHNSWFlat = None,
|
|
@@ -22,9 +21,9 @@ class IndexFAISS:
|
|
|
22
21
|
expansion_add: int = DEFAULT_EXPANSION_ADD,
|
|
23
22
|
expansion_search: int = DEFAULT_EXPANSION_SEARCH,
|
|
24
23
|
path: Optional[os.PathLike] = None,
|
|
25
|
-
*args,
|
|
24
|
+
*args,
|
|
25
|
+
**kwargs,
|
|
26
26
|
):
|
|
27
|
-
|
|
28
27
|
if index is None:
|
|
29
28
|
index = IndexHNSWFlat(ndim, connectivity)
|
|
30
29
|
index.hnsw.efConstruction = expansion_add
|
|
@@ -32,11 +31,11 @@ class IndexFAISS:
|
|
|
32
31
|
|
|
33
32
|
self._faiss = index
|
|
34
33
|
self._specs = {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
34
|
+
"Class": "usearch.IndexFAISS",
|
|
35
|
+
"Dimensions": ndim,
|
|
36
|
+
"Connectivity": connectivity,
|
|
37
|
+
"Expansion@Add": expansion_add,
|
|
38
|
+
"Expansion@Search": expansion_search,
|
|
40
39
|
}
|
|
41
40
|
|
|
42
41
|
self.path = path
|
|
@@ -58,9 +57,11 @@ class IndexFAISS:
|
|
|
58
57
|
|
|
59
58
|
@property
|
|
60
59
|
def specs(self) -> dict:
|
|
61
|
-
self._specs.update(
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
self._specs.update(
|
|
61
|
+
{
|
|
62
|
+
"Size": len(self),
|
|
63
|
+
}
|
|
64
|
+
)
|
|
64
65
|
return self._specs
|
|
65
66
|
|
|
66
67
|
def load(self, path: os.PathLike):
|
|
@@ -68,33 +69,32 @@ class IndexFAISS:
|
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
class IndexQuantizedFAISS(IndexFAISS):
|
|
71
|
-
|
|
72
72
|
def __init__(
|
|
73
73
|
self,
|
|
74
74
|
train: np.ndarray,
|
|
75
75
|
connectivity: int = DEFAULT_CONNECTIVITY,
|
|
76
76
|
expansion_add: int = DEFAULT_EXPANSION_ADD,
|
|
77
77
|
expansion_search: int = DEFAULT_EXPANSION_SEARCH,
|
|
78
|
-
*args,
|
|
78
|
+
*args,
|
|
79
|
+
**kwargs,
|
|
79
80
|
):
|
|
80
|
-
|
|
81
81
|
ndim = train.shape[1]
|
|
82
82
|
super().__init__(
|
|
83
83
|
ndim=ndim,
|
|
84
84
|
connectivity=connectivity,
|
|
85
85
|
expansion_add=expansion_add,
|
|
86
86
|
expansion_search=expansion_search,
|
|
87
|
-
*args,
|
|
87
|
+
*args,
|
|
88
|
+
**kwargs,
|
|
88
89
|
)
|
|
89
90
|
|
|
90
91
|
nlist = 10000 # Number of inverted lists (number of partitions).
|
|
91
92
|
nsegment = 16 # Number of segments for PQ (number of subquantizers).
|
|
92
|
-
nbit = 8
|
|
93
|
+
nbit = 8 # Number of bits to encode each segment.
|
|
93
94
|
|
|
94
95
|
self._original_faiss = self._faiss
|
|
95
|
-
self._faiss = IndexIVFPQ(
|
|
96
|
-
self._faiss, ndim, nlist, nsegment, nbit)
|
|
96
|
+
self._faiss = IndexIVFPQ(self._faiss, ndim, nlist, nsegment, nbit)
|
|
97
97
|
|
|
98
98
|
self._faiss.train(train)
|
|
99
99
|
self._faiss.nprobe = 10
|
|
100
|
-
self._specs[
|
|
100
|
+
self._specs["Class"] = "usearch.IndexQuantizedFAISS"
|