usearch 2.1.2 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +18 -3
- package/CMakeLists.txt +15 -0
- package/Cargo.toml +1 -1
- package/README.md +2 -2
- package/conanfile.py +1 -1
- package/include/usearch/index_plugins.hpp +50 -6
- package/package.json +1 -1
- package/python/lib.cpp +11 -0
- package/python/usearch/index.py +17 -4
- package/setup.py +3 -4
- package/simsimd/.github/workflows/prerelease.yml +42 -9
- package/simsimd/.github/workflows/release.yml +15 -11
- package/simsimd/.vscode/settings.json +5 -1
- package/simsimd/CMakeLists.txt +5 -2
- package/simsimd/README.md +69 -48
- package/simsimd/bench.cxx +5 -6
- package/simsimd/include/simsimd/simsimd.h +300 -140
- package/simsimd/include/simsimd/simsimd_bio.h +54 -0
- package/simsimd/include/simsimd/simsimd_chem.h +65 -24
- package/simsimd/include/simsimd/simsimd_text.h +92 -0
- package/simsimd/package.json +1 -1
- package/simsimd/pyproject.toml +2 -1
- package/simsimd/python/{python.c → lib.c} +44 -15
- package/simsimd/python/test.py +35 -20
- package/simsimd/setup.py +2 -6
- package/wasmer.toml +1 -1
- /package/{cluster.ipynb → python/scripts/cluster.ipynb} +0 -0
|
@@ -74,8 +74,9 @@ jobs:
|
|
|
74
74
|
name: Test Python
|
|
75
75
|
runs-on: ${{ matrix.os }}
|
|
76
76
|
strategy:
|
|
77
|
+
fail-fast: false
|
|
77
78
|
matrix:
|
|
78
|
-
os: [ubuntu-
|
|
79
|
+
os: [ubuntu-22.04, macOS-11, windows-2022]
|
|
79
80
|
python-version: ["3.11"]
|
|
80
81
|
|
|
81
82
|
steps:
|
|
@@ -89,8 +90,16 @@ jobs:
|
|
|
89
90
|
run: |
|
|
90
91
|
python -m pip install --upgrade pip
|
|
91
92
|
pip install pytest numpy
|
|
93
|
+
- name: Build locally on Ubuntu
|
|
94
|
+
run: |
|
|
95
|
+
export CC=/usr/bin/gcc-12
|
|
96
|
+
export CXX=/usr/bin/g++-12
|
|
97
|
+
export LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12/:$LD_LIBRARY_PATH
|
|
98
|
+
python -m pip install .
|
|
99
|
+
if: ${{ matrix.os == 'ubuntu-22.04' }}
|
|
92
100
|
- name: Build locally
|
|
93
101
|
run: python -m pip install .
|
|
102
|
+
if: ${{ matrix.os != 'ubuntu-22.04' }}
|
|
94
103
|
- name: Test with PyTest
|
|
95
104
|
run: pytest python/scripts/
|
|
96
105
|
|
|
@@ -99,8 +108,9 @@ jobs:
|
|
|
99
108
|
name: Test Python 3.7
|
|
100
109
|
runs-on: ${{ matrix.os }}
|
|
101
110
|
strategy:
|
|
111
|
+
fail-fast: false
|
|
102
112
|
matrix:
|
|
103
|
-
os: [ubuntu-
|
|
113
|
+
os: [ubuntu-22.04]
|
|
104
114
|
python-version: ["3.7"]
|
|
105
115
|
|
|
106
116
|
steps:
|
|
@@ -118,7 +128,12 @@ jobs:
|
|
|
118
128
|
pip install pytest numpy
|
|
119
129
|
|
|
120
130
|
- name: Build locally
|
|
121
|
-
run:
|
|
131
|
+
run: |
|
|
132
|
+
export CC=/usr/bin/gcc-12
|
|
133
|
+
export CXX=/usr/bin/g++-12
|
|
134
|
+
export LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12/:$LD_LIBRARY_PATH
|
|
135
|
+
python -m pip install .
|
|
136
|
+
if: ${{ matrix.os == 'ubuntu-22.04' }}
|
|
122
137
|
|
|
123
138
|
- name: Test with PyTest
|
|
124
139
|
run: pytest python/scripts/
|
package/CMakeLists.txt
CHANGED
|
@@ -49,11 +49,26 @@ else()
|
|
|
49
49
|
target_compile_features(${USEARCH_TARGET_NAME} INTERFACE cxx_std_11)
|
|
50
50
|
endif()
|
|
51
51
|
|
|
52
|
+
# Core compilation settings affecting "index.hpp"
|
|
52
53
|
target_compile_definitions(
|
|
53
54
|
${USEARCH_TARGET_NAME}
|
|
54
55
|
INTERFACE
|
|
55
56
|
$<$<NOT:$<BOOL:${JSON_GlobalUDLs}>>:USEARCH_USE_OPENMP=0>
|
|
56
57
|
)
|
|
58
|
+
|
|
59
|
+
# Supplementary compilation settings affecting "index_plugins.hpp"
|
|
60
|
+
target_compile_definitions(
|
|
61
|
+
${USEARCH_TARGET_NAME}
|
|
62
|
+
INTERFACE
|
|
63
|
+
$<$<NOT:$<BOOL:${JSON_GlobalUDLs}>>:USEARCH_USE_NATIVE_F16=0>
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
target_compile_definitions(
|
|
67
|
+
${USEARCH_TARGET_NAME}
|
|
68
|
+
INTERFACE
|
|
69
|
+
$<$<NOT:$<BOOL:${JSON_GlobalUDLs}>>:USEARCH_USE_SIMSIMD=0>
|
|
70
|
+
)
|
|
71
|
+
|
|
57
72
|
target_include_directories(
|
|
58
73
|
${USEARCH_TARGET_NAME}
|
|
59
74
|
${USEARCH_SYSTEM_INCLUDE} INTERFACE
|
package/Cargo.toml
CHANGED
package/README.md
CHANGED
|
@@ -39,8 +39,8 @@ Linux • MacOS • Windows • iOS • Docker • WebAssembly
|
|
|
39
39
|
<a href="https://pypi.org/project/usearch/"> <img alt="PyPI - Downloads" src="https://img.shields.io/pypi/dm/usearch?label=pypi%20downloads"> </a>
|
|
40
40
|
<a href="https://www.npmjs.com/package/usearch"> <img alt="npm" src="https://img.shields.io/npm/dy/usearch?label=npm%20dowloads"> </a>
|
|
41
41
|
<a href="https://crates.io/crates/usearch"> <img alt="Crates.io" src="https://img.shields.io/crates/d/usearch?label=crate%20downloads"> </a>
|
|
42
|
-
<a href="https://www.nuget.org/packages/Cloud.Unum.USearch"> <img alt="Nuget" src="https://img.shields.io/nuget/dt/Cloud.Unum.USearch?
|
|
43
|
-
<a href="https://hub.docker.com/r/unum/usearch"> <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/unum/usearch?
|
|
42
|
+
<a href="https://www.nuget.org/packages/Cloud.Unum.USearch"> <img alt="Nuget" src="https://img.shields.io/nuget/dt/Cloud.Unum.USearch?label=NuGet"> </a>
|
|
43
|
+
<a href="https://hub.docker.com/r/unum/usearch"> <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/unum/usearch?label=Docker"> </a>
|
|
44
44
|
<img alt="GitHub code size in bytes" src="https://img.shields.io/github/languages/code-size/unum-cloud/usearch">
|
|
45
45
|
</div>
|
|
46
46
|
|
package/conanfile.py
CHANGED
|
@@ -4,7 +4,7 @@ from conans import ConanFile
|
|
|
4
4
|
class USearchConan(ConanFile):
|
|
5
5
|
|
|
6
6
|
name = 'USearch'
|
|
7
|
-
version = '2.
|
|
7
|
+
version = '2.2.0'
|
|
8
8
|
license = 'Apache 2.0'
|
|
9
9
|
url = 'https://github.com/unum-cloud/usearch'
|
|
10
10
|
description = 'Smaller & Faster Single-File Vector Search Engine from Unum'
|
|
@@ -41,8 +41,10 @@
|
|
|
41
41
|
#endif
|
|
42
42
|
|
|
43
43
|
#if USEARCH_USE_SIMSIMD
|
|
44
|
+
#if __linux__
|
|
44
45
|
#include <simsimd/simsimd.h>
|
|
45
46
|
#endif
|
|
47
|
+
#endif
|
|
46
48
|
|
|
47
49
|
namespace unum {
|
|
48
50
|
namespace usearch {
|
|
@@ -127,7 +129,9 @@ enum class isa_kind_t {
|
|
|
127
129
|
neon_k,
|
|
128
130
|
sve_k,
|
|
129
131
|
avx2_k,
|
|
132
|
+
avx2f16_k,
|
|
130
133
|
avx512_k,
|
|
134
|
+
avx512f16_k,
|
|
131
135
|
};
|
|
132
136
|
|
|
133
137
|
enum class prefetching_kind_t {
|
|
@@ -193,6 +197,8 @@ inline char const* isa_name(isa_kind_t isa_kind) noexcept {
|
|
|
193
197
|
case isa_kind_t::sve_k: return "sve";
|
|
194
198
|
case isa_kind_t::avx2_k: return "avx2";
|
|
195
199
|
case isa_kind_t::avx512_k: return "avx512";
|
|
200
|
+
case isa_kind_t::avx2f16_k: return "avx2+f16";
|
|
201
|
+
case isa_kind_t::avx512f16_k: return "avx512+f16";
|
|
196
202
|
default: return "";
|
|
197
203
|
}
|
|
198
204
|
}
|
|
@@ -215,6 +221,13 @@ inline bool hardware_supports(isa_kind_t isa_kind) noexcept {
|
|
|
215
221
|
#if defined(USEARCH_DEFINED_X86) && defined(USEARCH_DEFINED_GCC)
|
|
216
222
|
__builtin_cpu_init();
|
|
217
223
|
switch (isa_kind) {
|
|
224
|
+
// Half-precision is only supported in newer versions of GCC
|
|
225
|
+
#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 0)
|
|
226
|
+
case isa_kind_t::avx2f16_k: return __builtin_cpu_supports("avx2") && __builtin_cpu_supports("f16c");
|
|
227
|
+
#endif
|
|
228
|
+
#if __GNUC__ > 12 || (__GNUC__ == 12 && __GNUC_MINOR__ >= 0)
|
|
229
|
+
case isa_kind_t::avx512f16_k: return __builtin_cpu_supports("avx512f") && __builtin_cpu_supports("avx512fp16");
|
|
230
|
+
#endif
|
|
218
231
|
case isa_kind_t::avx2_k: return __builtin_cpu_supports("avx2");
|
|
219
232
|
case isa_kind_t::avx512_k: return __builtin_cpu_supports("avx512f");
|
|
220
233
|
default: return false;
|
|
@@ -600,9 +613,7 @@ class aligned_allocator_gt {
|
|
|
600
613
|
using size_type = std::size_t;
|
|
601
614
|
using pointer = element_at*;
|
|
602
615
|
using const_pointer = element_at const*;
|
|
603
|
-
template <typename other_element_at> struct rebind {
|
|
604
|
-
using other = aligned_allocator_gt<other_element_at>;
|
|
605
|
-
};
|
|
616
|
+
template <typename other_element_at> struct rebind { using other = aligned_allocator_gt<other_element_at>; };
|
|
606
617
|
|
|
607
618
|
constexpr std::size_t alignment() const { return alignment_ak; }
|
|
608
619
|
|
|
@@ -1384,25 +1395,58 @@ class metric_punned_t {
|
|
|
1384
1395
|
// clang-format off
|
|
1385
1396
|
static metric_punned_t ip_metric_f32_(std::size_t bytes_per_vector) {
|
|
1386
1397
|
#if USEARCH_USE_SIMSIMD
|
|
1387
|
-
if
|
|
1398
|
+
#if SIMSIMD_TARGET_ARM_NEON
|
|
1388
1399
|
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 16 == 0) return {pun_stl_<f32_t>([=](f32_t const* a, f32_t const* b) { return simsimd_dot_f32x4_neon(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::neon_k};
|
|
1400
|
+
#endif
|
|
1401
|
+
#if SIMSIMD_TARGET_ARM_SVE
|
|
1402
|
+
if (hardware_supports(isa_kind_t::sve_k)) return {pun_stl_<f32_t>([=](f32_t const* a, f32_t const* b) { return simsimd_dot_f32_sve(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::sve_k};
|
|
1403
|
+
#endif
|
|
1404
|
+
#if SIMSIMD_TARGET_X86_AVX2
|
|
1389
1405
|
if (hardware_supports(isa_kind_t::avx2_k) && bytes_per_vector % 16 == 0) return {pun_stl_<f32_t>([=](f32_t const* a, f32_t const* b) { return simsimd_dot_f32x4_avx2(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::avx2_k};
|
|
1390
1406
|
#endif
|
|
1407
|
+
#endif
|
|
1391
1408
|
return {to_stl_<metric_ip_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::ip_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1392
1409
|
}
|
|
1393
1410
|
|
|
1394
1411
|
static metric_punned_t cos_metric_f16_(std::size_t bytes_per_vector) {
|
|
1395
1412
|
#if USEARCH_USE_SIMSIMD
|
|
1396
|
-
if
|
|
1413
|
+
#if SIMSIMD_TARGET_X86_AVX512
|
|
1414
|
+
if (hardware_supports(isa_kind_t::avx512f16_k) && bytes_per_vector % 32 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_cos_f16x16_avx512(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::avx512f16_k};
|
|
1415
|
+
#endif
|
|
1416
|
+
#if SIMSIMD_TARGET_X86_AVX2
|
|
1417
|
+
if (hardware_supports(isa_kind_t::avx2f16_k) && bytes_per_vector % 16 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_cos_f16x8_avx2(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::avx2f16_k};
|
|
1418
|
+
#endif
|
|
1419
|
+
#if SIMSIMD_TARGET_ARM_NEON
|
|
1397
1420
|
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 8 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_cos_f16x4_neon(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::neon_k};
|
|
1398
1421
|
#endif
|
|
1422
|
+
#if SIMSIMD_TARGET_ARM_SVE
|
|
1423
|
+
if (hardware_supports(isa_kind_t::sve_k)) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_cos_f16_sve(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::sve_k};
|
|
1424
|
+
#endif
|
|
1425
|
+
#endif
|
|
1399
1426
|
return {to_stl_<metric_cos_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1400
1427
|
}
|
|
1401
1428
|
|
|
1429
|
+
static metric_punned_t l2sq_metric_f16_(std::size_t bytes_per_vector) {
|
|
1430
|
+
#if USEARCH_USE_SIMSIMD
|
|
1431
|
+
#if SIMSIMD_TARGET_X86_AVX512
|
|
1432
|
+
if (hardware_supports(isa_kind_t::avx512f16_k) && bytes_per_vector % 32 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_l2sq_f16x16_avx512(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::avx512f16_k};
|
|
1433
|
+
#endif
|
|
1434
|
+
#if SIMSIMD_TARGET_X86_AVX2
|
|
1435
|
+
if (hardware_supports(isa_kind_t::avx2f16_k) && bytes_per_vector % 16 == 0) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_l2sq_f16x8_avx2(a, b, bytes_per_vector / 2); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::f16_k, isa_kind_t::avx2f16_k};
|
|
1436
|
+
#endif
|
|
1437
|
+
#if SIMSIMD_TARGET_ARM_SVE
|
|
1438
|
+
if (hardware_supports(isa_kind_t::sve_k)) return {pun_stl_<simsimd_f16_t>([=](simsimd_f16_t const* a, simsimd_f16_t const* b) { return simsimd_l2sq_f16_sve(a, b, bytes_per_vector / 4); }), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f16_k, isa_kind_t::sve_k};
|
|
1439
|
+
#endif
|
|
1440
|
+
#endif
|
|
1441
|
+
return {to_stl_<metric_l2sq_gt<f16_t, f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f16_k, isa_kind_t::auto_k};
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1402
1444
|
static metric_punned_t cos_metric_i8_(std::size_t bytes_per_vector) {
|
|
1403
1445
|
#if USEARCH_USE_SIMSIMD
|
|
1446
|
+
#if SIMSIMD_TARGET_ARM_NEON
|
|
1404
1447
|
if (hardware_supports(isa_kind_t::neon_k) && bytes_per_vector % 16 == 0) return {pun_stl_<int8_t>([=](int8_t const* a, int8_t const* b) { return simsimd_cos_i8x16_neon(a, b, bytes_per_vector); }), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::neon_k};
|
|
1405
1448
|
#endif
|
|
1449
|
+
#endif
|
|
1406
1450
|
return {to_stl_<cos_i8_t>(bytes_per_vector), bytes_per_vector, metric_kind_t::cos_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1407
1451
|
}
|
|
1408
1452
|
|
|
@@ -1419,7 +1463,7 @@ class metric_punned_t {
|
|
|
1419
1463
|
static metric_punned_t l2sq_metric_(std::size_t bytes_per_vector, scalar_kind_t scalar_kind) {
|
|
1420
1464
|
switch (scalar_kind) {
|
|
1421
1465
|
case scalar_kind_t::i8_k: return {to_stl_<l2sq_i8_t>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::i8_k, isa_kind_t::auto_k};
|
|
1422
|
-
case scalar_kind_t::f16_k: return
|
|
1466
|
+
case scalar_kind_t::f16_k: return l2sq_metric_f16_(bytes_per_vector);
|
|
1423
1467
|
case scalar_kind_t::f32_k: return {to_stl_<metric_l2sq_gt<f32_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f32_k, isa_kind_t::auto_k};
|
|
1424
1468
|
case scalar_kind_t::f64_k: return {to_stl_<metric_l2sq_gt<f64_t>>(bytes_per_vector), bytes_per_vector, metric_kind_t::l2sq_k, scalar_kind_t::f64_k, isa_kind_t::auto_k};
|
|
1425
1469
|
default: return {};
|
package/package.json
CHANGED
package/python/lib.cpp
CHANGED
|
@@ -957,6 +957,17 @@ PYBIND11_MODULE(compiled, m) {
|
|
|
957
957
|
py::arg("metric_pointer") = 0 //
|
|
958
958
|
);
|
|
959
959
|
|
|
960
|
+
m.def(
|
|
961
|
+
"hardware_acceleration",
|
|
962
|
+
[](scalar_kind_t scalar_kind, std::size_t dimensions, metric_kind_t metric_kind) -> bool {
|
|
963
|
+
return metric_t(dimensions, metric_kind, scalar_kind).isa_kind() != isa_kind_t::auto_k;
|
|
964
|
+
},
|
|
965
|
+
py::kw_only(), //
|
|
966
|
+
py::arg("dtype") = scalar_kind_t::f32_k, //
|
|
967
|
+
py::arg("ndim") = 0, //
|
|
968
|
+
py::arg("metric_kind") = metric_kind_t::cos_k //
|
|
969
|
+
);
|
|
970
|
+
|
|
960
971
|
auto i = py::class_<dense_index_py_t, std::shared_ptr<dense_index_py_t>>(m, "Index");
|
|
961
972
|
|
|
962
973
|
i.def(py::init(&make_index), //
|
package/python/usearch/index.py
CHANGED
|
@@ -29,6 +29,7 @@ from usearch.compiled import (
|
|
|
29
29
|
index_dense_metadata_from_path as _index_dense_metadata_from_path,
|
|
30
30
|
index_dense_metadata_from_buffer as _index_dense_metadata_from_buffer,
|
|
31
31
|
exact_search as _exact_search,
|
|
32
|
+
hardware_acceleration as _hardware_acceleration,
|
|
32
33
|
)
|
|
33
34
|
|
|
34
35
|
# Precompiled symbols that will be exposed
|
|
@@ -72,9 +73,17 @@ NoneType = type(None)
|
|
|
72
73
|
PathOrBuffer = Union[str, os.PathLike, bytes]
|
|
73
74
|
|
|
74
75
|
|
|
75
|
-
def _normalize_dtype(
|
|
76
|
+
def _normalize_dtype(
|
|
77
|
+
dtype,
|
|
78
|
+
ndim: int = 0,
|
|
79
|
+
metric: MetricKind = MetricKind.Cos,
|
|
80
|
+
) -> ScalarKind:
|
|
76
81
|
if dtype is None or dtype == "":
|
|
77
|
-
|
|
82
|
+
if metric in MetricKindBitwise:
|
|
83
|
+
return ScalarKind.B1
|
|
84
|
+
if _hardware_acceleration(dtype=ScalarKind.F16, ndim=ndim, metric_kind=metric):
|
|
85
|
+
return ScalarKind.F16
|
|
86
|
+
return ScalarKind.F32
|
|
78
87
|
|
|
79
88
|
if isinstance(dtype, ScalarKind):
|
|
80
89
|
return dtype
|
|
@@ -491,8 +500,12 @@ class Index:
|
|
|
491
500
|
:type dtype: Optional[DTypeLike], defaults to None
|
|
492
501
|
For continuous metrics can be: f16, f32, f64, or i8.
|
|
493
502
|
For bitwise metrics it's implementation-defined, and can't change.
|
|
503
|
+
If nothing is provided, the optimal data type is selected based on the metric
|
|
504
|
+
kind and hardware support.
|
|
494
505
|
Example: you can use the `f16` index with `f32` vectors in Euclidean space,
|
|
495
|
-
which will be automatically downcasted.
|
|
506
|
+
which will be automatically downcasted. Moreover, if `dtype=None` is passed,
|
|
507
|
+
and hardware supports `f16` SIMD-instructions, this choice will be done for you.
|
|
508
|
+
You can later double-check the used representation with `index.dtype`.
|
|
496
509
|
|
|
497
510
|
:param connectivity: Connections per node in HNSW
|
|
498
511
|
:type connectivity: Optional[int], defaults to None
|
|
@@ -548,7 +561,7 @@ class Index:
|
|
|
548
561
|
)
|
|
549
562
|
|
|
550
563
|
# Validate, that the right scalar type is defined
|
|
551
|
-
dtype = _normalize_dtype(dtype, self._metric_kind)
|
|
564
|
+
dtype = _normalize_dtype(dtype, ndim, self._metric_kind)
|
|
552
565
|
self._compiled = _CompiledIndex(
|
|
553
566
|
ndim=ndim,
|
|
554
567
|
dtype=dtype,
|
package/setup.py
CHANGED
|
@@ -7,10 +7,7 @@ from pybind11.setup_helpers import Pybind11Extension
|
|
|
7
7
|
|
|
8
8
|
compile_args = []
|
|
9
9
|
link_args = []
|
|
10
|
-
macros_args = [
|
|
11
|
-
("USEARCH_USE_NATIVE_F16", "0"),
|
|
12
|
-
("USEARCH_USE_SIMSIMD", "0"),
|
|
13
|
-
]
|
|
10
|
+
macros_args = []
|
|
14
11
|
|
|
15
12
|
if sys.platform == "linux":
|
|
16
13
|
compile_args.append("-std=c++17")
|
|
@@ -18,6 +15,8 @@ if sys.platform == "linux":
|
|
|
18
15
|
compile_args.append("-g") # Simplify debugging
|
|
19
16
|
compile_args.append("-Wno-unknown-pragmas")
|
|
20
17
|
|
|
18
|
+
macros_args.append(("USEARCH_USE_NATIVE_F16", "0"))
|
|
19
|
+
macros_args.append(("USEARCH_USE_SIMSIMD", "1"))
|
|
21
20
|
macros_args.append(("USEARCH_USE_OPENMP", "1"))
|
|
22
21
|
compile_args.append("-fopenmp")
|
|
23
22
|
link_args.append("-lgomp")
|
|
@@ -21,8 +21,9 @@ jobs:
|
|
|
21
21
|
name: Test Python
|
|
22
22
|
runs-on: ${{ matrix.os }}
|
|
23
23
|
strategy:
|
|
24
|
+
fail-fast: false
|
|
24
25
|
matrix:
|
|
25
|
-
os: [ubuntu-
|
|
26
|
+
os: [ubuntu-22.04, macOS-11] # windows-2022
|
|
26
27
|
python-version: ["3.11"]
|
|
27
28
|
|
|
28
29
|
steps:
|
|
@@ -32,22 +33,52 @@ jobs:
|
|
|
32
33
|
uses: actions/setup-python@v4
|
|
33
34
|
with:
|
|
34
35
|
python-version: ${{ matrix.python-version }}
|
|
36
|
+
|
|
37
|
+
- name: Set up Clang
|
|
38
|
+
run: brew install gcc@12
|
|
39
|
+
if: ${{ matrix.os == 'macOS-11' }}
|
|
40
|
+
|
|
41
|
+
- name: Set up MinGW
|
|
42
|
+
uses: egor-tensin/setup-mingw@v2
|
|
43
|
+
with:
|
|
44
|
+
platform: x64
|
|
45
|
+
if: ${{ matrix.os == 'windows-2022' }}
|
|
46
|
+
|
|
35
47
|
- name: Install dependencies
|
|
36
48
|
run: |
|
|
37
49
|
python -m pip install --no-cache-dir --upgrade pip
|
|
38
50
|
pip install --no-cache-dir pytest
|
|
39
|
-
|
|
40
|
-
|
|
51
|
+
|
|
52
|
+
- name: Build locally on Ubuntu
|
|
53
|
+
run: |
|
|
54
|
+
export CC=/usr/bin/gcc-12
|
|
55
|
+
export CXX=/usr/bin/g++-12
|
|
56
|
+
export LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12/:$LD_LIBRARY_PATH
|
|
57
|
+
python -m pip install .
|
|
58
|
+
if: ${{ matrix.os == 'ubuntu-22.04' }}
|
|
59
|
+
|
|
60
|
+
- name: Build locally on MacOS
|
|
61
|
+
run: |
|
|
62
|
+
export CC=/usr/local/bin/gcc-12
|
|
63
|
+
export CXX=/usr/local/bin/g++-12
|
|
64
|
+
python -m pip install .
|
|
65
|
+
if: ${{ matrix.os == 'macOS-11' }}
|
|
66
|
+
|
|
67
|
+
- name: Build locally on Windows
|
|
68
|
+
run: |
|
|
69
|
+
python -m pip install .
|
|
70
|
+
if: ${{ matrix.os == 'windows-2022' }}
|
|
71
|
+
|
|
41
72
|
- name: Test with PyTest
|
|
42
73
|
run: pytest python/test.py
|
|
43
|
-
|
|
74
|
+
|
|
44
75
|
|
|
45
76
|
test_python_37:
|
|
46
77
|
name: Test Python 3.7
|
|
47
78
|
runs-on: ${{ matrix.os }}
|
|
48
79
|
strategy:
|
|
49
80
|
matrix:
|
|
50
|
-
os: [ubuntu-
|
|
81
|
+
os: [ubuntu-22.04]
|
|
51
82
|
python-version: ["3.7"]
|
|
52
83
|
|
|
53
84
|
steps:
|
|
@@ -65,9 +96,11 @@ jobs:
|
|
|
65
96
|
pip install --no-cache-dir pytest
|
|
66
97
|
|
|
67
98
|
- name: Build locally
|
|
68
|
-
run:
|
|
69
|
-
|
|
99
|
+
run: |
|
|
100
|
+
export CC=/usr/bin/gcc-12
|
|
101
|
+
export CXX=/usr/bin/g++-12
|
|
102
|
+
export LD_LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12/:$LD_LIBRARY_PATH
|
|
103
|
+
python -m pip install .
|
|
104
|
+
|
|
70
105
|
- name: Test with PyTest
|
|
71
106
|
run: pytest python/test.py
|
|
72
|
-
|
|
73
|
-
|
|
@@ -29,7 +29,7 @@ jobs:
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
rebase:
|
|
32
|
-
name: Rebase
|
|
32
|
+
name: Rebase Dev. Branch
|
|
33
33
|
needs: versioning
|
|
34
34
|
runs-on: ubuntu-latest
|
|
35
35
|
steps:
|
|
@@ -57,8 +57,7 @@ jobs:
|
|
|
57
57
|
needs: versioning
|
|
58
58
|
strategy:
|
|
59
59
|
matrix:
|
|
60
|
-
os: [ubuntu-20.04, macOS-11
|
|
61
|
-
|
|
60
|
+
os: [ubuntu-20.04, macOS-11] # windows-2022
|
|
62
61
|
steps:
|
|
63
62
|
- uses: actions/checkout@v3
|
|
64
63
|
with:
|
|
@@ -66,20 +65,25 @@ jobs:
|
|
|
66
65
|
- uses: actions/setup-python@v3
|
|
67
66
|
|
|
68
67
|
- name: Setup Docker
|
|
69
|
-
if: matrix.os
|
|
70
|
-
uses:
|
|
71
|
-
|
|
72
|
-
version: 23.0.1
|
|
73
|
-
|
|
68
|
+
if: matrix.os == 'ubuntu-20.04'
|
|
69
|
+
uses: docker-practice/actions-setup-docker@master
|
|
70
|
+
|
|
74
71
|
- name: Setup QEMU
|
|
75
|
-
if: matrix.os
|
|
72
|
+
if: matrix.os == 'ubuntu-20.04'
|
|
76
73
|
uses: docker/setup-qemu-action@v2.1.0
|
|
77
74
|
|
|
78
75
|
- name: Install CIBuildWheel
|
|
79
76
|
run: python -m pip install --no-cache-dir cibuildwheel
|
|
80
77
|
|
|
81
|
-
- name: Build wheels
|
|
78
|
+
- name: Build wheels on Ubuntu
|
|
79
|
+
run: python -m cibuildwheel
|
|
80
|
+
if: matrix.os == 'ubuntu-20.04'
|
|
81
|
+
|
|
82
|
+
- name: Build wheels on MacOS
|
|
82
83
|
run: python -m cibuildwheel
|
|
84
|
+
env:
|
|
85
|
+
CIBW_ENVIRONMENT: CC=/usr/local/bin/gcc-12 CXX=/usr/local/bin/g++-12
|
|
86
|
+
if: matrix.os == 'macOS-11'
|
|
83
87
|
|
|
84
88
|
- uses: actions/upload-artifact@v3
|
|
85
89
|
with:
|
|
@@ -92,7 +96,7 @@ jobs:
|
|
|
92
96
|
runs-on: ubuntu-20.04
|
|
93
97
|
environment:
|
|
94
98
|
name: pypi
|
|
95
|
-
url: https://pypi.org/p/
|
|
99
|
+
url: https://pypi.org/p/simsimd
|
|
96
100
|
permissions:
|
|
97
101
|
id-token: write
|
|
98
102
|
|
|
@@ -85,13 +85,17 @@
|
|
|
85
85
|
"variant": "cpp",
|
|
86
86
|
"__nullptr": "cpp",
|
|
87
87
|
"unordered_set": "cpp",
|
|
88
|
-
"span": "cpp"
|
|
88
|
+
"span": "cpp",
|
|
89
|
+
"arm_neon.h": "c"
|
|
89
90
|
},
|
|
90
91
|
"cSpell.words": [
|
|
91
92
|
"dtype",
|
|
92
93
|
"dtypes",
|
|
93
94
|
"kann",
|
|
95
|
+
"maccs",
|
|
96
|
+
"simd",
|
|
94
97
|
"simsimd",
|
|
98
|
+
"tanimoto",
|
|
95
99
|
"unsw"
|
|
96
100
|
]
|
|
97
101
|
}
|
package/simsimd/CMakeLists.txt
CHANGED
|
@@ -34,14 +34,17 @@ endif()
|
|
|
34
34
|
|
|
35
35
|
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -fsanitize=address")
|
|
36
36
|
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
37
|
+
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -fsanitize=address")
|
|
38
|
+
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
|
|
37
39
|
|
|
38
40
|
# List of all possible compiler IDs:
|
|
39
41
|
# https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html
|
|
40
42
|
if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
|
|
41
43
|
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
|
42
44
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
|
|
43
|
-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic")
|
|
44
|
-
set(
|
|
45
|
+
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic -fmax-errors=1")
|
|
46
|
+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native")
|
|
47
|
+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pedantic -fmax-errors=1")
|
|
45
48
|
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "NVIDIA" OR CMAKE_CXX_COMPILER_ID STREQUAL "NVHPC")
|
|
46
49
|
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr --extended-lambda")
|
|
47
50
|
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode arch=compute_86,code=sm_86")
|