usearch 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/prerelease.yml +2 -21
- package/.github/workflows/release.yml +4 -36
- package/.github/workflows/update_version.sh +1 -1
- package/.gitmodules +1 -4
- package/Cargo.toml +1 -1
- package/Package.swift +2 -1
- package/README.md +86 -42
- package/binding.gyp +16 -12
- package/build.gradle +1 -1
- package/build.rs +0 -1
- package/c/CMakeLists.txt +11 -7
- package/c/lib.cpp +36 -24
- package/c/usearch.h +42 -12
- package/cluster.ipynb +167 -0
- package/conanfile.py +1 -1
- package/cpp/CMakeLists.txt +8 -3
- package/cpp/bench.cpp +2 -2
- package/cpp/test.cpp +42 -25
- package/docs/benchmarks.md +8 -0
- package/docs/compilation.md +7 -7
- package/docs/conf.py +27 -24
- package/docs/javascript/reference.rst +3 -0
- package/golang/README.md +31 -7
- package/golang/lib.go +1 -1
- package/include/usearch/index.hpp +348 -69
- package/include/usearch/index_dense.hpp +587 -179
- package/include/usearch/index_plugins.hpp +194 -16
- package/java/cloud/unum/usearch/Index.java +13 -0
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.cpp +40 -9
- package/java/cloud/unum/usearch/cloud_unum_usearch_Index.h +16 -0
- package/javascript/docs.js +111 -0
- package/javascript/lib.cpp +29 -9
- package/javascript/usearch.d.ts +9 -1
- package/objc/USearchObjective.mm +11 -11
- package/package.json +2 -2
- package/pyproject.toml +2 -2
- package/python/lib.cpp +488 -158
- package/python/scripts/index_faiss.py +3 -3
- package/python/scripts/join.py +6 -6
- package/python/scripts/test_index.py +219 -0
- package/python/scripts/test_sparse.py +21 -21
- package/python/scripts/test_tooling.py +111 -0
- package/python/usearch/eval.py +9 -5
- package/python/usearch/index.py +491 -216
- package/rust/lib.cpp +6 -24
- package/rust/lib.hpp +1 -3
- package/rust/lib.rs +89 -0
- package/setup.py +5 -6
- package/wasm/CMakeLists.txt +0 -1
- package/wasmer.toml +1 -1
- package/python/scripts/test.py +0 -364
- package/robin-map/.codecov.yml +0 -5
- package/robin-map/.github/workflows/ci.yml +0 -136
- package/robin-map/CMakeLists.txt +0 -86
- package/robin-map/LICENSE +0 -21
- package/robin-map/README.md +0 -483
- package/robin-map/cmake/tsl-robin-mapConfig.cmake.in +0 -9
- package/robin-map/doxygen.conf +0 -2483
- package/robin-map/include/tsl/robin_growth_policy.h +0 -406
- package/robin-map/include/tsl/robin_hash.h +0 -1639
- package/robin-map/include/tsl/robin_map.h +0 -807
- package/robin-map/include/tsl/robin_set.h +0 -660
- package/robin-map/tests/CMakeLists.txt +0 -26
- package/robin-map/tests/custom_allocator_tests.cpp +0 -138
- package/robin-map/tests/main.cpp +0 -26
- package/robin-map/tests/policy_tests.cpp +0 -97
- package/robin-map/tests/robin_map_tests.cpp +0 -1451
- package/robin-map/tests/robin_set_tests.cpp +0 -164
- package/robin-map/tests/utils.h +0 -443
- package/robin-map/tsl-robin-map.natvis +0 -78
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
#include <thread> // `std::thread`
|
|
7
7
|
#include <vector> // `std::vector`
|
|
8
8
|
|
|
9
|
+
#include <atomic> // `std::atomic`
|
|
10
|
+
#include <thread> // `std::thread`
|
|
11
|
+
|
|
9
12
|
#include <usearch/index.hpp> // `expected_gt` and macros
|
|
10
13
|
|
|
11
14
|
#if USEARCH_USE_OPENMP
|
|
@@ -30,7 +33,6 @@
|
|
|
30
33
|
#include <fp16/fp16.h>
|
|
31
34
|
#endif
|
|
32
35
|
#else
|
|
33
|
-
#define USEARCH_USE_NATIVE_F16 0
|
|
34
36
|
#include <fp16/fp16.h>
|
|
35
37
|
#endif
|
|
36
38
|
|
|
@@ -196,6 +198,9 @@ inline char const* isa_name(isa_kind_t isa_kind) noexcept {
|
|
|
196
198
|
}
|
|
197
199
|
|
|
198
200
|
inline bool hardware_supports(isa_kind_t isa_kind) noexcept {
|
|
201
|
+
|
|
202
|
+
// On Linux Arm machines the `getauxval` can be queried to check
|
|
203
|
+
// if SVE extensions are available. Arm Neon has no separate capability check.
|
|
199
204
|
#if defined(USEARCH_DEFINED_ARM) && defined(USEARCH_DEFINED_LINUX)
|
|
200
205
|
unsigned long capabilities = getauxval(AT_HWCAP);
|
|
201
206
|
switch (isa_kind) {
|
|
@@ -205,6 +210,8 @@ inline bool hardware_supports(isa_kind_t isa_kind) noexcept {
|
|
|
205
210
|
}
|
|
206
211
|
#endif
|
|
207
212
|
|
|
213
|
+
// When compiling with GCC, one may use the "built-ins", including ones
|
|
214
|
+
// designed for CPU capability detection.
|
|
208
215
|
#if defined(USEARCH_DEFINED_X86) && defined(USEARCH_DEFINED_GCC)
|
|
209
216
|
__builtin_cpu_init();
|
|
210
217
|
switch (isa_kind) {
|
|
@@ -214,6 +221,19 @@ inline bool hardware_supports(isa_kind_t isa_kind) noexcept {
|
|
|
214
221
|
}
|
|
215
222
|
#endif
|
|
216
223
|
|
|
224
|
+
// On Apple we can expect Arm devices to support Neon extesions,
|
|
225
|
+
// and the x86 machines to support AVX2 extensions.
|
|
226
|
+
#if defined(USEARCH_DEFINED_APPLE)
|
|
227
|
+
switch (isa_kind) {
|
|
228
|
+
#if defined(USEARCH_DEFINED_ARM)
|
|
229
|
+
case isa_kind_t::neon_k: return true;
|
|
230
|
+
#else
|
|
231
|
+
case isa_kind_t::avx2_k: return true;
|
|
232
|
+
#endif
|
|
233
|
+
default: return false;
|
|
234
|
+
}
|
|
235
|
+
#endif
|
|
236
|
+
|
|
217
237
|
(void)isa_kind;
|
|
218
238
|
return false;
|
|
219
239
|
}
|
|
@@ -429,6 +449,20 @@ inline f16_bits_t::f16_bits_t(i8_bits_t v) noexcept : f16_bits_t(float(v)) {}
|
|
|
429
449
|
class executor_stl_t {
|
|
430
450
|
std::size_t threads_count_{};
|
|
431
451
|
|
|
452
|
+
struct jthread_t {
|
|
453
|
+
std::thread native_;
|
|
454
|
+
|
|
455
|
+
jthread_t() = default;
|
|
456
|
+
jthread_t(jthread_t&&) = default;
|
|
457
|
+
jthread_t(jthread_t const&) = delete;
|
|
458
|
+
template <typename callable_at> jthread_t(callable_at&& func) : native_([=]() { func(); }) {}
|
|
459
|
+
|
|
460
|
+
~jthread_t() {
|
|
461
|
+
if (native_.joinable())
|
|
462
|
+
native_.join();
|
|
463
|
+
}
|
|
464
|
+
};
|
|
465
|
+
|
|
432
466
|
public:
|
|
433
467
|
/**
|
|
434
468
|
* @param threads_count The number of threads to be used for parallel execution.
|
|
@@ -442,18 +476,19 @@ class executor_stl_t {
|
|
|
442
476
|
std::size_t size() const noexcept { return threads_count_; }
|
|
443
477
|
|
|
444
478
|
/**
|
|
445
|
-
* @brief Executes
|
|
479
|
+
* @brief Executes a fixed number of tasks using the specified thread-aware function.
|
|
446
480
|
* @param tasks The total number of tasks to be executed.
|
|
447
481
|
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
448
482
|
* @throws If an exception occurs during execution of the thread-aware function.
|
|
449
483
|
*/
|
|
450
484
|
template <typename thread_aware_function_at>
|
|
451
|
-
void
|
|
452
|
-
std::vector<
|
|
485
|
+
void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
486
|
+
std::vector<jthread_t> threads_pool;
|
|
453
487
|
std::size_t tasks_per_thread = tasks;
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
488
|
+
std::size_t threads_count = (std::min)(threads_count_, tasks);
|
|
489
|
+
if (threads_count > 1) {
|
|
490
|
+
tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
|
|
491
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
|
|
457
492
|
threads_pool.emplace_back([=]() {
|
|
458
493
|
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
459
494
|
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread); ++task_idx)
|
|
@@ -463,8 +498,37 @@ class executor_stl_t {
|
|
|
463
498
|
}
|
|
464
499
|
for (std::size_t task_idx = 0; task_idx < (std::min)(tasks, tasks_per_thread); ++task_idx)
|
|
465
500
|
thread_aware_function(0, task_idx);
|
|
466
|
-
|
|
467
|
-
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* @brief Executes limited number of tasks using the specified thread-aware function.
|
|
505
|
+
* @param tasks The upper bound on the number of tasks.
|
|
506
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
507
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
508
|
+
*/
|
|
509
|
+
template <typename thread_aware_function_at>
|
|
510
|
+
void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
511
|
+
std::vector<jthread_t> threads_pool;
|
|
512
|
+
std::size_t tasks_per_thread = tasks;
|
|
513
|
+
std::size_t threads_count = (std::min)(threads_count_, tasks);
|
|
514
|
+
std::atomic_bool stop{false};
|
|
515
|
+
if (threads_count > 1) {
|
|
516
|
+
tasks_per_thread = (tasks / threads_count) + ((tasks % threads_count) != 0);
|
|
517
|
+
for (std::size_t thread_idx = 1; thread_idx < threads_count; ++thread_idx) {
|
|
518
|
+
threads_pool.emplace_back([=, &stop]() {
|
|
519
|
+
for (std::size_t task_idx = thread_idx * tasks_per_thread;
|
|
520
|
+
task_idx < (std::min)(tasks, thread_idx * tasks_per_thread + tasks_per_thread) &&
|
|
521
|
+
!stop.load(std::memory_order_relaxed);
|
|
522
|
+
++task_idx)
|
|
523
|
+
if (!thread_aware_function(thread_idx, task_idx))
|
|
524
|
+
stop.store(true, std::memory_order_relaxed);
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
for (std::size_t task_idx = 0;
|
|
529
|
+
task_idx < (std::min)(tasks, tasks_per_thread) && !stop.load(std::memory_order_relaxed); ++task_idx)
|
|
530
|
+
if (!thread_aware_function(0, task_idx))
|
|
531
|
+
stop.store(true, std::memory_order_relaxed);
|
|
468
532
|
}
|
|
469
533
|
|
|
470
534
|
/**
|
|
@@ -473,15 +537,13 @@ class executor_stl_t {
|
|
|
473
537
|
* @throws If an exception occurs during execution of the thread-aware function.
|
|
474
538
|
*/
|
|
475
539
|
template <typename thread_aware_function_at>
|
|
476
|
-
void
|
|
540
|
+
void parallel(thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
477
541
|
if (threads_count_ == 1)
|
|
478
542
|
return thread_aware_function(0);
|
|
479
|
-
std::vector<
|
|
543
|
+
std::vector<jthread_t> threads_pool;
|
|
480
544
|
for (std::size_t thread_idx = 1; thread_idx < threads_count_; ++thread_idx)
|
|
481
545
|
threads_pool.emplace_back([=]() { thread_aware_function(thread_idx); });
|
|
482
546
|
thread_aware_function(0);
|
|
483
|
-
for (std::thread& thread : threads_pool)
|
|
484
|
-
thread.join();
|
|
485
547
|
}
|
|
486
548
|
};
|
|
487
549
|
|
|
@@ -512,20 +574,48 @@ class executor_openmp_t {
|
|
|
512
574
|
* @throws If an exception occurs during execution of the thread-aware function.
|
|
513
575
|
*/
|
|
514
576
|
template <typename thread_aware_function_at>
|
|
515
|
-
void
|
|
516
|
-
#pragma omp parallel for schedule(dynamic)
|
|
577
|
+
void fixed(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
578
|
+
#pragma omp parallel for schedule(dynamic, 1)
|
|
517
579
|
for (std::size_t i = 0; i != tasks; ++i) {
|
|
518
580
|
thread_aware_function(omp_get_thread_num(), i);
|
|
519
581
|
}
|
|
520
582
|
}
|
|
521
583
|
|
|
584
|
+
/**
|
|
585
|
+
* @brief Executes tasks in bulk using the specified thread-aware function.
|
|
586
|
+
* @param tasks The total number of tasks to be executed.
|
|
587
|
+
* @param thread_aware_function The thread-aware function to be called for each thread index and task index.
|
|
588
|
+
* @throws If an exception occurs during execution of the thread-aware function.
|
|
589
|
+
*/
|
|
590
|
+
template <typename thread_aware_function_at>
|
|
591
|
+
void dynamic(std::size_t tasks, thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
592
|
+
// OpenMP cancellation points are not yet available on most platforms, and require
|
|
593
|
+
// the `OMP_CANCELLATION` environment variable to be set.
|
|
594
|
+
// http://jakascorner.com/blog/2016/08/omp-cancel.html
|
|
595
|
+
// if (omp_get_cancellation()) {
|
|
596
|
+
// #pragma omp parallel for schedule(dynamic, 1)
|
|
597
|
+
// for (std::size_t i = 0; i != tasks; ++i) {
|
|
598
|
+
// #pragma omp cancellation point for
|
|
599
|
+
// if (!thread_aware_function(omp_get_thread_num(), i)) {
|
|
600
|
+
// #pragma omp cancel for
|
|
601
|
+
// }
|
|
602
|
+
// }
|
|
603
|
+
// }
|
|
604
|
+
std::atomic_bool stop{false};
|
|
605
|
+
#pragma omp parallel for schedule(dynamic, 1) shared(stop)
|
|
606
|
+
for (std::size_t i = 0; i != tasks; ++i) {
|
|
607
|
+
if (!stop.load(std::memory_order_relaxed) && !thread_aware_function(omp_get_thread_num(), i))
|
|
608
|
+
stop.store(true, std::memory_order_relaxed);
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
522
612
|
/**
|
|
523
613
|
* @brief Saturates every available thread with the given workload, until they finish.
|
|
524
614
|
* @param thread_aware_function The thread-aware function to be called for each thread index.
|
|
525
615
|
* @throws If an exception occurs during execution of the thread-aware function.
|
|
526
616
|
*/
|
|
527
617
|
template <typename thread_aware_function_at>
|
|
528
|
-
void
|
|
618
|
+
void parallel(thread_aware_function_at&& thread_aware_function) noexcept(false) {
|
|
529
619
|
#pragma omp parallel
|
|
530
620
|
{ thread_aware_function(omp_get_thread_num()); }
|
|
531
621
|
}
|
|
@@ -735,6 +825,94 @@ template <std::size_t alignment_ak = 1> class memory_mapping_allocator_gt {
|
|
|
735
825
|
|
|
736
826
|
using memory_mapping_allocator_t = memory_mapping_allocator_gt<>;
|
|
737
827
|
|
|
828
|
+
/**
|
|
829
|
+
* @brief C++11 userspace implementation of an oversimplified `std::shared_mutex`,
|
|
830
|
+
* that assumes rare interleaving of shared and unique locks. It's not fair,
|
|
831
|
+
* but requires only a single 32-bit atomic integer to work.
|
|
832
|
+
*/
|
|
833
|
+
class unfair_shared_mutex_t {
|
|
834
|
+
/** Any positive integer describes the number of concurrent readers */
|
|
835
|
+
enum state_t : std::int32_t {
|
|
836
|
+
idle_k = 0,
|
|
837
|
+
writing_k = -1,
|
|
838
|
+
};
|
|
839
|
+
std::atomic<std::int32_t> state_{idle_k};
|
|
840
|
+
|
|
841
|
+
public:
|
|
842
|
+
inline void lock() noexcept {
|
|
843
|
+
std::int32_t raw;
|
|
844
|
+
relock:
|
|
845
|
+
raw = idle_k;
|
|
846
|
+
if (!state_.compare_exchange_weak(raw, writing_k, std::memory_order_acquire, std::memory_order_relaxed)) {
|
|
847
|
+
std::this_thread::yield();
|
|
848
|
+
goto relock;
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
inline void unlock() noexcept { state_.store(idle_k, std::memory_order_release); }
|
|
853
|
+
|
|
854
|
+
inline void lock_shared() noexcept {
|
|
855
|
+
std::int32_t raw;
|
|
856
|
+
relock_shared:
|
|
857
|
+
raw = state_.load(std::memory_order_acquire);
|
|
858
|
+
// Spin while it's uniquely locked
|
|
859
|
+
if (raw == writing_k) {
|
|
860
|
+
std::this_thread::yield();
|
|
861
|
+
goto relock_shared;
|
|
862
|
+
}
|
|
863
|
+
// Try incrementing the counter
|
|
864
|
+
if (!state_.compare_exchange_weak(raw, raw + 1, std::memory_order_acquire, std::memory_order_relaxed)) {
|
|
865
|
+
std::this_thread::yield();
|
|
866
|
+
goto relock_shared;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
inline void unlock_shared() noexcept { state_.fetch_sub(1, std::memory_order_release); }
|
|
871
|
+
|
|
872
|
+
/**
|
|
873
|
+
* @brief Try upgrades the current `lock_shared()` to a unique `lock()` state.
|
|
874
|
+
*/
|
|
875
|
+
inline bool try_escalate() noexcept {
|
|
876
|
+
std::int32_t one_read = 1;
|
|
877
|
+
return state_.compare_exchange_weak(one_read, writing_k, std::memory_order_acquire, std::memory_order_relaxed);
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
/**
|
|
881
|
+
* @brief Escalates current lock potentially loosing control in the middle.
|
|
882
|
+
* It's a shortcut for `try_escalate`-`unlock_shared`-`lock` trio.
|
|
883
|
+
*/
|
|
884
|
+
inline void unsafe_escalate() noexcept {
|
|
885
|
+
if (!try_escalate()) {
|
|
886
|
+
unlock_shared();
|
|
887
|
+
lock();
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
/**
|
|
892
|
+
* @brief Upgrades the current `lock_shared()` to a unique `lock()` state.
|
|
893
|
+
*/
|
|
894
|
+
inline void escalate() noexcept {
|
|
895
|
+
while (!try_escalate())
|
|
896
|
+
std::this_thread::yield();
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/**
|
|
900
|
+
* @brief De-escalation of a previously escalated state.
|
|
901
|
+
*/
|
|
902
|
+
inline void de_escalate() noexcept {
|
|
903
|
+
std::int32_t one_read = 1;
|
|
904
|
+
state_.store(one_read, std::memory_order_release);
|
|
905
|
+
}
|
|
906
|
+
};
|
|
907
|
+
|
|
908
|
+
template <typename mutex_at = unfair_shared_mutex_t> class shared_lock_gt {
|
|
909
|
+
mutex_at& mutex_;
|
|
910
|
+
|
|
911
|
+
public:
|
|
912
|
+
inline explicit shared_lock_gt(mutex_at& m) noexcept : mutex_(m) { mutex_.lock_shared(); }
|
|
913
|
+
inline ~shared_lock_gt() noexcept { mutex_.unlock_shared(); }
|
|
914
|
+
};
|
|
915
|
+
|
|
738
916
|
/**
|
|
739
917
|
* @brief Utility class used to cast arrays of one scalar type to another,
|
|
740
918
|
* avoiding unnecessary conversions.
|
|
@@ -69,6 +69,14 @@ public class Index {
|
|
|
69
69
|
c_view(c_ptr, path);
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
+
public boolean remove(int key) {
|
|
73
|
+
return c_remove(c_ptr, key);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
public boolean rename(int from, int to) {
|
|
77
|
+
return c_rename(c_ptr, from, to);
|
|
78
|
+
}
|
|
79
|
+
|
|
72
80
|
public static class Config {
|
|
73
81
|
private String _metric = "ip";
|
|
74
82
|
private String _quantization = "f32";
|
|
@@ -135,6 +143,7 @@ public class Index {
|
|
|
135
143
|
public static void main(String[] args) {
|
|
136
144
|
Index index = new Index.Config().metric("cos").dimensions(100).build();
|
|
137
145
|
index.size();
|
|
146
|
+
System.out.println("Java tests passed!");
|
|
138
147
|
}
|
|
139
148
|
|
|
140
149
|
private static native long c_create(//
|
|
@@ -163,4 +172,8 @@ public class Index {
|
|
|
163
172
|
private static native void c_load(long ptr, String path);
|
|
164
173
|
|
|
165
174
|
private static native void c_view(long ptr, String path);
|
|
175
|
+
|
|
176
|
+
private static native boolean c_remove(long ptr, int key);
|
|
177
|
+
|
|
178
|
+
private static native boolean c_rename(long ptr, int from, int to);
|
|
166
179
|
}
|
|
@@ -58,30 +58,33 @@ JNIEXPORT jlong JNICALL Java_cloud_unum_usearch_Index_c_1create( //
|
|
|
58
58
|
|
|
59
59
|
JNIEXPORT void JNICALL Java_cloud_unum_usearch_Index_c_1save(JNIEnv* env, jclass, jlong c_ptr, jstring path) {
|
|
60
60
|
char const* path_cstr = (*env).GetStringUTFChars(path, 0);
|
|
61
|
-
|
|
61
|
+
serialization_result_t result = reinterpret_cast<index_dense_t*>(c_ptr)->save(path_cstr);
|
|
62
|
+
if (!result) {
|
|
62
63
|
jclass jc = (*env).FindClass("java/lang/Error");
|
|
63
64
|
if (jc)
|
|
64
|
-
(*env).ThrowNew(jc,
|
|
65
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
65
66
|
}
|
|
66
67
|
(*env).ReleaseStringUTFChars(path, path_cstr);
|
|
67
68
|
}
|
|
68
69
|
|
|
69
70
|
JNIEXPORT void JNICALL Java_cloud_unum_usearch_Index_c_1load(JNIEnv* env, jclass, jlong c_ptr, jstring path) {
|
|
70
71
|
char const* path_cstr = (*env).GetStringUTFChars(path, 0);
|
|
71
|
-
|
|
72
|
+
serialization_result_t result = reinterpret_cast<index_dense_t*>(c_ptr)->load(path_cstr);
|
|
73
|
+
if (!result) {
|
|
72
74
|
jclass jc = (*env).FindClass("java/lang/Error");
|
|
73
75
|
if (jc)
|
|
74
|
-
(*env).ThrowNew(jc,
|
|
76
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
75
77
|
}
|
|
76
78
|
(*env).ReleaseStringUTFChars(path, path_cstr);
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
JNIEXPORT void JNICALL Java_cloud_unum_usearch_Index_c_1view(JNIEnv* env, jclass, jlong c_ptr, jstring path) {
|
|
80
82
|
char const* path_cstr = (*env).GetStringUTFChars(path, 0);
|
|
81
|
-
|
|
83
|
+
serialization_result_t result = reinterpret_cast<index_dense_t*>(c_ptr)->view(path_cstr);
|
|
84
|
+
if (!result) {
|
|
82
85
|
jclass jc = (*env).FindClass("java/lang/Error");
|
|
83
86
|
if (jc)
|
|
84
|
-
(*env).ThrowNew(jc,
|
|
87
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
85
88
|
}
|
|
86
89
|
(*env).ReleaseStringUTFChars(path, path_cstr);
|
|
87
90
|
}
|
|
@@ -122,10 +125,13 @@ JNIEXPORT void JNICALL Java_cloud_unum_usearch_Index_c_1add( //
|
|
|
122
125
|
float_span_t vector_span = float_span_t{vector_data, static_cast<std::size_t>(vector_dims)};
|
|
123
126
|
|
|
124
127
|
using key_t = typename index_dense_t::key_t;
|
|
125
|
-
|
|
128
|
+
using add_result_t = typename index_dense_t::add_result_t;
|
|
129
|
+
|
|
130
|
+
add_result_t result = reinterpret_cast<index_dense_t*>(c_ptr)->add(static_cast<key_t>(key), vector_span);
|
|
131
|
+
if (!result) {
|
|
126
132
|
jclass jc = (*env).FindClass("java/lang/Error");
|
|
127
133
|
if (jc)
|
|
128
|
-
(*env).ThrowNew(jc,
|
|
134
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
129
135
|
}
|
|
130
136
|
(*env).ReleaseFloatArrayElements(vector, vector_data, 0);
|
|
131
137
|
}
|
|
@@ -157,10 +163,35 @@ JNIEXPORT jintArray JNICALL Java_cloud_unum_usearch_Index_c_1search( //
|
|
|
157
163
|
} else {
|
|
158
164
|
jclass jc = (*env).FindClass("java/lang/Error");
|
|
159
165
|
if (jc)
|
|
160
|
-
(*env).ThrowNew(jc,
|
|
166
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
161
167
|
}
|
|
162
168
|
|
|
163
169
|
(*env).ReleaseFloatArrayElements(vector, vector_data, 0);
|
|
164
170
|
std::free(matches_data);
|
|
165
171
|
return matches;
|
|
166
172
|
}
|
|
173
|
+
|
|
174
|
+
JNIEXPORT bool JNICALL Java_cloud_unum_usearch_Index_c_1remove(JNIEnv* env, jclass, jlong c_ptr, jlong key) {
|
|
175
|
+
using key_t = typename index_dense_t::key_t;
|
|
176
|
+
using labeling_result_t = typename index_dense_t::labeling_result_t;
|
|
177
|
+
labeling_result_t result = reinterpret_cast<index_dense_t*>(c_ptr)->remove(static_cast<key_t>(key));
|
|
178
|
+
if (!result) {
|
|
179
|
+
jclass jc = (*env).FindClass("java/lang/Error");
|
|
180
|
+
if (jc)
|
|
181
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
182
|
+
}
|
|
183
|
+
return result.completed;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
JNIEXPORT bool JNICALL Java_cloud_unum_usearch_Index_c_1rename(JNIEnv* env, jclass, jlong c_ptr, jlong from, jlong to) {
|
|
187
|
+
using key_t = typename index_dense_t::key_t;
|
|
188
|
+
using labeling_result_t = typename index_dense_t::labeling_result_t;
|
|
189
|
+
labeling_result_t result =
|
|
190
|
+
reinterpret_cast<index_dense_t*>(c_ptr)->rename(static_cast<key_t>(from), static_cast<key_t>(to));
|
|
191
|
+
if (!result) {
|
|
192
|
+
jclass jc = (*env).FindClass("java/lang/Error");
|
|
193
|
+
if (jc)
|
|
194
|
+
(*env).ThrowNew(jc, result.error.release());
|
|
195
|
+
}
|
|
196
|
+
return result.completed;
|
|
197
|
+
}
|
|
@@ -103,6 +103,22 @@ JNIEXPORT void JNICALL Java_cloud_unum_usearch_Index_c_1load
|
|
|
103
103
|
JNIEXPORT void JNICALL Java_cloud_unum_usearch_Index_c_1view
|
|
104
104
|
(JNIEnv *, jclass, jlong, jstring);
|
|
105
105
|
|
|
106
|
+
/*
|
|
107
|
+
* Class: cloud_unum_usearch_Index
|
|
108
|
+
* Method: c_remove
|
|
109
|
+
* Signature: (JLjava/lang/String;)V
|
|
110
|
+
*/
|
|
111
|
+
JNIEXPORT bool JNICALL Java_cloud_unum_usearch_Index_c_1remove
|
|
112
|
+
(JNIEnv *, jclass, jlong);
|
|
113
|
+
|
|
114
|
+
/*
|
|
115
|
+
* Class: cloud_unum_usearch_Index
|
|
116
|
+
* Method: c_rename
|
|
117
|
+
* Signature: (JLjava/lang/String;)V
|
|
118
|
+
*/
|
|
119
|
+
JNIEXPORT bool JNICALL Java_cloud_unum_usearch_Index_c_1rename
|
|
120
|
+
(JNIEnv *, jclass, jlong, jlong);
|
|
121
|
+
|
|
106
122
|
#ifdef __cplusplus
|
|
107
123
|
}
|
|
108
124
|
#endif
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/** Search result object. */
|
|
2
|
+
class Matches {
|
|
3
|
+
/**
|
|
4
|
+
* @param {BigUint64Array} keys - The keys of the nearest neighbors found.
|
|
5
|
+
* @param {Float32Array} distances - The distances of the nearest neighbors found.
|
|
6
|
+
* @param {bigint} count - The count of nearest neighbors found.
|
|
7
|
+
*/
|
|
8
|
+
constructor(keys, distances, count) {
|
|
9
|
+
this.keys = keys;
|
|
10
|
+
this.distances = distances;
|
|
11
|
+
this.count = count;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** K-Approximate Nearest Neighbors search index. */
|
|
16
|
+
class Index {
|
|
17
|
+
/**
|
|
18
|
+
* Constructs a new index.
|
|
19
|
+
*
|
|
20
|
+
* @param {bigint} dimensions
|
|
21
|
+
* @param {string} metric
|
|
22
|
+
* @param {string} quantization
|
|
23
|
+
* @param {bigint} capacity
|
|
24
|
+
* @param {bigint} connectivity
|
|
25
|
+
* @param {bigint} expansion_add
|
|
26
|
+
* @param {bigint} expansion_search
|
|
27
|
+
*/
|
|
28
|
+
constructor(
|
|
29
|
+
dimensions,
|
|
30
|
+
metric,
|
|
31
|
+
quantization,
|
|
32
|
+
capacity,
|
|
33
|
+
connectivity,
|
|
34
|
+
expansion_add,
|
|
35
|
+
expansion_search
|
|
36
|
+
) {}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Returns the dimensionality of vectors.
|
|
40
|
+
* @return {bigint} The dimensionality of vectors.
|
|
41
|
+
*/
|
|
42
|
+
dimensions() {}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Returns the bigint of vectors currently indexed.
|
|
46
|
+
* @return {bigint} The bigint of vectors currently indexed.
|
|
47
|
+
*/
|
|
48
|
+
size() {}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Returns index capacity.
|
|
52
|
+
* @return {bigint} The capacity of index.
|
|
53
|
+
*/
|
|
54
|
+
capacity() {}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Returns connectivity.
|
|
58
|
+
* @return {bigint} The connectivity of index.
|
|
59
|
+
*/
|
|
60
|
+
connectivity() {}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Write index to a file.
|
|
64
|
+
* @param {string} path File path to write.
|
|
65
|
+
*/
|
|
66
|
+
save(path) {}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Load index from a file.
|
|
70
|
+
* @param {string} path File path to read.
|
|
71
|
+
*/
|
|
72
|
+
load(path) {}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* View index from a file, without loading into RAM.
|
|
76
|
+
* @param {string} path File path to read.
|
|
77
|
+
*/
|
|
78
|
+
view(path) {}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Add n vectors of dimension d to the index.
|
|
82
|
+
*
|
|
83
|
+
* @param {bigint | bigint[]} keys Input identifiers for every vector.
|
|
84
|
+
* @param {Float32Array | Float32Array[]} mat Input matrix, matrix of size n * d.
|
|
85
|
+
*/
|
|
86
|
+
add(keys, mat) {}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Query n vectors of dimension d to the index. Return at most k vectors for each.
|
|
90
|
+
* If there are not enough results for a query, the result array is padded with -1s.
|
|
91
|
+
*
|
|
92
|
+
* @param {Float32Array} mat Input vectors to search, matrix of size n * d.
|
|
93
|
+
* @param {bigint} k The bigint of nearest neighbors to search for.
|
|
94
|
+
* @return {Matches} Output of the search result.
|
|
95
|
+
*/
|
|
96
|
+
search(mat, k) {}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Check if an entry is contained in the index.
|
|
100
|
+
*
|
|
101
|
+
* @param {bigint} key Identifier to look up.
|
|
102
|
+
*/
|
|
103
|
+
contains(key) {}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Remove a vector from the index.
|
|
107
|
+
*
|
|
108
|
+
* @param {bigint} key Input identifier for every vector to be removed.
|
|
109
|
+
*/
|
|
110
|
+
remove(key) {}
|
|
111
|
+
}
|
package/javascript/lib.cpp
CHANGED
|
@@ -106,7 +106,7 @@ Index::Index(Napi::CallbackInfo const& ctx) : Napi::ObjectWrap<Index>(ctx) {
|
|
|
106
106
|
std::string quantization_str = params.Get("quantization").As<Napi::String>().Utf8Value();
|
|
107
107
|
expected_gt<scalar_kind_t> expected = scalar_kind_from_name(quantization_str.c_str(), quantization_str.size());
|
|
108
108
|
if (!expected) {
|
|
109
|
-
Napi::TypeError::New(env, expected.error.
|
|
109
|
+
Napi::TypeError::New(env, expected.error.release()).ThrowAsJavaScriptException();
|
|
110
110
|
return;
|
|
111
111
|
}
|
|
112
112
|
quantization = *expected;
|
|
@@ -118,7 +118,7 @@ Index::Index(Napi::CallbackInfo const& ctx) : Napi::ObjectWrap<Index>(ctx) {
|
|
|
118
118
|
std::string metric_str = params.Get("metric").As<Napi::String>().Utf8Value();
|
|
119
119
|
expected_gt<metric_kind_t> expected = metric_from_name(metric_str.c_str(), metric_str.size());
|
|
120
120
|
if (!expected) {
|
|
121
|
-
Napi::TypeError::New(env, expected.error.
|
|
121
|
+
Napi::TypeError::New(env, expected.error.release()).ThrowAsJavaScriptException();
|
|
122
122
|
return;
|
|
123
123
|
}
|
|
124
124
|
metric_kind = *expected;
|
|
@@ -154,7 +154,10 @@ void Index::Save(Napi::CallbackInfo const& ctx) {
|
|
|
154
154
|
|
|
155
155
|
try {
|
|
156
156
|
std::string path = ctx[0].As<Napi::String>();
|
|
157
|
-
native_->save(path.c_str());
|
|
157
|
+
auto result = native_->save(path.c_str());
|
|
158
|
+
if (!result)
|
|
159
|
+
return Napi::TypeError::New(env, result.error.release()).ThrowAsJavaScriptException();
|
|
160
|
+
|
|
158
161
|
} catch (...) {
|
|
159
162
|
Napi::TypeError::New(env, "Serialization failed").ThrowAsJavaScriptException();
|
|
160
163
|
}
|
|
@@ -171,7 +174,10 @@ void Index::Load(Napi::CallbackInfo const& ctx) {
|
|
|
171
174
|
|
|
172
175
|
try {
|
|
173
176
|
std::string path = ctx[0].As<Napi::String>();
|
|
174
|
-
native_->load(path.c_str());
|
|
177
|
+
auto result = native_->load(path.c_str());
|
|
178
|
+
if (!result)
|
|
179
|
+
return Napi::TypeError::New(env, result.error.release()).ThrowAsJavaScriptException();
|
|
180
|
+
|
|
175
181
|
} catch (...) {
|
|
176
182
|
Napi::TypeError::New(env, "Loading failed").ThrowAsJavaScriptException();
|
|
177
183
|
}
|
|
@@ -188,7 +194,10 @@ void Index::View(Napi::CallbackInfo const& ctx) {
|
|
|
188
194
|
|
|
189
195
|
try {
|
|
190
196
|
std::string path = ctx[0].As<Napi::String>();
|
|
191
|
-
native_->view(path.c_str());
|
|
197
|
+
auto result = native_->view(path.c_str());
|
|
198
|
+
if (!result)
|
|
199
|
+
return Napi::TypeError::New(env, result.error.release()).ThrowAsJavaScriptException();
|
|
200
|
+
|
|
192
201
|
} catch (...) {
|
|
193
202
|
Napi::TypeError::New(env, "Memory-mapping failed").ThrowAsJavaScriptException();
|
|
194
203
|
}
|
|
@@ -216,7 +225,10 @@ void Index::Add(Napi::CallbackInfo const& ctx) {
|
|
|
216
225
|
return Napi::TypeError::New(env, "Wrong number of dimensions").ThrowAsJavaScriptException();
|
|
217
226
|
|
|
218
227
|
try {
|
|
219
|
-
native_->add(key, vector);
|
|
228
|
+
auto result = native_->add(key, vector);
|
|
229
|
+
if (!result)
|
|
230
|
+
return Napi::TypeError::New(env, result.error.release()).ThrowAsJavaScriptException();
|
|
231
|
+
|
|
220
232
|
} catch (std::bad_alloc const&) {
|
|
221
233
|
return Napi::TypeError::New(env, "Out of memory").ThrowAsJavaScriptException();
|
|
222
234
|
} catch (...) {
|
|
@@ -234,7 +246,8 @@ void Index::Add(Napi::CallbackInfo const& ctx) {
|
|
|
234
246
|
.ThrowAsJavaScriptException();
|
|
235
247
|
|
|
236
248
|
if (native_->size() + length >= native_->capacity())
|
|
237
|
-
native_->reserve(ceil2(native_->size() + length))
|
|
249
|
+
if (!native_->reserve(ceil2(native_->size() + length)))
|
|
250
|
+
return Napi::TypeError::New(env, "Out of memory!").ThrowAsJavaScriptException();
|
|
238
251
|
|
|
239
252
|
for (std::size_t i = 0; i < length; i++) {
|
|
240
253
|
Napi::Value key_js = keys_js[i];
|
|
@@ -280,7 +293,14 @@ Napi::Value Index::Search(Napi::CallbackInfo const& ctx) {
|
|
|
280
293
|
static_assert(std::is_same<std::uint64_t, key_t>::value, "Matches.key interface expects BigUint64Array");
|
|
281
294
|
Napi::Float32Array distances_js = Napi::Float32Array::New(env, wanted);
|
|
282
295
|
try {
|
|
283
|
-
|
|
296
|
+
|
|
297
|
+
auto result = native_->search(vector, wanted);
|
|
298
|
+
if (!result) {
|
|
299
|
+
Napi::TypeError::New(env, result.error.release()).ThrowAsJavaScriptException();
|
|
300
|
+
return {};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
std::uint64_t count = result.dump_to(matches_js.Data(), distances_js.Data());
|
|
284
304
|
Napi::Object result_js = Napi::Object::New(env);
|
|
285
305
|
result_js.Set("keys", matches_js);
|
|
286
306
|
result_js.Set("distances", distances_js);
|
|
@@ -313,7 +333,7 @@ Napi::Value Index::Remove(Napi::CallbackInfo const& ctx) {
|
|
|
313
333
|
try {
|
|
314
334
|
auto result = native_->remove(key);
|
|
315
335
|
if (!result) {
|
|
316
|
-
Napi::TypeError::New(env,
|
|
336
|
+
Napi::TypeError::New(env, result.error.release()).ThrowAsJavaScriptException();
|
|
317
337
|
return {};
|
|
318
338
|
}
|
|
319
339
|
return Napi::Boolean::New(env, result.completed);
|
package/javascript/usearch.d.ts
CHANGED
|
@@ -23,7 +23,15 @@ export class Index {
|
|
|
23
23
|
* @param {bigint} expansion_add
|
|
24
24
|
* @param {bigint} expansion_search
|
|
25
25
|
*/
|
|
26
|
-
constructor(
|
|
26
|
+
constructor(
|
|
27
|
+
dimensions: bigint,
|
|
28
|
+
metric: string,
|
|
29
|
+
quantization: string,
|
|
30
|
+
capacity: bigint,
|
|
31
|
+
connectivity: bigint,
|
|
32
|
+
expansion_add: bigint,
|
|
33
|
+
expansion_search: bigint
|
|
34
|
+
);
|
|
27
35
|
|
|
28
36
|
/**
|
|
29
37
|
* Returns the dimensionality of vectors.
|