usearch 0.11.1 → 0.12.0

package/.github/workflows/prerelease.yml

@@ -37,12 +37,8 @@ jobs:
           pip install pytest numpy
       - name: Build locally
         run: python -m pip install .
-      - name: Test with PyTest on NIX
-        if: matrix.os != 'windows-2022'
+      - name: Test with PyTest
         run: pytest python/scripts/test.py
-      - name: Test with PyTest on Windows
-        if: matrix.os == 'windows-2022'
-        run: pytest python\scripts\test.py
 
   test_javascript:
     name: Test JavaScript

package/.github/workflows/release.yml

@@ -11,7 +11,7 @@ env:
 
 # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
 permissions:
-  contents: read
+  contents: write
   pages: write
   id-token: write
 
@@ -27,13 +27,33 @@ jobs:
     - uses: actions/setup-node@v3
     - run: npm install --save-dev @semantic-release/exec @semantic-release/git conventional-changelog-eslint semantic-release && npx semantic-release
 
+  rebase:
+    name: Rebase from main into main-dev
+    needs: versioning
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the latest code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Perform Rebase
+        run: |
+          git checkout main-dev
+          git rebase main
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: main-dev
+
   build_wheels:
     name: Build Wheels for ${{ matrix.os }}
     needs: versioning
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, macOS-11]
+        os: [ubuntu-20.04, macOS-11, windows-2022]
 
     steps:
       - uses: actions/checkout@v3

package/.gitignore

@@ -0,0 +1,84 @@
+# Ignore all files without an extension
+*
+!/**/
+!*.*
+
+# Temporary files
+.DS_Store
+*.fbin
+*.ibin
+datasets
+bin
+*.usearch
+
+# C++ builds
+build
+build_release
+build_debug
+_deps
+CMakeCache.txt
+CMakeFiles
+Makefile
+cmake_install.cmake
+
+# Profiling with GProf and `perf` on Linux
+gmon.out
+perf.data
+
+# Python wrappers
+__pycache__
+*.egg-info
+dist/
+wheelhouse/
+
+# JS wrappers
+package-lock.json
+node_modules/
+
+# Rust builds
+Cargo.lock
+target/
+
+# Java binings
+.gradle
+*.class
+
+# ObjC and Swift
+.build
+.swiftpm
+
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Allow golang go.mod
+!go.mod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app

package/.vscode/launch.json

@@ -50,7 +50,13 @@
             "stopAtEntry": false,
             "linux": {
                 "preLaunchTask": "Linux Build C++ Bench Debug",
-                "MIMode": "gdb"
+                "MIMode": "gdb",
+                "environment": [
+                    {
+                        "name": "MALLOC_CHECK_",
+                        "value": "2"
+                    }
+                ]
             },
             "osx": {
                 "preLaunchTask": "MacOS Build C++ Bench Debug",

package/.vscode/settings.json

@@ -99,9 +99,12 @@
         "__availability": "cpp",
         "version": "cpp",
         "future": "cpp",
-        "__verbose_abort": "cpp"
+        "__verbose_abort": "cpp",
+        "shared_mutex": "cpp"
     },
     "cSpell.words": [
+        "allclose",
+        "arange",
         "astype",
         "Availible",
         "BLAS",
@@ -110,11 +113,15 @@
         "downcasted",
         "Downcasting",
         "dtype",
+        "equi",
         "equidimensional",
         "FAISS",
+        "fbin",
+        "groundtruth",
         "hashable",
         "HNSW",
         "hnswlib",
+        "ibin",
         "jaccard",
         "longlong",
         "memmap",
@@ -122,12 +129,19 @@
         "ndarray",
         "ndim",
         "Numba",
+        "numpy",
         "OPENMP",
         "preprocess",
+        "Println",
+        "pytest",
         "Quickstart",
         "rtype",
         "simsimd",
         "SLOC",
+        "sorensen",
+        "tanimoto",
+        "uninitialize",
+        "unumusearch",
         "usearch",
         "usecases"
     ],

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "0.11.1"
+version = "0.12.0"
 authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/README.md

@@ -66,11 +66,11 @@ Both USearch and FAISS implement the same HNSW algorithm.
 But they differ in a lot of design decisions.
 USearch is designed to be compact and broadly compatible without sacrificing performance.
 
-|            | FAISS, `f32` | USearch, `f32` | USearch, `f16` | USearch, `f8` | USearch + Numba, `f32` |
-| :--------- | -----------: | -------------: | -------------: | ------------: | ---------------------: |
-| Insertions |       76 K/s |         89 K/s |         73 K/s |   **137 K/s** |                 99 K/s |
-| Queries    |      118 K/s |        167 K/s |        140 K/s |   **281 K/s** |                165 K/s |
-| Recall @1  |          99% |          99.2% |            98% |     **99.2%** |                  99.2% |
+|            | FAISS, `f32` | USearch, `f32` | USearch, `f16` | USearch, `f8` |
+| :--------- | -----------: | -------------: | -------------: | ------------: |
+| Insertions |       76 K/s |        105 K/s |        115 K/s |   **202 K/s** |
+| Queries    |      118 K/s |        174 K/s |        173 K/s |   **304 K/s** |
+| Recall @1  |          99% |          99.2% |          99.1% |     **99.2%** |
 
 > Dataset: 1M vectors sample of the Deep1B dataset.
 > Hardware: `c7g.metal` AWS instance with 64 cores and DDR5 memory.
@@ -202,9 +202,11 @@ The following distances are pre-packaged:
 - `ip_gt<scalar_t>` for "Inner Product" or "Dot Product" distance.
 - `l2sq_gt<scalar_t>` for the squared "L2" or "Euclidean" distance.
 - `jaccard_gt<scalar_t>` for "Jaccard" distance between two ordered sets of unique elements.
-- `bit_hamming_gt<scalar_t>` for "Hamming" distance, as the number of shared bits in hashes.
+- `bitwise_hamming_gt<scalar_t>` for "Hamming" distance, as the number of shared bits in hashes.
+- `bitwise_tanimoto_gt<scalar_t>` for "Tanimoto" coefficient for bit-strings.
+- `bitwise_sorensen_gt<scalar_t>` for "Dice-Sorensen" coefficient for bit-strings.
 - `pearson_correlation_gt<scalar_t>` for "Pearson" correlation between probability distributions.
-- `haversine_gt<scalar_t>` for "Haversine" or "Great Circle" distance between coordinates.
+- `haversine_gt<scalar_t>` for "Haversine" or "Great Circle" distance between coordinates used in GIS applications.
 
 #### Multi-Threading
 
@@ -311,6 +313,12 @@ def python_dot(a, b, n, m):
 index = Index(ndim=ndim, metric=python_dot.address)
 ```
 
+To use Numba JIT, install USearch with extras:
+
+```sh
+pip install usearch[jit]
+```
+
 [numba]: https://numba.readthedocs.io/en/stable/reference/jit-compilation.html#c-callbacks
 
 #### Tooling
@@ -457,6 +465,33 @@ assert(results.0[0] == 42)
 
 ### GoLang
 
+#### Installation
+
+```golang
+import (
+	"github.com/unum-cloud/usearch/golang"
+)
+```
+
+#### Quickstart
+
+```golang
+package main
+
+import (
+	"fmt"
+	"github.com/unum-cloud/usearch/golang"
+)
+
+func main() {
+	conf := usearch.DefaultConfig(128)
+	index := usearch.NewIndex(conf)
+	v := make([]float32, 128)
+	index.Add(42, v)
+	results := index.Search(v, 1)
+}
+```
+
 ### Wolfram
 
 ## TODO

package/binding.gyp

@@ -16,8 +16,8 @@
             "dependencies": [
                 "<!(node -p \"require('node-addon-api').gyp\")"
             ],
-            "cflags": ["-fexceptions", "-Wno-unknown-pragmas"],
-            "cflags_cc": ["-fexceptions", "-Wno-unknown-pragmas", "-std=c++11"],
+            "cflags": ["-fexceptions", "-Wno-unknown-pragmas", "-Wno-maybe-uninitialized"],
+            "cflags_cc": ["-fexceptions", "-Wno-unknown-pragmas", "-Wno-maybe-uninitialized", "-std=c++11"],
             "xcode_settings": {
                 "GCC_ENABLE_CPP_EXCEPTIONS": "YES",
                 "CLANG_CXX_LIBRARY": "libc++",

package/c/lib.cpp

@@ -0,0 +1,141 @@
+#include "../src/punned.hpp"
+
+extern "C" {
+#include "usearch.h"
+}
+
+using namespace unum::usearch;
+using namespace unum;
+
+using label_t = int;
+using distance_t = float;
+using punned_index_t = punned_gt<label_t>;
+using span_t = span_gt<float>;
+
+void* usearch_new(                                        //
+    usearch_metric_t metric, usearch_accuracy_t accuracy, //
+    int dimensions, int capacity, int connectivity,       //
+    int expansion_add, int expansion_search) {
+
+    try {
+        config_t config;
+        config.expansion_add = static_cast<std::size_t>(expansion_add);
+        config.expansion_search = static_cast<std::size_t>(expansion_search);
+        config.connectivity = static_cast<std::size_t>(connectivity);
+        config.max_elements = static_cast<std::size_t>(capacity);
+        config.max_threads_add = std::thread::hardware_concurrency();
+        config.max_threads_search = std::thread::hardware_concurrency();
+
+        // TODO: Implement safe buffers for error messages, either here on in
+        // the C++ type-punned implementation.
+        // static constexpr std::size_t error_limit_k = 128;
+        // struct safe_punned_index_t {
+        //     punned_index_t punned_index;
+        //     char error_buffers[1];
+        // };
+        // std::size_t bytes_for_error_messages = error_limit_k * config.max_threads();
+        // std::size_t bytes_to_alloc = sizeof(punned_index_t) + bytes_for_error_messages;
+        // safe_punned_index_t* result = (safe_punned_index_t*)std::malloc(bytes_to_alloc);
+
+        accuracy_t accuracy = accuracy_from_name(accuracy_str, accuracy_len);
+        common_metric_kind_t metric_kind = common_metric_from_name(metric_str, metric_len);
+        punned_index_t index = make_punned<punned_index_t>( //
+            metric_kind, static_cast<std::size_t>(dimensions), accuracy, config);
+
+        punned_index_t* result_ptr = new punned_index_t(std::move(index));
+        return result_ptr;
+    } catch (std::exception const&) {
+    }
+    return NULL;
+}
+
+void usearch_destroy(punned_index_t* index) { delete index; }
+
+char const* usearch_save(punned_index_t* index, char const* path) {
+    try {
+        index->save(path);
+        return NULL;
+    } catch (std::exception& e) {
+        // q:: added these to make sure I do not pass a potentially dangling e.what()
+        // though I realize allocation below may fail and I am not checking for it.
+        // what can I do there? Golang side tries to free whatever string is passed from
+        // so passing a compile time constant "double fault" string is not an option
+        char* res = new char[strlen(e.what()) + 1];
+        strcpy(res, e.what());
+        return res;
+    }
+}
+
+char const* usearch_load(punned_index_t* index, char const* path) {
+    try {
+        index->load(path);
+        return NULL;
+    } catch (std::exception& e) {
+        char* res = new char[strlen(e.what()) + 1];
+        strcpy(res, e.what());
+        return res;
+    }
+}
+
+char const* usearch_view(punned_index_t* index, char const* path) {
+    try {
+        index->view(path);
+        return NULL;
+    } catch (std::exception& e) {
+        char* res = new char[strlen(e.what()) + 1];
+        strcpy(res, e.what());
+        return res;
+    }
+}
+
+int usearch_size(punned_index_t* index) { return index->size(); }
+int usearch_connectivity(punned_index_t* index) { return index->connectivity(); }
+int usearch_dimensions(punned_index_t* index) { return index->dimensions(); }
+int usearch_capacity(punned_index_t* index) { return index->capacity(); }
+
+char const* usearch_reserve(punned_index_t* index, int capacity) {
+    try {
+        index->reserve(capacity);
+        return NULL;
+    } catch (std::exception& e) {
+        char* res = new char[strlen(e.what()) + 1];
+        strcpy(res, e.what());
+        return res;
+    }
+}
+
+char const* usearch_add(punned_index_t* index, int label, float* vector) {
+    // q:: I followed the java example to have try catches everywhere
+    // but they are kind of useless as most errors are outside of cpp so
+    // those translate into segfaults and are not caught by the runtime
+    try {
+        index->add(label, vector);
+    } catch (std::exception& e) {
+        char* res = new char[strlen(e.what()) + 1];
+        strcpy(res, e.what());
+        return res;
+    }
+    return NULL;
+}
+
+SearchResults usearch_search(punned_index_t* index, float* query, int query_len, int limit) {
+    // todo:: this could be allocated as golang slice
+    // to avoid a copy. not sure how it interacts with gc
+    // that is why doing this now
+    label_t* matches_data = new label_t[limit];
+
+    // todo:: pass the distances to outside world
+    distance_t* dist_data = new distance_t[limit];
+    SearchResults res{0};
+    try {
+        span_t vector_span = span_t{query, static_cast<std::size_t>(query_len)};
+        res.Len = index->search(vector_span, static_cast<std::size_t>(limit)).dump_to(matches_data, dist_data);
+    } catch (std::exception& e) {
+        res.Error = new char[strlen(e.what()) + 1];
+        strcpy(res.Error, e.what());
+        // res.Error = res;
+    }
+    res.Labels = matches_data;
+    res.Distances = dist_data;
+    return res;
+}

package/c/usearch.h

@@ -0,0 +1,43 @@
+#ifndef UNUM_USEARCH_H
+#define UNUM_USEARCH_H
+
+#include <stdlib.h>
+
+typedef enum {
+    usearch_metric_ip_k = 0,
+    usearch_metric_l2sq_k = 1,
+    usearch_metric_cos_k = 2,
+    usearch_metric_haversine_k = 3
+} usearch_metric_t;
+
+typedef enum {
+    usearch_accuracy_f32_k = 0,
+    usearch_accuracy_f16_k = 1,
+    usearch_accuracy_f8_k = 2,
+} usearch_accuracy_t;
+
+void* usearch_new(                                        //
+    usearch_metric_t metric, usearch_accuracy_t accuracy, //
+    int dimensions, int capacity, int connectivity,       //
+    int expansion_add, int expansion_search);
+
+void usearch_destroy(void* index);
+
+char const* usearch_save(void* index, char const* path);
+char const* usearch_load(void* index, char const* path);
+char const* usearch_view(void* index, char const* path);
+
+int usearch_size(void* index);
+int usearch_connectivity(void* index);
+int usearch_dimensions(void* index);
+int usearch_capacity(void* index);
+
+char const* usearch_reserve(void* index, int capacity);
+char const* usearch_add(void* index, int label, float const* vec);
+
+char const* usearch_search(        //
+    void* index,                   //
+    float const* query, int limit, //
+    int* labels, float* distances, int* count);
+
+#endif

package/conanfile.py

@@ -4,7 +4,7 @@ from conans import ConanFile
 class USearchConan(ConanFile):
 
     name = 'USearch'
-    version = '0.11.1'
+    version = '0.12.0'
     license = 'Apache 2.0'
     url = 'https://github.com/unum-cloud/usearch'
     description = 'Smaller & Faster Single-File Vector Search Engine from Unum'

package/docs/compilation.md

@@ -62,8 +62,12 @@ Benchmarking:
 
 ## Python
 
+Use PyTest to validate the build.
+The `-s` option will disable capturing the logs.
+The `-x` option will exit after first failure to simplify debugging.
+
 ```sh
-pip install -e . && pytest python/scripts/test.py
+pip install -e . && pytest python/scripts/test.py -s -x
 ```
 
 Linting:
@@ -95,8 +99,7 @@ python python/scripts/bench_params.py connectivity
 ## JavaScript
 
 ```sh
-npm install
-node javascript/test.js
+npm install && node javascript/test.js
 npm publish
 ```
 
@@ -148,6 +151,14 @@ java -cp . -Djava.library.path=/Users/av/github/usearch/java cloud.unum.usearch.
 brew install --cask wolfram-engine
 ```
 
+## C
+
+### Linux
+
+```sh
+g++ -shared -fPIC lib.cpp -I ../include/  -I ../fp16/include/ -o libusearch.so
+```
+
 ## Docker
 
 ```sh

package/golang/go.mod

@@ -0,0 +1,3 @@
+module github.com/unum-cloud/usearch/golang
+
+go 1.19