usearch 0.13.0 → 0.15.0

package/.github/workflows/release.yml

@@ -27,8 +27,9 @@ jobs:
     - uses: actions/setup-node@v3
     - run: npm install --save-dev @semantic-release/exec @semantic-release/git conventional-changelog-eslint semantic-release && npx semantic-release
 
+
   rebase:
-    name: Rebase Developmtn Branch
+    name: Rebase Development Branch
     needs: versioning
     runs-on: ubuntu-latest
     steps:
@@ -41,11 +42,59 @@ jobs:
         run: |
           git checkout main-dev
           git rebase main
+          
       - name: Push changes
-        uses: ad-m/github-push-action@master
+        uses: CasperWA/push-protected@v2
         with:
-          github_token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
+          token: ${{ secrets.SEMANTIC_REBASE_TOKEN }}
           branch: main-dev
+          unprotect_reviews: True
+          force: True
+
+
+  publish_hashes:
+    name: Add hashes to release notes
+    runs-on: ubuntu-latest
+    needs: versioning
+    steps:
+    - name: Get last release ID
+      id: last_release
+      run: |
+          response=$(curl -s -H "Authorization: Bearer ${{ secrets.SEMANTIC_RELEASE_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/releases/latest")
+          echo "last_release_id=$(echo "$response" | jq -r '.id')" >> $GITHUB_OUTPUT
+
+    - uses: robinraju/release-downloader@v1.8
+      name: Download release assets
+      with:
+        latest: true
+        fileName: "*"
+        out-file-path: "downloads"
+        tarBall: true
+        zipBall: true
+
+    - name: Calculate source hashes
+      id: hashes
+      run: |
+        source_zip_path=$(find $GITHUB_WORKSPACE/downloads -name "*.zip")
+        zip_hash=$(sha256sum "$source_zip_path" | awk '{print $1}')
+        source_tar_path=$(find $GITHUB_WORKSPACE/downloads -name "*.tar.gz" ! -name "docs.tar.gz")
+        tar_hash=$(sha256sum "$source_tar_path" | awk '{print $1}')
+        echo "zip_hash=$zip_hash" >> $GITHUB_OUTPUT
+        echo "tar_hash=$tar_hash" >> $GITHUB_OUTPUT
+          
+    - name: Upload hashes to release
+      uses: irongut/EditRelease@v1.2.0
+      with:
+        token: ${{ secrets.SEMANTIC_RELEASE_TOKEN }}
+        id: ${{ steps.last_release.outputs.last_release_id }}
+        prerelease: false
+        replacebody: false
+        body: |
+            ## Hashes
+            * Source code (zip) : `${{ steps.hashes.outputs.zip_hash }}` 
+            * Source code (tar.gz) : `${{ steps.hashes.outputs.tar_hash }}`
+
 
   build_wheels:
     name: Build Wheels for ${{ matrix.os }}
@@ -236,11 +285,15 @@ jobs:
         with:
           ref: 'main'   
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y doxygen graphviz dia git && pip install sphinx breathe furo m2r2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery
+        run: |
+              sudo apt update && 
+              sudo apt install -y doxygen graphviz dia git && 
+              pip install sphinx sphinx-js breathe furo m2r2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery && 
+              npm install -g jsdoc
       - name: Install USearch from PyPi
         run: pip install usearch
       - name: Build documentation
-        run: cd docs && make html
+        run: cd docs && doxygen conf.dox && make html
       - name: Copy assets
         run: cp -r assets build/docs/html/
       - name: Compress assets
@@ -260,16 +313,19 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     runs-on: ubuntu-22.04
     needs: build_docs
-    if: ${{ github.event.build_docs.conclusion == 'success' }}
     steps:
       - uses: robinraju/release-downloader@v1.8
         with:
           latest: true
           fileName: docs.tar.gz
+      - name: Unpack docs
+        run: tar -xf ./docs.tar.gz
       - name: Setup GitHub Pages
         uses: actions/configure-pages@v2
       - name: Upload artifacts
         uses: actions/upload-pages-artifact@v1
+        with:
+          path: ./build/docs/html
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v1
@@ -278,7 +334,6 @@ jobs:
     name: Deploy Vercel
     runs-on: ubuntu-22.04
     needs: build_docs
-    if: ${{ github.event.build_docs.conclusion == 'success' }}
     steps:
       - name: Notify Vercel
         uses: fjogeleit/http-request-action@v1

package/.github/workflows/update_version.sh

@@ -4,4 +4,9 @@ echo $1 > VERSION &&
     sed -i "s/\"version\": \".*\"/\"version\": \"$1\"/" package.json &&
     sed -i "s/^version: .*/version: $1/" CITATION.cff &&
     sed -i "s/version = \".*\"/version = \"$1\"/" Cargo.toml &&
-    sed -i "s/version = '.*'/version = '$1'/" conanfile.py
+    sed -i "s/version = '.*'/version = '$1'/" conanfile.py &&
+    sed -i "s/^\(#define USEARCH_VERSION_MAJOR \).*/\1$(echo "$1" | cut -d. -f1)/" ./include/usearch/index.hpp &&
+    sed -i "s/^\(#define USEARCH_VERSION_MINOR \).*/\1$(echo "$1" | cut -d. -f2)/" ./include/usearch/index.hpp &&
+    sed -i "s/^\(#define USEARCH_VERSION_PATCH \).*/\1$(echo "$1" | cut -d. -f3)/" ./include/usearch/index.hpp &&
+    sed -i "s/<version>[0-9]\+\.[0-9]\+\.[0-9]\+/<version>$1/" README.md &&
+    sed -i "s/version = {0\.[0-9]\+\.[0-9]\+}/version = {$1}/" README.md

package/.vscode/launch.json

@@ -113,33 +113,32 @@
             "MIMode": "gdb"
         },
         {
-            "name": "Debug PyBind11 Test",
-            "type": "pythoncpp",
+            "name": "Debug PyTest",
+            "type": "python",
             "request": "launch",
-            "pythonLaunchName": "Python: Current File",
-            "cppAttachName": "Debug PyBind11: Attach with GDB"
-        },
-        {
-            "name": "Debug PyBind11: Attach with GDB",
-            "type": "cppdbg",
-            "request": "attach",
-            "program": "/Users/av/miniconda3/bin/python",
-            "processId": "",
-            "MIMode": "lldb",
-            "setupCommands": [
-                {
-                    "description": "Enable pretty-printing for gdb",
-                    "text": "-enable-pretty-printing",
-                    "ignoreFailures": true
-                }
-            ]
+            "module": "pytest",
+            "console": "internalConsole",
+            "args": [
+                "${file}"
+            ],
+            "justMyCode": true
         },
         {
-            "name": "Python: Current File",
+            "name": "Debug Python Benchmark",
             "type": "python",
             "request": "launch",
-            "program": "${file}",
-            "console": "integratedTerminal"
+            "program": "python/scripts/bench.py",
+            "console": "internalConsole",
+            "args": [
+                "speed",
+                "--vectors",
+                "datasets/wiki_1M/base.1M.fbin",
+                "--queries",
+                "datasets/wiki_1M/query.public.100K.fbin",
+                "--neighbors",
+                "datasets/wiki_1M/groundtruth.public.100K.ibin",
+            ],
+            "justMyCode": true
         }
     ]
 }

package/.vscode/settings.json

@@ -107,6 +107,7 @@
         "arange",
         "astype",
         "Availible",
+        "bidict",
         "BLAS",
         "cflags",
         "citerator",
@@ -146,5 +147,10 @@
         "usecases"
     ],
     "autoDocstring.docstringFormat": "sphinx",
-    "java.configuration.updateBuildConfiguration": "interactive"
+    "java.configuration.updateBuildConfiguration": "interactive",
+    "python.testing.pytestArgs": [
+        "python"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
 }

package/CITATION.cff

@@ -2,10 +2,10 @@ cff-version: 1.2.0
 message: "If you use this software, please cite it as below."
 authors:
 - family-names: "Vardanian"
-  given-names: "Ashot"
+  given-names: "Ash"
   orcid: "https://orcid.org/0000-0002-4882-1815"
-title: "USearch"
-version: 0.3.0
+title: "USearch by Unum Cloud"
+version: 0.13.0
 doi: 10.5281/zenodo.7949416
 date-released: 2022-06-12
-url: "https://github.com/unum-cloud/usearch"
+url: "https://github.com/unum-cloud/usearch"

package/CMakeLists.txt

@@ -112,7 +112,7 @@ if(USEARCH_INSTALL)
 endif()
 
 if(${USEARCH_BUILD_TEST} OR ${USEARCH_BUILD_BENCHMARK})
-    add_subdirectory(src)
+    add_subdirectory(cpp)
 endif()
 
 if(${USEARCH_BUILD_WOLFRAM})

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "0.13.0"
+version = "0.15.0"
 authors = ["Ashot Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/Package.swift

@@ -18,7 +18,6 @@ let package = Package(
             sources: ["USearchObjective.mm"],
             cxxSettings: [
                 .headerSearchPath("../include/"),
-                .headerSearchPath("../src/"),
                 .headerSearchPath("../fp16/include/"),
                 .headerSearchPath("../robin-map/include/"),
                 .headerSearchPath("../simismd/include/")

package/README.md

@@ -48,9 +48,9 @@ Linux • MacOS • Windows • Docker • WebAssembly 🔜
 - [ ] Thread-safe `reserve`.
 - [ ] On-the-fly deletions.
 - [x] USearch + UForm Transformers = [Semantic Search](#ai--vector-search--semantic-search).
-- [x] USearch + RDKit = [Molecular Search](#ai--vector-search--semantic-search).
+- [x] USearch + RDKit = [Molecule Search](#ai--vector-search--semantic-search).
 
-[usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/usearch.hpp
+[usearch-header]: https://github.com/unum-cloud/usearch/blob/main/include/usearch/index.hpp
 [obscure-use-cases]: https://ashvardanian.com/posts/abusing-vector-search
 
 ---
@@ -133,14 +133,14 @@ Of course, the latency of external memory access will be higher, but it is in pa
 
 There are two usage patters:
 
-1. Bare-bones with `usearch/usearch.hpp`, only available in C++.
+1. Bare-bones with `usearch/index.hpp`, only available in C++.
 2. Full-fat version with it's own threads, mutexes, type-punning, quantization, that is available both in C++ and is wrapped for higher-level bindings.
 
 ### C++
 
 #### Installation
 
-To use in a C++ project simply copy the `include/usearch/usearch.hpp` header into your project.
+To use in a C++ project simply copy the `include/usearch/index.hpp` header into your project.
 Alternatively fetch it with CMake:
 
 ```cmake
@@ -164,7 +164,7 @@ index.add(/* label: */ 42, /* vector: */ {&vec[0], 3});
 auto results = index.search(/* query: */ {&vec[0], 3}, 5 /* neighbors */);
 
 for (std::size_t i = 0; i != results.size(); ++i)
-    results[i].member.label, results[i].member.vector, results[i].distance;
+    results[i].element.label, results[i].element.vector, results[i].distance;
 ```
 
 The `add` is thread-safe for concurrent index construction.
@@ -205,9 +205,9 @@ The following distances are pre-packaged:
 - `ip_gt<scalar_t>` for "Inner Product" or "Dot Product" distance.
 - `l2sq_gt<scalar_t>` for the squared "L2" or "Euclidean" distance.
 - `jaccard_gt<scalar_t>` for "Jaccard" distance between two ordered sets of unique elements.
-- `bitwise_hamming_gt<scalar_t>` for "Hamming" distance, as the number of shared bits in hashes.
-- `bitwise_tanimoto_gt<scalar_t>` for "Tanimoto" coefficient for bit-strings.
-- `bitwise_sorensen_gt<scalar_t>` for "Dice-Sorensen" coefficient for bit-strings.
+- `hamming_gt<scalar_t>` for "Hamming" distance, as the number of shared bits in hashes.
+- `tanimoto_gt<scalar_t>` for "Tanimoto" coefficient for bit-strings.
+- `sorensen_gt<scalar_t>` for "Dice-Sorensen" coefficient for bit-strings.
 - `pearson_correlation_gt<scalar_t>` for "Pearson" correlation between probability distributions.
 - `haversine_gt<scalar_t>` for "Haversine" or "Great Circle" distance between coordinates used in GIS applications.
 
@@ -299,19 +299,19 @@ Luckily, with the help of [Numba][numba], we can JIT compile a function with a m
 ```py
 from numba import cfunc, types, carray
 
+ndim = 256
 signature = types.float32(
     types.CPointer(types.float32),
-    types.CPointer(types.float32),
-    types.uint64, types.uint64)
+    types.CPointer(types.float32))
 
 @cfunc(signature)
-def python_dot(a, b, n, m):
-    a_array = carray(a, n)
-    b_array = carray(b, n)
+def python_dot(a, b):
+    a_array = carray(a, ndim)
+    b_array = carray(b, ndim)
     c = 0.0
-    for i in range(n):
+    for i in range(ndim):
         c += a_array[i] * b_array[i]
-    return c
+    return 1 - c
 
 index = Index(ndim=ndim, metric=python_dot.address)
 ```
@@ -374,8 +374,17 @@ cargo add usearch
 #### Quickstart
 
 ```rust
-let quant: &str = "f16";
-let index = new_ip(3, &quant, 0, 0, 0).unwrap();
+
+let options = IndexOptions {
+            dimensions: 5,
+            metric: MetricKind::IP,
+            quantization: ScalarKind::F16,
+            connectivity: 0,
+            expansion_add: 0,
+            expansion_search: 0
+        };
+
+let index = new_index(&options).unwrap();
 
 assert!(index.reserve(10).is_ok());
 assert!(index.capacity() >= 10);
@@ -497,11 +506,6 @@ func main() {
 
 ### Wolfram
 
-## TODO
-
-- JavaScript: Allow calling from "worker threads".
-- Rust: Allow passing a custom thread ID.
-- C# .NET bindings.
 
 ## Application Examples
 
@@ -538,12 +542,33 @@ def search(query: str) -> np.ndarray:
 server.run()
 ```
 
+We have pre-processed some commonly used datasets, cleaning the images, producing the vectors, and pre-building the index.
+
+| Dataset                             | Size | Images |          Preprocessed |
+| :---------------------------------- | ---: | -----: | --------------------: |
+| [Unsplash 25K][unsplash-25k-origin] |    - |   25 K | [HF][unsplash-25k-hf] |
+| [Unsplash 2M][unsplash-2m-origin]   |    - |    2 M |  [HF][unsplash-2m-hf] |
+| [LAION 400M][laion-400m-origin]     |    - |  400 M |   [HF][laion-400m-hf] |
+| [LAION 5B][laion-5b-origin]         |    - |    5 B |     [HF][laion-5b-hf] |
+
+
+[unsplash-25k-origin]: https://unum.cloud
+[unsplash-2m-origin]: https://unum.cloud
+[laion-400m-origin]: https://unum.cloud
+[laion-5b-origin]: https://unum.cloud
+[unsplash-25k-hf]: https://unum.cloud
+[unsplash-2m-hf]: https://unum.cloud
+[laion-400m-hf]: https://unum.cloud
+[laion-5b-hf]: https://unum.cloud
+
 ### USearch + RDKit = Molecular Search
 
-Research in natural sciences is expensive.
-To search for properties of different molecules, large databases are used.
-Comparing the molecules, however, is hard.
-Instead, binary hash-like fingerprints are produced from the graph structure, using packages like RDKit.
+Comparing molecule graphs and searching for similar structures is expensive and slow.
+It can be seen as a special case of the NP-Complete Subgraph Isomorphism problem.
+Luckily, domain-specific approximate methods exists.
+The one commonly used in Chemistry, is to generate structures from [SMILES][smiles], and later hash them into binary fingerprints.
+The later are searchable with bitwise similarity metrics, like the Tanimoto coefficient.
+Below is na example using the RDKit package.
 
 ```python
 from usearch.index import Index, MetricKind
@@ -565,10 +590,58 @@ index.add(labels, fingerprints)
 matches = index.search(fingerprints, 10)
 ```
 
-RDKit [provides][rdkit-fingerprints] Atom-Pair, Topological Torsion, Morgan, and Layered Fingerprints, among other options.
+RDKit [provides][rdkit-fingerprints] following fingerprinting techniques:
+
+- Atom-Pair, 
+- Topological Torsion, 
+- Morgan,
+- Layered Fingerprints.
+
+We have preprocessed some of the most commonly used datasets, and made it available for free on the HuggingFace portal, together with visual interface.
+
+| Dataset                   |     Size |   Molecules |     Preprocessed |
+| :------------------------ | -------: | ----------: | ---------------: |
+| [PubChem][pubchem-origin] |     8 GB | 115'034'339 | [HF][pubchem-hf] |
+| [GDB 13][gdb13-origin]    |   2.3 GB | 977'468'301 |   [HF][gdb13-hf] |
+| [REAL][real-origin]       | > 100 GB |         6 B |    [HF][real-hf] |
 
+[smiles]: https://en.wikipedia.org/wiki/Simplified_molecular-input_line-entry_system
 [rdkit-fingerprints]: https://www.rdkit.org/docs/RDKit_Book.html#additional-information-about-the-fingerprints
 
+[gdb13-origin]: https://zenodo.org/record/5172018/files/gdb13.tgz?download=1
+[pubchem-origin]: ftp://ftp.ncbi.nlm.nih.gov/pubchem/Compound/Extras/CID-SMILES.gz
+[real-origin]: https://enamine.net/compound-collections/real-compounds/real-database
+[pubchem-hf]: https://unum.cloud
+[gdb13-hf]: https://unum.cloud
+[real-hf]: https://unum.cloud
+
+## TODO
+
+- JavaScript: Allow calling from "worker threads".
+- Rust: Allow passing a custom thread ID.
+- C# .NET bindings.
+
+## Integrations
+
+- [x] GPT-Cache.
+- [ ] Langchain.
+- [ ] Microsoft Semantic Kernel.
+- [ ] PyTorch.
+
+## Citations
+
+```txt
+@software{Vardanian_USearch_2022,
+doi = {10.5281/zenodo.7949416},
+author = {Vardanian, Ash},
+title = {{USearch by Unum Cloud}},
+url = {https://github.com/unum-cloud/usearch},
+version = {0.13.0},
+year = {2022}
+month = jun,
+}
+```
+
 ---
 
 Check [that](https://github.com/ashvardanian/image-search) and [other](https://github.com/unum-cloud/examples) examples on our corporate GitHub 🤗

package/binding.gyp

@@ -8,7 +8,6 @@
             "include_dirs": [
                 "<!@(node -p \"require('node-addon-api').include\")",
                 "include",
-                "src",
                 "fp16/include",
                 "robin-map/include",
                 "simsimd/include"

package/build.gradle

@@ -45,7 +45,7 @@ model {
                         include "**/*.cpp"
                     }
                     exportedHeaders {
-                        srcDirs "include", "src", "fp16/include", "robin-map/include", "simsimd/include", "${Jvm.current().javaHome}/include"
+                        srcDirs "include", "fp16/include", "robin-map/include", "simsimd/include", "${Jvm.current().javaHome}/include"
                     }
                 }
             }

package/build.rs

@@ -5,7 +5,6 @@ fn main() {
         .flag_if_supported("-Wno-unknown-pragmas")
         .include("include")
         .include("rust")
-        .include("src")
         .include("fp16/include")
         .include("robin-map/include")
         .include("simsimd/include")
@@ -14,6 +13,7 @@ fn main() {
     println!("cargo:rerun-if-changed=rust/lib.rs");
     println!("cargo:rerun-if-changed=rust/lib.cpp");
     println!("cargo:rerun-if-changed=rust/lib.hpp");
-    println!("cargo:rerun-if-changed=src/punned.hpp");
-    println!("cargo:rerun-if-changed=include/usearch/usearch.hpp");
+    println!("cargo:rerun-if-changed=include/index_punned_helpers.hpp");
+    println!("cargo:rerun-if-changed=include/index_punned_dense.hpp");
+    println!("cargo:rerun-if-changed=include/usearch/index.hpp");
 }

package/c/lib.cpp

@@ -1,4 +1,4 @@
-#include "../src/punned.hpp"
+#include "../src/index_punned_dense.hpp"
 
 extern "C" {
 #include "usearch.h"
@@ -9,13 +9,36 @@ using namespace unum;
 
 using label_t = int;
 using distance_t = float;
-using punned_index_t = punned_gt<label_t>;
+using punned_index_t = index_punned_dense_gt<label_t>;
 using span_t = span_gt<float>;
 
-void* usearch_new(                                        //
-    usearch_metric_t metric, usearch_accuracy_t accuracy, //
-    int dimensions, int capacity, int connectivity,       //
-    int expansion_add, int expansion_search) {
+metric_kind_t to_native_metric(usearch_metric_kind_t kind) {
+    switch (kind) {
+    case usearch_metric_ip_k: return metric_kind_t::ip_k;
+    case usearch_metric_l2sq_k: return metric_kind_t::l2sq_k;
+    case usearch_metric_cos_k: return metric_kind_t::cos_k;
+    case usearch_metric_haversine_k: return metric_kind_t::haversine_k;
+    case usearch_metric_pearson_k: return metric_kind_t::pearson_k;
+    case usearch_metric_jaccard_k: return metric_kind_t::jaccard_k;
+    case usearch_metric_hamming_k: return metric_kind_t::hamming_k;
+    case usearch_metric_tanimoto_k: return metric_kind_t::tanimoto_k;
+    case usearch_metric_sorensen_k: return metric_kind_t::sorensen_k;
+    default: return metric_kind_t::unknown_k;
+    }
+}
+
+scalar_kind_t to_native_scalar(usearch_scalar_kind_t kind) {
+    switch (kind) {
+    case usearch_scalar_f32_k: return scalar_kind_t::f32_k;
+    case usearch_scalar_f64_k: return scalar_kind_t::f64_k;
+    case usearch_scalar_f16_k: return scalar_kind_t::f16_k;
+    case usearch_scalar_f8_k: return scalar_kind_t::f8_k;
+    case usearch_scalar_b1_k: return scalar_kind_t::b1_k;
+    default: return scalar_kind_t::unknown_k;
+    }
+}
+
+void usearch_init(usearch_init_options_t* options, usearch_index_t* index, usearch_error_t* error) {
 
     try {
         config_t config;
@@ -37,8 +60,8 @@ void* usearch_new(                                        //
         // std::size_t bytes_to_alloc = sizeof(punned_index_t) + bytes_for_error_messages;
         // safe_punned_index_t* result = (safe_punned_index_t*)std::malloc(bytes_to_alloc);
 
-        accuracy_t accuracy = accuracy_from_name(accuracy_str, accuracy_len);
-        common_metric_kind_t metric_kind = common_metric_from_name(metric_str, metric_len);
+        scalar_kind_t accuracy = scalar_kind_from_name(accuracy_str, accuracy_len);
+        metric_kind_t metric_kind = metric_from_name(metric_str, metric_len);
         punned_index_t index = make_punned<punned_index_t>( //
             metric_kind, static_cast<std::size_t>(dimensions), accuracy, config);
 
@@ -49,7 +72,7 @@ void* usearch_new(                                        //
     return NULL;
 }
 
-void usearch_destroy(punned_index_t* index) { delete index; }
+void usearch_free(punned_index_t* index) { delete index; }
 
 char const* usearch_save(punned_index_t* index, char const* path) {
     try {