usearch 1.2.2 → 2.0.0

package/.github/workflows/prerelease.yml

@@ -17,6 +17,32 @@ permissions:
 
 jobs:
 
+  test_c:
+    name: Test C
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: main-dev
+    - run: git submodule update --init --recursive
+    - name: Prepare Environment
+      run: |
+        sudo apt update && 
+        sudo apt install -y cmake g++-12 build-essential libjemalloc-dev
+    - name: Build
+      run: |
+        cmake -B ./build_release \
+        -DCMAKE_CXX_COMPILER="g++-12" \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DUSEARCH_BUILD_TEST=1 \
+        -DUSEARCH_USE_OPENMP=1 \
+        -DUSEARCH_USE_SIMSIMD=1 \
+        -DUSEARCH_USE_JEMALLOC=1 \
+        -DUSEARCH_BUILD_BENCHMARK=0 &&
+        make -C ./build_release -j
+    - name: Run tests
+      run: ./build_release/test_c
+
   test_cpp:
     name: Test C++
     runs-on: ubuntu-latest
@@ -148,6 +174,51 @@ jobs:
     - name: Run tests
       run: swift test
 
+  test_csharp:
+    name: Test C#
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        dotnet-version: ['7.0.x']
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive
+      - name: Setup .NET ${{ matrix.dotnet-version }}
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: ${{ matrix.dotnet-version }}
+
+      - name: Ubuntu/macOS build native lib
+        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
+        run: ./build_lib.sh
+        shell: bash
+        working-directory: ${{ github.workspace }}/csharp/eng/
+
+      # Use cl.exe https://www.appveyor.com/docs/lang/cpp/#visual-studio-2017
+      # There is no support for c++11
+      - name: Windows MSVC build
+        if: matrix.os == 'windows-latest'
+        run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+          where cl
+          cl /EHsc /std:c++14 /O2 /LD /Fe:libusearch_c.dll lib.cpp /I. /I../include/ /I../fp16/include/
+        working-directory: ${{ github.workspace }}/c
+        shell: cmd
+
+      - name: Windows move dll to lib dir
+        if: matrix.os == 'windows-latest'
+        run: |
+          mkdir -p "${GITHUB_WORKSPACE}/csharp/lib/runtimes/win-x64/native"
+          mv "${GITHUB_WORKSPACE}/c/libusearch_c.dll" "${GITHUB_WORKSPACE}/csharp/lib/runtimes/win-x64/native"
+        shell: bash
+
+      - name: Build and test
+        run: dotnet test -c Release
+        shell: bash
+        working-directory: ${{ github.workspace }}/csharp/
+
   build_docker:
     name: Test Docker Image
     runs-on: ubuntu-latest

package/.github/workflows/release.yml

@@ -271,11 +271,86 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
 
+  build_csharp:
+    name: Build NuGet
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive
+
+      - name: Ubuntu/macOS build usearch shared obj
+        if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
+        run: ./build_lib.sh
+        working-directory: ${{ github.workspace }}/csharp/eng
+
+      - name: Windows MSVC build usearch dll
+        if: matrix.os == 'windows-latest'
+        run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
+          where cl
+          cl /EHsc /std:c++14 /O2 /LD /Fe:libusearch_c.dll lib.cpp /I. /I../include/ /I../fp16/include/
+        working-directory: ${{ github.workspace }}/c
+        shell: cmd
+
+      - name: Windows copy dll
+        if: matrix.os == 'windows-latest'
+        run: |
+          mkdir -p "${GITHUB_WORKSPACE}/csharp/lib/runtimes/win-x64/native"
+          cp "${GITHUB_WORKSPACE}/c/libusearch_c.dll" "${GITHUB_WORKSPACE}/csharp/lib/runtimes/win-x64/native"
+        shell: bash
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: usearch-libs
+          path: ${{ github.workspace }}/csharp/lib/**/*
+
+  publish_csharp:
+    name: Publish C#
+    runs-on: ubuntu-latest
+    needs: [build_csharp]
+    env:
+      USEARCH_LIBS: ${{ github.workspace }}/csharp/lib
+      NUGET_DIR: ${{ github.workspace }}/csharp/packages
+      # NUGET_SOURCE: https://apiint.nugettest.org/v3/index.json
+      NUGET_SOURCE: https://api.nuget.org/v3/index.json
+      SOLUTION_DIR: ${{ github.workspace }}/csharp
+    steps:
+      - uses: actions/checkout@v3
+      - run: git submodule update --init --recursive
+        
+      - name: Download usearch libs artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: usearch-libs
+          path: ${{ env.USEARCH_LIBS }}
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 7.0.x     
+
+      - name: Pack project
+        run: |
+          VERSION_CONTENT=$(< VERSION)
+          dotnet pack "${{ env.SOLUTION_DIR }}" -c Release -p:Version="$VERSION_CONTENT" --output ${{ env.NUGET_DIR }}
+        working-directory: ${{ github.workspace }}
+
+      - name: Publish NuGet packages
+        run: |
+          dotnet nuget push "${{  env.NUGET_DIR }}/*.nupkg" --api-key "${{ secrets.NUGET_APIKEY }}" --source "${{ env.NUGET_SOURCE }}"
+
   build_docs:
     name: Build Docs
     runs-on: ubuntu-22.04
     if: ${{ always() }}
-    needs: [publish_python, publish_javascript, publish_rust, publish_java, publish_swift, publish_docker]
+    needs: [publish_python, publish_javascript, publish_rust, publish_java, publish_swift, publish_docker, publish_csharp]
     permissions:
       contents: write
     steps:
@@ -287,7 +362,7 @@ jobs:
         run: |
               sudo apt update && 
               sudo apt install -y doxygen graphviz dia git && 
-              pip install sphinx==7.1.2 sphinx-js breathe furo m2r2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery && 
+              pip install sphinx==5.3.0 sphinx-js==3.2.1 breathe==4.35.0 furo==2023.3.27 m2r2==0.3.3.post2 sphinxcontrib-googleanalytics==0.2.dev20220708 sphinxcontrib-jquery==4.1 && 
               npm install -g jsdoc
       - name: Install USearch from PyPi
         run: pip install usearch

package/.github/workflows/update_version.sh

@@ -10,4 +10,5 @@ echo $1 > VERSION &&
     sed -i "s/^\(#define USEARCH_VERSION_PATCH \).*/\1$(echo "$1" | cut -d. -f3)/" ./include/usearch/index.hpp &&
     sed -i "s/<version>[0-9]\+\.[0-9]\+\.[0-9]\+/<version>$1/" README.md &&
     sed -i "s/version = {[0-9]\+\.[0-9]\+\.[0-9]\+}/version = {$1}/" README.md &&
+    sed -i "s/>[0-9]\+\.[0-9]\+\.[0-9]\+<\/Version>/>$1<\/Version>/" ./csharp/nuget/nuget-package.props &&
     sed -i "s/version=\".*\"/version=\"$1\"/" wasmer.toml

package/.gitignore

@@ -31,6 +31,9 @@ __pycache__
 dist/
 wheelhouse/
 
+# Python Environments
+venv/
+
 # JS wrappers
 package-lock.json
 node_modules/
@@ -39,7 +42,7 @@ node_modules/
 Cargo.lock
 target/
 
-# Java binings
+# Java bindings
 .gradle
 *.class
 
@@ -47,6 +50,12 @@ target/
 .build
 .swiftpm
 
+# C# builds
+csharp/**/[Bb]in/
+csharp/**/[Oo]bj/
+csharp/lib
+csharp/packages
+
 # Prerequisites
 *.d
 
@@ -82,3 +91,4 @@ target/
 *.exe
 *.out
 *.app
+

package/.vscode/settings.json

@@ -111,6 +111,7 @@
         "Availible",
         "bidict",
         "BLAS",
+        "Cdecl",
         "cflags",
         "citerator",
         "downcasted",
@@ -126,6 +127,7 @@
         "hnswlib",
         "ibin",
         "jaccard",
+        "libusearch",
         "longlong",
         "memmap",
         "Napi",
@@ -146,7 +148,8 @@
         "uninitialize",
         "unumusearch",
         "usearch",
-        "usecases"
+        "usecases",
+        "Xunit"
     ],
     "autoDocstring.docstringFormat": "sphinx",
     "java.configuration.updateBuildConfiguration": "interactive",
@@ -163,4 +166,5 @@
         "editor.formatOnSave": true,
         "editor.defaultFormatter": "golang.go"
     },
+    "dotnet.defaultSolution": "csharp/Cloud.Unum.USearch.sln",
 }

package/CMakeLists.txt

@@ -113,6 +113,7 @@ if(USEARCH_INSTALL)
 endif()
 
 if(${USEARCH_BUILD_TEST} OR ${USEARCH_BUILD_BENCHMARK})
+    add_subdirectory(c)
     add_subdirectory(cpp)
 endif()
 

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "usearch"
-version = "1.2.2"
+version = "2.0.0"
 authors = ["Ash Vardanian <1983160+ashvardanian@users.noreply.github.com>"]
 description = "Smaller & Faster Single-File Vector Search Engine from Unum"
 edition = "2021"

package/README.md

@@ -109,10 +109,10 @@ Comparing the performance of FAISS against USearch on 1 Million 96-dimensional v
 
 |              | FAISS, `f32` | USearch, `f32` | USearch, `f16` |     USearch, `i8` |
 | :----------- | -----------: | -------------: | -------------: | ----------------: |
-| Batch Insert |       16 K/s |         73 K/s |        100 K/s | 104 K/s **+550%** |
-| Batch Search |       82 K/s |        103 K/s |        113 K/s |  134 K/s **+63%** |
-| Bulk Insert  |       76 K/s |        105 K/s |        115 K/s | 202 K/s **+165%** |
-| Bulk Search  |      118 K/s |        174 K/s |        173 K/s | 304 K/s **+157%** |
+| Batch Insert |       16 K/s |         73 K/s |        100 K/s | 104 K/s __+550%__ |
+| Batch Search |       82 K/s |        103 K/s |        113 K/s |  134 K/s __+63%__ |
+| Bulk Insert  |       76 K/s |        105 K/s |        115 K/s | 202 K/s __+165%__ |
+| Bulk Search  |      118 K/s |        174 K/s |        173 K/s | 304 K/s __+157%__ |
 | Recall @ 10  |          99% |          99.2% |          99.1% |             99.2% |
 
 > HNSW was configured with identical hyper-parameters:
@@ -152,10 +152,16 @@ Instead, we have focused on high-precision arithmetic over low-precision downcas
 The same index, and `add` and `search` operations will automatically down-cast or up-cast between `f32_t`, `f16_t`, `f64_t`, and `i8_t` representations, even if the hardware doesn't natively support it.
 Continuing the topic of memory efficiency, we provide a `uint40_t` to allow collection with over 4B+ vectors without allocating 8 bytes for every neighbor reference in the proximity graph.
 
-## Serving `Index` from Disk
+## Serialization & Serving `Index` from Disk
 
-With USearch, you can serve indexes from external memory, enabling you to optimize your server choices for indexing speed and serving costs.
-This can result in **20x cost reduction** on AWS and other public clouds.
+USearch supports multiple forms of serialization:
+
+- Into a __file__ defined with a path.
+- Into a __stream__ defined with a callback, serializing or reconstructing incrementally.
+- Into a __buffer__ of fixed length, or a memory-mapped file, that supports random access.
+
+The latter allows you to serve indexes from external memory, enabling you to optimize your server choices for indexing speed and serving costs.
+This can result in __20x cost reduction__ on AWS and other public clouds.
 
 ```py
 index.save("index.usearch")
@@ -186,10 +192,10 @@ many_in_many: BatchMatches = search(vectors, vectors, 50, MetricKind.L2sq, exact
 ```
 
 By passing the `exact=True` argument, the system bypasses indexing altogether and performs a brute-force search through the entire dataset using SIMD-optimized similarity metrics from [SimSIMD](https://github.com/ashvardanian/simsimd).
-When compared to FAISS's `IndexFlatL2` in Google Colab, **[USearch may offer up to a 20x performance improvement](https://github.com/unum-cloud/usearch/issues/176#issuecomment-1666650778)**:
+When compared to FAISS's `IndexFlatL2` in Google Colab, __[USearch may offer up to a 20x performance improvement](https://github.com/unum-cloud/usearch/issues/176#issuecomment-1666650778)__:
 
-- `faiss.IndexFlatL2`: **55.3 ms**.
-- `usearch.index.search`: **2.54 ms**.
+- `faiss.IndexFlatL2`: __55.3 ms__.
+- `usearch.index.search`: __2.54 ms__.
 
 ## `Indexes` for Multi-Index Lookups
 
@@ -277,8 +283,10 @@ Broader functionality is ported per request.
 
 ### USearch + AI = Multi-Modal Semantic Search
 
+[![USearch Semantic Image Search](https://github.com/ashvardanian/usearch-images/raw/main/assets/usearch-images-slow.gif)](https://github.com/ashvardanian/usearch-images)
+
 AI has a growing number of applications, but one of the coolest classic ideas is to use it for Semantic Search.
-One can take an encoder model, like the multi-modal UForm, and a web-programming framework, like UCall, and build a text-to-image search platform in just 20 lines of Python.
+One can take an encoder model, like the multi-modal [UForm](https://github.com/unum-cloud/uform), and a web-programming framework, like UCall, and build a text-to-image search platform in just 20 lines of Python.
 
 ```python
 import ucall
@@ -308,13 +316,14 @@ def search(query: str) -> np.ndarray:
 server.run()
 ```
 
+A more complete [demo with Streamlit is available on GitHub](https://github.com/ashvardanian/usearch-images).
 We have pre-processed some commonly used datasets, cleaned the images, produced the vectors, and pre-built the index.
 
-| Dataset                                |            Modalities | Images |                              Download |
-| :------------------------------------- | --------------------: | -----: | ------------------------------------: |
-| [Unsplash 25K][unsplash-25k-origin]    | Images & Descriptions |   25 K | [HuggingFace / Unum][unsplash-25k-hf] |
-| [Conceptual Captions 3M][cc-3m-origin] | Images & Descriptions |    3 M |        [HuggingFace / Unum][cc-3m-hf] |
-| [Arxiv 2M][arxiv-2m-origin]            |    Titles & Abstracts |    2 M |     [HuggingFace / Unum][arxiv-2m-hf] |
+| Dataset                             |            Modalities | Images |                              Download |
+| :---------------------------------- | --------------------: | -----: | ------------------------------------: |
+| [Unsplash][unsplash-25k-origin]     | Images & Descriptions |   25 K | [HuggingFace / Unum][unsplash-25k-hf] |
+| [Conceptual Captions][cc-3m-origin] | Images & Descriptions |    3 M |        [HuggingFace / Unum][cc-3m-hf] |
+| [Arxiv][arxiv-2m-origin]            |    Titles & Abstracts |    2 M |     [HuggingFace / Unum][arxiv-2m-hf] |
 
 [unsplash-25k-origin]: https://github.com/unsplash/datasets
 [cc-3m-origin]: https://huggingface.co/datasets/conceptual_captions
@@ -358,15 +367,19 @@ matches = index.search(fingerprints, 10)
 
 ### USearch + POI Coordinates = GIS Applications... on iOS?
 
-With Objective-C and iOS bindings, USearch can be easily used in mobile applications
+[![USearch Maps with SwiftUI](https://github.com/ashvardanian/SwiftVectorSearch/raw/main/USearch+SwiftUI.gif)](https://github.com/ashvardanian/SwiftVectorSearch)
 
+With Objective-C and Swift iOS bindings, USearch can be easily used in mobile applications.
+The [SwiftVectorSearch](https://github.com/ashvardanian/SwiftVectorSearch) project illustrates how to build a dynamic, real-time search system on iOS.
+In this example, we use 2-dimensional vectors—encoded as latitude and longitude—to find the closest Points of Interest (POIs) on a map.
+The search is based on the Haversine distance metric, but can easily be extended to support high-dimensional vectors.
 
 ## Integrations
 
 - [x] GPTCache: [Python](https://github.com/zilliztech/GPTCache/releases/tag/0.1.29).
 - [x] LangChain: [Python](https://github.com/langchain-ai/langchain/releases/tag/v0.0.257) and [JavaScipt](https://github.com/hwchase17/langchainjs/releases/tag/0.0.125).
 - [x] ClickHouse: [C++](https://github.com/ClickHouse/ClickHouse/pull/53447).
-- [ ] Microsoft Semantic Kernel: [Python](https://github.com/microsoft/semantic-kernel/pull/2358) and C#.
+- [x] Microsoft Semantic Kernel: [Python](https://github.com/microsoft/semantic-kernel/releases/tag/python-0.3.9.dev) and C#.
 
 ## Citations
 

package/c/lib.cpp

@@ -48,18 +48,18 @@ add_result_t add_(index_dense_t* index, usearch_key_t key, void const* vector, s
     case scalar_kind_t::f32_k: return index->add(key, (f32_t const*)vector);
     case scalar_kind_t::f64_k: return index->add(key, (f64_t const*)vector);
     case scalar_kind_t::f16_k: return index->add(key, (f16_t const*)vector);
-    case scalar_kind_t::i8_k: return index->add(key, (i8_bits_t const*)vector);
+    case scalar_kind_t::i8_k: return index->add(key, (i8_t const*)vector);
     case scalar_kind_t::b1x8_k: return index->add(key, (b1x8_t const*)vector);
     default: return add_result_t{}.failed("Unknown scalar kind!");
     }
 }
 
-bool get_(index_dense_t* index, usearch_key_t key, size_t count, void* vector, scalar_kind_t kind) {
+std::size_t get_(index_dense_t* index, usearch_key_t key, size_t count, void* vector, scalar_kind_t kind) {
     switch (kind) {
     case scalar_kind_t::f32_k: return index->get(key, (f32_t*)vector, count);
     case scalar_kind_t::f64_k: return index->get(key, (f64_t*)vector, count);
     case scalar_kind_t::f16_k: return index->get(key, (f16_t*)vector, count);
-    case scalar_kind_t::i8_k: return index->get(key, (i8_bits_t*)vector, count);
+    case scalar_kind_t::i8_k: return index->get(key, (i8_t*)vector, count);
     case scalar_kind_t::b1x8_k: return index->get(key, (b1x8_t*)vector, count);
     default: return search_result_t().failed("Unknown scalar kind!");
     }
@@ -70,7 +70,7 @@ search_result_t search_(index_dense_t* index, void const* vector, scalar_kind_t
     case scalar_kind_t::f32_k: return index->search((f32_t const*)vector, n);
     case scalar_kind_t::f64_k: return index->search((f64_t const*)vector, n);
     case scalar_kind_t::f16_k: return index->search((f16_t const*)vector, n);
-    case scalar_kind_t::i8_k: return index->search((i8_bits_t const*)vector, n);
+    case scalar_kind_t::i8_k: return index->search((i8_t const*)vector, n);
     case scalar_kind_t::b1x8_k: return index->search((b1x8_t const*)vector, n);
     default: return search_result_t().failed("Unknown scalar kind!");
     }
@@ -103,6 +103,11 @@ USEARCH_EXPORT void usearch_free(usearch_index_t index, usearch_error_t*) {
     delete reinterpret_cast<index_dense_t*>(index);
 }
 
+USEARCH_EXPORT size_t usearch_serialized_length(usearch_index_t index, usearch_error_t*) {
+    assert(index);
+    return reinterpret_cast<index_dense_t*>(index)->serialized_length();
+}
+
 USEARCH_EXPORT void usearch_save(usearch_index_t index, char const* path, usearch_error_t* error) {
 
     assert(index && path && error);
@@ -127,6 +132,35 @@ USEARCH_EXPORT void usearch_view(usearch_index_t index, char const* path, usearc
         *error = result.error.release();
 }
 
+USEARCH_EXPORT void usearch_save_buffer(usearch_index_t index, void* buffer, size_t length, usearch_error_t* error) {
+
+    assert(index && buffer && length && error);
+    memory_mapped_file_t memory_map((byte_t*)buffer, length);
+    serialization_result_t result = reinterpret_cast<index_dense_t*>(index)->save(std::move(memory_map));
+    if (!result)
+        *error = result.error.release();
+}
+
+USEARCH_EXPORT void usearch_load_buffer(usearch_index_t index, void const* buffer, size_t length,
+                                        usearch_error_t* error) {
+
+    assert(index && buffer && length && error);
+    memory_mapped_file_t memory_map((byte_t*)buffer, length);
+    serialization_result_t result = reinterpret_cast<index_dense_t*>(index)->load(std::move(memory_map));
+    if (!result)
+        *error = result.error.release();
+}
+
+USEARCH_EXPORT void usearch_view_buffer(usearch_index_t index, void const* buffer, size_t length,
+                                        usearch_error_t* error) {
+
+    assert(index && buffer && length && error);
+    memory_mapped_file_t memory_map((byte_t*)buffer, length);
+    serialization_result_t result = reinterpret_cast<index_dense_t*>(index)->view(std::move(memory_map));
+    if (!result)
+        *error = result.error.release();
+}
+
 USEARCH_EXPORT size_t usearch_size(usearch_index_t index, usearch_error_t*) { //
     return reinterpret_cast<index_dense_t*>(index)->size();
 }