usearch 0.1.9 → 0.1.10

package/src/bench.cpp

@@ -26,6 +26,8 @@
 using namespace unum::usearch;
 using namespace unum;
 
+using vector_id_t = std::uint32_t;
+
 template <typename element_at>
 std::size_t offset_of(element_at const* begin, element_at const* end, element_at v) noexcept {
     auto iterator = begin;
@@ -344,6 +346,34 @@ void run_big_or_small(dataset_at& dataset, args_t const& args, config_t config)
         run_type_punned<auto_index_gt<std::size_t, neighbor_id_at>>(dataset, args, config);
 }
 
+void report_expected_losses(persisted_dataset_gt<float, vector_id_t> const& dataset) {
+
+    auto vec1 = dataset.vector(0);
+    auto vec2 = dataset.query(0);
+    std::vector<f16_converted_t> vec1f16(dataset.dimensions());
+    std::vector<f16_converted_t> vec2f16(dataset.dimensions());
+    std::transform(vec1, vec1 + dataset.dimensions(), vec1f16.data(), [](float v) { return v; });
+    std::transform(vec2, vec2 + dataset.dimensions(), vec2f16.data(), [](float v) { return v; });
+    std::vector<i8q100_converted_t> vec1i8(dataset.dimensions());
+    std::vector<i8q100_converted_t> vec2i8(dataset.dimensions());
+    std::transform(vec1, vec1 + dataset.dimensions(), vec1i8.data(), [](float v) { return v; });
+    std::transform(vec2, vec2 + dataset.dimensions(), vec2i8.data(), [](float v) { return v; });
+
+#if 0
+    auto ip_default = ip_gt<float>{}(vec1, vec2, dataset.dimensions());
+    auto ip_sve = 1.f - simsimd_dot_f32sve(vec1, vec2, dataset.dimensions());
+    auto ip_neon = 1.f - simsimd_dot_f32x4neon(vec1, vec2, dataset.dimensions());
+    auto ip_f16 = ip_gt<f16_converted_t>{}(vec1f16.data(), vec2f16.data(), dataset.dimensions());
+    auto ip_f16sve = 1.f - simsimd_dot_f16sve((simsimd_f16_t const*)vec1f16.data(),
+                                              (simsimd_f16_t const*)vec2f16.data(), dataset.dimensions());
+    auto ip_f16neon = 1.f - simsimd_dot_f16x8neon((simsimd_f16_t const*)vec1f16.data(),
+                                                  (simsimd_f16_t const*)vec2f16.data(), dataset.dimensions());
+    auto ip_i8 = ip_i8q100_converted_t{}(vec1i8.data(), vec2i8.data(), dataset.dimensions());
+
+    exit(0);
+#endif
+}
+
 int main(int argc, char** argv) {
 
     // Print backtrace if something goes wrong.
@@ -393,8 +423,6 @@ int main(int argc, char** argv) {
     std::printf("-- Query vectors path: %s\n", args.path_queries.c_str());
     std::printf("-- Ground truth neighbors path: %s\n", args.path_neighbors.c_str());
 
-    using vector_id_t = std::uint32_t;
-
     persisted_dataset_gt<float, vector_id_t> dataset{
         args.path_vectors.c_str(),
         args.path_queries.c_str(),

package/wolfram/CMakeLists.txt

@@ -0,0 +1,17 @@
+
+if(NOT WOLFRAM_PATH)
+    set(WOLFRAM_PATH "/usr/local/Wolfram/Mathematica")
+endif()
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${WOLFRAM_PATH}/SystemFiles/Links/WSTP/DeveloperKit/Linux-x86-64/CompilerAdditions/")
+include(WSTP)
+include_directories("${WOLFRAM_PATH}/SystemFiles/IncludeFiles/C/")
+link_directories("${WOLFRAM_PATH}/SystemFiles/Libraries/Linux-x86-64/")
+
+add_library(usearchWFM SHARED lib.cpp)
+
+# Link against the Wolfram LibraryLink library
+target_link_libraries(usearchWFM WSTP::STATIC_LIBRARY)
+
+# Set the output name of the library to be compatible with Wolfram Language
+set_target_properties(usearchWFM PROPERTIES PREFIX "" OUTPUT_NAME "usearchWFM")

package/wolfram/lib.cpp

@@ -1,21 +1,156 @@
 #include "WolframLibrary.h"
+#include "advanced.hpp"
 
-EXTERN_C DLLEXPORT mint WolframLibrary_getVersion() { return WolframLibraryVersion; }
-EXTERN_C DLLEXPORT int WolframLibrary_initialize(WolframLibraryData libData) { return 0; }
+using namespace unum::usearch;
+using distance_t = punned_distance_t;
+using native_index_t = auto_index_gt<int>;
+using span_t = span_gt<float>;
+
+EXTERN_C DLLEXPORT int WolframLibrary_initialize(WolframLibraryData libData) { return LIBRARY_NO_ERROR; }
 EXTERN_C DLLEXPORT void WolframLibrary_uninitialize(WolframLibraryData libData) { return; }
 
-EXTERN_C DLLEXPORT int IndexInit(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+EXTERN_C DLLEXPORT int IndexCreate(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    config_t config;
+    char* accuracy_cstr = nullptr;
+    char* metric_cstr = nullptr;
+    try {
+        accuracy_cstr = MArgument_getUTF8String(Args[1]);
+        metric_cstr = MArgument_getUTF8String(Args[0]);
+        long dimensions = static_cast<std::size_t>(MArgument_getInteger(Args[2]));
+        config.max_elements = static_cast<std::size_t>(MArgument_getInteger(Args[3]));
+        config.connectivity = static_cast<std::size_t>(MArgument_getInteger(Args[4]));
+        config.expansion_add = static_cast<std::size_t>(MArgument_getInteger(Args[5]));
+        config.expansion_search = static_cast<std::size_t>(MArgument_getInteger(Args[6]));
+        config.max_threads_add = 4;
+        config.max_threads_search = 4;
+
+        accuracy_t accuracy = accuracy_from_name(accuracy_cstr, strlen(accuracy_cstr));
+        native_index_t index = index_from_name<native_index_t>( //
+            metric_cstr, strlen(metric_cstr), dimensions, accuracy, config);
+
+        native_index_t* result_ptr = new native_index_t(std::move(index));
+        MArgument_setInteger(Res, (long)result_ptr);
+
+    } catch (...) {
+        return LIBRARY_FUNCTION_ERROR;
+    }
+
+    libData->UTF8String_disown(accuracy_cstr);
+    libData->UTF8String_disown(metric_cstr);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexSave(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    char* path_cstr = nullptr;
+    native_index_t* c_ptr = (native_index_t*)MArgument_getUTF8String(Args[0]);
+    try {
+        path_cstr = MArgument_getUTF8String(Args[1]);
+        c_ptr->save(path_cstr);
+    } catch (...) {
+        return LIBRARY_FUNCTION_ERROR;
+    }
+    libData->UTF8String_disown(path_cstr);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexLoad(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    char* path_cstr = nullptr;
+    native_index_t* c_ptr = (native_index_t*)MArgument_getUTF8String(Args[0]);
+    try {
+        path_cstr = MArgument_getUTF8String(Args[1]);
+        c_ptr->load(path_cstr);
+    } catch (...) {
+        return LIBRARY_FUNCTION_ERROR;
+    }
+    libData->UTF8String_disown(path_cstr);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexView(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    char* path_cstr = nullptr;
+    native_index_t* c_ptr = (native_index_t*)MArgument_getUTF8String(Args[0]);
+    try {
+        path_cstr = MArgument_getUTF8String(Args[1]);
+        c_ptr->view(path_cstr);
+    } catch (...) {
+        return LIBRARY_FUNCTION_ERROR;
+    }
+    libData->UTF8String_disown(path_cstr);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexDestroy(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    delete (native_index_t*)MArgument_getUTF8String(Args[0]);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexSize(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    size_t res = ((native_index_t*)MArgument_getUTF8String(Args[0]))->size();
+    MArgument_setInteger(Res, res);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexConnectivity(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    size_t res = ((native_index_t*)MArgument_getUTF8String(Args[0]))->connectivity();
+    MArgument_setInteger(Res, res);
     return LIBRARY_NO_ERROR;
 }
 
-EXTERN_C DLLEXPORT int IndexReserve(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+EXTERN_C DLLEXPORT int IndexDimensions(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    size_t res = ((native_index_t*)MArgument_getUTF8String(Args[0]))->dimensions();
+    MArgument_setInteger(Res, res);
+    return LIBRARY_NO_ERROR;
+}
+
+EXTERN_C DLLEXPORT int IndexCapacity(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    size_t res = ((native_index_t*)MArgument_getUTF8String(Args[0]))->capacity();
+    MArgument_setInteger(Res, res);
     return LIBRARY_NO_ERROR;
 }
 
 EXTERN_C DLLEXPORT int IndexAdd(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    char* path_cstr = nullptr;
+    native_index_t* c_ptr = (native_index_t*)MArgument_getUTF8String(Args[0]);
+    float* vector_data = nullptr;
+    try {
+        int label = MArgument_getInteger(Args[1]);
+        MTensor tens = MArgument_getMTensor(Args[2]);
+        size_t len = libData->MTensor_getFlattenedLength(tens);
+        vector_data = (float*)libData->MTensor_getRealData(tens);
+        span_t vector_span = span_t{vector_data, len};
+        c_ptr->add(label, vector_span);
+    } catch (...) {
+        return LIBRARY_FUNCTION_ERROR;
+    }
     return LIBRARY_NO_ERROR;
 }
 
 EXTERN_C DLLEXPORT int IndexSearch(WolframLibraryData libData, mint Argc, MArgument* Args, MArgument Res) {
+    native_index_t* c_ptr = (native_index_t*)MArgument_getUTF8String(Args[0]);
+    MTensor matches;
+    mint dims[] = {1};
+    int wanted = MArgument_getInteger(Args[2]);
+    float* vector_data = nullptr;
+    int* matches_data = nullptr;
+    std::size_t found = 0;
+
+    try {
+        libData->MTensor_new(MType_Integer, 1, dims, &matches);
+
+        MTensor tens = MArgument_getMTensor(Args[1]);
+        size_t len = libData->MTensor_getFlattenedLength(tens);
+        vector_data = (float*)libData->MTensor_getRealData(tens);
+        span_t vector_span = span_t{vector_data, len};
+
+        matches_data = (int*)std::malloc(sizeof(int) * wanted);
+        found = c_ptr->search(vector_span, (size_t)wanted, matches_data, nullptr);
+        for (mint i = 0; i < found; i++)
+            libData->MTensor_setInteger(matches, &i, matches_data[i]);
+
+        MArgument_setMTensor(Res, matches);
+    } catch (...) {
+        return LIBRARY_FUNCTION_ERROR;
+    }
+    std::free(matches_data);
     return LIBRARY_NO_ERROR;
 }

package/wolfram/test.wls

@@ -0,0 +1,27 @@
+#!/usr/bin/env wolframscript
+
+IndexCreate = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexCreate", {"UTF8String", "UTF8String", Integer, Integer, Integer, Integer, Integer}, Integer]
+IndexSave = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexSave", {Integer, "UTF8String"}, "Void"]
+IndexLoad = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexLoad", {Integer, "UTF8String"}, "Void"]
+IndexView = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexView", {Integer, "UTF8String"}, "Void"]
+IndexDestroy = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexDestroy", {Integer}, "Void"]
+IndexSize = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexSize", {Integer}, Integer]
+IndexConnectivity = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexConnectivity", {Integer}, Integer]
+IndexDimensions = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexDimensions", {Integer}, Integer]
+IndexCapacity = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexCapacity", {Integer}, Integer]
+IndexAdd = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexAdd", {Integer, Integer, {Real, 1}}, "Void"]
+IndexSearch = LibraryFunctionLoad["wolfram/usearchWFM.so", "IndexSearch", {Integer, {Real, 1}, Integer}, {Integer, 1}]
+Print["Functions Loaded"]
+
+$index = IndexCreate["cos", "f16", 10, 10, 10, 10, 10]
+Print["New Index Address: ", $index]
+Print["Index Size: ", IndexSize[$index]]
+Print["Index Capacity: ", IndexCapacity[$index]]
+Print["Index Connectivity: ", IndexConnectivity[$index]]
+$arr = {1.0, 2.0, 3.0, 4.0, 5.0};
+IndexAdd[$index, 42, $arr]
+Print["Added Array: ", $arr]
+$res = IndexSearch[$index, $arr, 42]
+Print["Searched: ", $res]
+IndexDestroy[$index]
+Print["Index Destroyed"]

package/docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = ../build/docs
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)