usearch 2.8.16 → 2.9.1

package/stringzilla/javascript/lib.c

@@ -0,0 +1,125 @@
+/**
+ *  @file       lib.c
+ *  @brief      JavaScript bindings for StringZilla.
+ *  @author     Ash Vardanian
+ *  @date       September 18, 2023
+ *
+ *  @copyright  Copyright (c) 2023
+ *  @see        NodeJS docs: https://nodejs.org/api/n-api.html
+ */
+#include <stdio.h>  // `printf` for debug builds
+#include <stdlib.h> // `malloc` to export strings into UTF-8
+
+#include <node_api.h> // `napi_*` functions
+
+#include <stringzilla/stringzilla.h> // `sz_*` functions
+
+napi_value indexOfAPI(napi_env env, napi_callback_info info) {
+    size_t argc = 2;
+    napi_value args[2];
+    napi_get_cb_info(env, info, &argc, args, NULL, NULL);
+
+    // Extract the C string from the JavaScript string for haystack and needle
+    sz_string_view_t haystack = {NULL, 0};
+    sz_string_view_t needle = {NULL, 0};
+
+    // For haystack
+    napi_get_value_string_utf8(env, args[0], NULL, 0, (size_t *)&haystack.length);
+    haystack.start = malloc(haystack.length + 1);
+    napi_get_value_string_utf8(env, args[0], (char *)haystack.start, haystack.length + 1, (size_t *)&haystack.length);
+
+    // For needle
+    napi_get_value_string_utf8(env, args[1], NULL, 0, (size_t *)&needle.length);
+    needle.start = malloc(needle.length + 1);
+    napi_get_value_string_utf8(env, args[1], (char *)needle.start, needle.length + 1, (size_t *)&needle.length);
+
+    // Convert the result to JavaScript BigInt and return
+    napi_value js_result;
+    if (needle.length == 0) { napi_create_bigint_int64(env, 0, &js_result); }
+    else {
+        sz_cptr_t result = sz_find(haystack.start, haystack.length, needle.start, needle.length);
+
+        // In JavaScript, if `indexOf` is unable to indexOf the specified value, then it should return -1
+        if (result == NULL) { napi_create_bigint_int64(env, -1, &js_result); }
+        else { napi_create_bigint_uint64(env, result - haystack.start, &js_result); }
+    }
+
+    // Cleanup
+    free((void *)haystack.start);
+    free((void *)needle.start);
+    return js_result;
+}
+
+napi_value countAPI(napi_env env, napi_callback_info info) {
+    size_t argc = 3;
+    napi_value args[3];
+    napi_get_cb_info(env, info, &argc, args, NULL, NULL);
+
+    // Extract the C string from the JavaScript string for haystack and needle
+    sz_string_view_t haystack = {NULL, 0};
+    sz_string_view_t needle = {NULL, 0};
+
+    // For haystack
+    napi_get_value_string_utf8(env, args[0], NULL, 0, (size_t *)&haystack.length);
+    haystack.start = malloc(haystack.length + 1);
+    napi_get_value_string_utf8(env, args[0], (char *)haystack.start, haystack.length + 1, (size_t *)&haystack.length);
+
+    // For needle
+    napi_get_value_string_utf8(env, args[1], NULL, 0, (size_t *)&needle.length);
+    needle.start = malloc(needle.length + 1);
+    napi_get_value_string_utf8(env, args[1], (char *)needle.start, needle.length + 1, (size_t *)&needle.length);
+
+    bool overlap = false;
+    if (argc > 2) { napi_get_value_bool(env, args[2], &overlap); }
+
+    void const *haystack_start = haystack.start, *needle_start = needle.start;
+
+    size_t count = 0;
+    if (needle.length == 0 || haystack.length == 0 || haystack.length < needle.length) { count = 0; }
+    else if (overlap) {
+        while (haystack.length) {
+            sz_cptr_t ptr = sz_find(haystack.start, haystack.length, needle.start, needle.length);
+            sz_bool_t found = ptr != NULL;
+            sz_size_t offset = found ? (sz_size_t)(ptr - haystack.start) : haystack.length;
+            count += found;
+            haystack.start += offset + found;
+            haystack.length -= offset + found;
+        }
+    }
+    else {
+        while (haystack.length) {
+            sz_cptr_t ptr = sz_find(haystack.start, haystack.length, needle.start, needle.length);
+            sz_bool_t found = ptr != NULL;
+            sz_size_t offset = found ? (sz_size_t)(ptr - haystack.start) : haystack.length;
+            count += found;
+            haystack.start += offset + needle.length;
+            haystack.length -= offset + needle.length * found;
+        }
+    }
+
+    // Cleanup
+    free((void *)haystack_start);
+    free((void *)needle_start);
+
+    // Convert the `count` to JavaScript `BigInt` and return
+    napi_value js_count;
+    napi_create_bigint_uint64(env, count, &js_count);
+
+    return js_count;
+}
+
+napi_value Init(napi_env env, napi_value exports) {
+
+    // Define an array of property descriptors
+    napi_property_descriptor findDesc = {"indexOf", 0, indexOfAPI, 0, 0, 0, napi_default, 0};
+    napi_property_descriptor countDesc = {"count", 0, countAPI, 0, 0, 0, napi_default, 0};
+    napi_property_descriptor properties[] = {findDesc, countDesc};
+
+    // Define the properties on the `exports` object
+    size_t propertyCount = sizeof(properties) / sizeof(properties[0]);
+    napi_define_properties(env, exports, propertyCount, properties);
+
+    return exports;
+}
+
+NAPI_MODULE(NODE_GYP_MODULE_NAME, Init)

package/stringzilla/javascript/stringzilla.js

@@ -0,0 +1,22 @@
+const compiled = require('bindings')('stringzilla');
+
+module.exports = {
+    /**
+     * Searches for a short string in a long one.
+     * 
+     * @param {string} haystack 
+     * @param {string} needle 
+     * @returns {bigint}
+     */
+    find: compiled.find,
+
+    /**
+     * Searches for a substring in a larger string.
+     * 
+     * @param {string} haystack 
+     * @param {string} needle 
+     * @param {boolean} overlap 
+     * @returns {bigint}
+     */
+    count: compiled.count
+};

package/stringzilla/package-ci.json

@@ -0,0 +1,10 @@
+{
+    "name": "stringzilla-ci",
+    "version": "1.0.0",
+    "devDependencies": {
+        "@semantic-release/exec": "^6.0.3",
+        "@semantic-release/git": "^10.0.1",
+        "conventional-changelog-eslint": "^3.0.9",
+        "semantic-release": "^21.1.2"
+    }
+}

package/stringzilla/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "stringzilla",
+  "version": "3.2.0",
+  "description": "SIMD-accelerated string search, sort, hashes, fingerprints, & edit distances",
+  "author": "Ash Vardanian",
+  "license": "Apache 2.0",
+  "main": "javascript/stringzilla.js",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ashvardanian/stringzilla.git"
+  },
+  "gypfile": true,
+  "engines": {
+    "node": "~10 >=10.20 || >=12.17"
+  },
+  "dependencies": {
+    "@types/node": "^20.4.5",
+    "bindings": "~1.2.1",
+    "node-addon-api": "^3.0.0"
+  },
+  "scripts": {
+    "test": "node --test ./scripts/test.js"
+  },
+  "devDependencies": {
+    "@semantic-release/exec": "^6.0.3",
+    "@semantic-release/git": "^10.0.1",
+    "conventional-changelog-eslint": "^3.0.9",
+    "semantic-release": "^21.1.2",
+    "typescript": "^5.1.6"
+  }
+}

package/stringzilla/rust/lib.rs

@@ -0,0 +1,362 @@
+#![cfg_attr(not(test), no_std)]
+
+use core::ffi::c_void;
+
+// Import the functions from the StringZilla C library.
+extern "C" {
+    fn sz_find(
+        haystack: *const c_void,
+        haystack_length: usize,
+        needle: *const c_void,
+        needle_length: usize,
+    ) -> *const c_void;
+
+    fn sz_rfind(
+        haystack: *const c_void,
+        haystack_length: usize,
+        needle: *const c_void,
+        needle_length: usize,
+    ) -> *const c_void;
+
+    fn sz_find_char_from(
+        haystack: *const c_void,
+        haystack_length: usize,
+        needle: *const c_void,
+        needle_length: usize,
+    ) -> *const c_void;
+
+    fn sz_rfind_char_from(
+        haystack: *const c_void,
+        haystack_length: usize,
+        needle: *const c_void,
+        needle_length: usize,
+    ) -> *const c_void;
+
+    fn sz_find_char_not_from(
+        haystack: *const c_void,
+        haystack_length: usize,
+        needle: *const c_void,
+        needle_length: usize,
+    ) -> *const c_void;
+
+    fn sz_rfind_char_not_from(
+        haystack: *const c_void,
+        haystack_length: usize,
+        needle: *const c_void,
+        needle_length: usize,
+    ) -> *const c_void;
+
+    fn sz_edit_distance(
+        haystack1: *const c_void,
+        haystack1_length: usize,
+        haystack2: *const c_void,
+        haystack2_length: usize,
+        bound: usize,
+        allocator: *const c_void,
+    ) -> usize;
+
+    fn sz_alignment_score(
+        haystack1: *const c_void,
+        haystack1_length: usize,
+        haystack2: *const c_void,
+        haystack2_length: usize,
+        matrix: *const c_void,
+        gap: i8,
+        allocator: *const c_void,
+    ) -> isize;
+}
+
+/// The [StringZilla] trait provides a collection of string searching and manipulation functionalities.
+pub trait StringZilla<N>
+where
+    N: AsRef<[u8]>,
+{
+    /// Locates first matching substring. Equivalent to `memmem(haystack, h_length, needle, n_length)` in LibC. Similar
+    /// to `strstr(haystack, needle)` in LibC, but requires known length.
+    fn sz_find(&self, needle: N) -> Option<usize>;
+
+    /// Locates the last matching substring.
+    fn sz_rfind(&self, needle: N) -> Option<usize>;
+
+    /// Finds the first character in the haystack, that is present in the needle.
+    fn sz_find_char_from(&self, needles: N) -> Option<usize>;
+
+    /// Finds the last character in the haystack, that is present in the needle.
+    fn sz_rfind_char_from(&self, needles: N) -> Option<usize>;
+
+    /// Finds the first character in the haystack, that is __not__ present in the needle.
+    fn sz_find_char_not_from(&self, needles: N) -> Option<usize>;
+
+    /// Finds the last character in the haystack, that is __not__ present in the needle.
+    fn sz_rfind_char_not_from(&self, needles: N) -> Option<usize>;
+
+    /// Computes the Levenshtein edit-distance between two strings using the Wagner-Fisher algorithm.
+    /// Similar to the Needleman-Wunsch alignment algorithm. Often used in fuzzy string matching.
+    fn sz_edit_distance(&self, needle: N) -> usize;
+
+    /// Computes Needleman–Wunsch alignment score for two strings. Often used in bioinformatics and cheminformatics.
+    /// Similar to the Levenshtein edit-distance, parameterized for gap and substitution penalties.
+    fn sz_alignment_score(&self, needle: N, matrix: [[i8; 256]; 256], gap: i8) -> isize;
+}
+
+impl<T, N> StringZilla<N> for T
+where
+    T: AsRef<[u8]>,
+    N: AsRef<[u8]>,
+{
+    fn sz_find(&self, needle: N) -> Option<usize> {
+        let haystack_ref = self.as_ref();
+        let needle_ref = needle.as_ref();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let haystack_length = haystack_ref.len();
+        let needle_pointer = needle_ref.as_ptr() as _;
+        let needle_length = needle_ref.len();
+        let result = unsafe {
+            sz_find(
+                haystack_pointer,
+                haystack_length,
+                needle_pointer,
+                needle_length,
+            )
+        };
+
+        if result.is_null() {
+            None
+        } else {
+            Some(unsafe { result.offset_from(haystack_pointer) } as usize)
+        }
+    }
+
+    fn sz_rfind(&self, needle: N) -> Option<usize> {
+        let haystack_ref = self.as_ref();
+        let needle_ref = needle.as_ref();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let haystack_length = haystack_ref.len();
+        let needle_pointer = needle_ref.as_ptr() as _;
+        let needle_length = needle_ref.len();
+        let result = unsafe {
+            sz_rfind(
+                haystack_pointer,
+                haystack_length,
+                needle_pointer,
+                needle_length,
+            )
+        };
+
+        if result.is_null() {
+            None
+        } else {
+            Some(unsafe { result.offset_from(haystack_pointer) } as usize)
+        }
+    }
+
+    fn sz_find_char_from(&self, needles: N) -> Option<usize> {
+        let haystack_ref = self.as_ref();
+        let needles_ref = needles.as_ref();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let haystack_length = haystack_ref.len();
+        let needles_pointer = needles_ref.as_ptr() as _;
+        let needles_length = needles_ref.len();
+        let result = unsafe {
+            sz_find_char_from(
+                haystack_pointer,
+                haystack_length,
+                needles_pointer,
+                needles_length,
+            )
+        };
+        if result.is_null() {
+            None
+        } else {
+            Some(unsafe { result.offset_from(haystack_pointer) } as usize)
+        }
+    }
+
+    fn sz_rfind_char_from(&self, needles: N) -> Option<usize> {
+        let haystack_ref = self.as_ref();
+        let needles_ref = needles.as_ref();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let haystack_length = haystack_ref.len();
+        let needles_pointer = needles_ref.as_ptr() as _;
+        let needles_length = needles_ref.len();
+        let result = unsafe {
+            sz_rfind_char_from(
+                haystack_pointer,
+                haystack_length,
+                needles_pointer,
+                needles_length,
+            )
+        };
+        if result.is_null() {
+            None
+        } else {
+            Some(unsafe { result.offset_from(haystack_pointer) } as usize)
+        }
+    }
+
+    fn sz_find_char_not_from(&self, needles: N) -> Option<usize> {
+        let haystack_ref = self.as_ref();
+        let needles_ref = needles.as_ref();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let haystack_length = haystack_ref.len();
+        let needles_pointer = needles_ref.as_ptr() as _;
+        let needles_length = needles_ref.len();
+        let result = unsafe {
+            sz_find_char_not_from(
+                haystack_pointer,
+                haystack_length,
+                needles_pointer,
+                needles_length,
+            )
+        };
+        if result.is_null() {
+            None
+        } else {
+            Some(unsafe { result.offset_from(haystack_pointer) } as usize)
+        }
+    }
+
+    fn sz_rfind_char_not_from(&self, needles: N) -> Option<usize> {
+        let haystack_ref = self.as_ref();
+        let needles_ref = needles.as_ref();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let haystack_length = haystack_ref.len();
+        let needles_pointer = needles_ref.as_ptr() as _;
+        let needles_length = needles_ref.len();
+        let result = unsafe {
+            sz_rfind_char_not_from(
+                haystack_pointer,
+                haystack_length,
+                needles_pointer,
+                needles_length,
+            )
+        };
+        if result.is_null() {
+            None
+        } else {
+            Some(unsafe { result.offset_from(haystack_pointer) } as usize)
+        }
+    }
+
+    fn sz_edit_distance(&self, needle: N) -> usize {
+        let haystack_ref = self.as_ref();
+        let needle_ref = needle.as_ref();
+        let haystack_length = haystack_ref.len();
+        let needle_length = needle_ref.len();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let needle_pointer = needle_ref.as_ptr() as _;
+        unsafe {
+            sz_edit_distance(
+                haystack_pointer,
+                haystack_length,
+                needle_pointer,
+                needle_length,
+                // Upper bound on the distance, that allows us to exit early. If zero is
+                // passed, the maximum possible distance will be equal to the length of
+                // the longer input.
+                0,
+                // Uses the default allocator
+                core::ptr::null(),
+            )
+        }
+    }
+
+    fn sz_alignment_score(&self, needle: N, matrix: [[i8; 256]; 256], gap: i8) -> isize {
+        let haystack_ref = self.as_ref();
+        let needle_ref = needle.as_ref();
+        let haystack_length = haystack_ref.len();
+        let needle_length = needle_ref.len();
+        let haystack_pointer = haystack_ref.as_ptr() as _;
+        let needle_pointer = needle_ref.as_ptr() as _;
+        unsafe {
+            sz_alignment_score(
+                haystack_pointer,
+                haystack_length,
+                needle_pointer,
+                needle_length,
+                matrix.as_ptr() as _,
+                gap,
+                core::ptr::null(),
+            )
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::borrow::Cow;
+
+    use crate::StringZilla;
+
+    fn unary_substitution_costs() -> [[i8; 256]; 256] {
+        let mut result = [[0; 256]; 256];
+
+        for i in 0..256 {
+            for j in 0..256 {
+                result[i][j] = if i == j { 0 } else { -1 };
+            }
+        }
+
+        result
+    }
+
+    #[test]
+    fn levenshtein() {
+        assert_eq!("hello".sz_edit_distance("hell"), 1);
+        assert_eq!("hello".sz_edit_distance("hell"), 1);
+        assert_eq!("abc".sz_edit_distance(""), 3);
+        assert_eq!("abc".sz_edit_distance("ac"), 1);
+        assert_eq!("abc".sz_edit_distance("a_bc"), 1);
+        assert_eq!("abc".sz_edit_distance("adc"), 1);
+        assert_eq!("ggbuzgjux{}l".sz_edit_distance("gbuzgjux{}l"), 1);
+        assert_eq!("abcdefgABCDEFG".sz_edit_distance("ABCDEFGabcdefg"), 14);
+        assert_eq!("fitting".sz_edit_distance("kitty"), 4);
+        assert_eq!("smitten".sz_edit_distance("mitten"), 1);
+    }
+
+    #[test]
+    fn needleman() {
+        let costs_vector = unary_substitution_costs();
+        assert_eq!("listen".sz_alignment_score("silent", costs_vector, -1), -4);
+        assert_eq!(
+            "abcdefgABCDEFG".sz_alignment_score("ABCDEFGabcdefg", costs_vector, -1),
+            -14
+        );
+        assert_eq!("hello".sz_alignment_score("hello", costs_vector, -1), 0);
+        assert_eq!("hello".sz_alignment_score("hell", costs_vector, -1), -1);
+    }
+
+    #[test]
+    fn basics() {
+        let my_string: String = String::from("Hello, world!");
+        let my_str: &str = my_string.as_str();
+        let my_cow_str: Cow<'_, str> = Cow::from(&my_string);
+
+        // Use the generic function with a String
+        assert_eq!(my_string.sz_find("world"), Some(7));
+        assert_eq!(my_string.sz_rfind("world"), Some(7));
+        assert_eq!(my_string.sz_find_char_from("world"), Some(2));
+        assert_eq!(my_string.sz_rfind_char_from("world"), Some(11));
+        assert_eq!(my_string.sz_find_char_not_from("world"), Some(0));
+        assert_eq!(my_string.sz_rfind_char_not_from("world"), Some(12));
+
+        // Use the generic function with a &str
+        assert_eq!(my_str.sz_find("world"), Some(7));
+        assert_eq!(my_str.sz_find("world"), Some(7));
+        assert_eq!(my_str.sz_find_char_from("world"), Some(2));
+        assert_eq!(my_str.sz_rfind_char_from("world"), Some(11));
+        assert_eq!(my_str.sz_find_char_not_from("world"), Some(0));
+        assert_eq!(my_str.sz_rfind_char_not_from("world"), Some(12));
+
+        // Use the generic function with a Cow<'_, str>
+        assert_eq!(my_cow_str.as_ref().sz_find("world"), Some(7));
+        assert_eq!(my_cow_str.as_ref().sz_find("world"), Some(7));
+        assert_eq!(my_cow_str.as_ref().sz_find_char_from("world"), Some(2));
+        assert_eq!(my_cow_str.as_ref().sz_rfind_char_from("world"), Some(11));
+        assert_eq!(my_cow_str.as_ref().sz_find_char_not_from("world"), Some(0));
+        assert_eq!(
+            my_cow_str.as_ref().sz_rfind_char_not_from("world"),
+            Some(12)
+        );
+    }
+}