usearch 2.9.2 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -38
- package/javascript/{types → dist/cjs}/usearch.d.ts +90 -115
- package/javascript/dist/cjs/usearch.js +531 -0
- package/javascript/dist/esm/usearch.d.ts +212 -0
- package/javascript/{usearch.js → dist/esm/usearch.js} +199 -153
- package/package.json +25 -23
- package/prebuilds/darwin-arm64+x64/usearch.node +0 -0
- package/prebuilds/linux-arm64/usearch.node +0 -0
- package/prebuilds/linux-x64/usearch.node +0 -0
- package/prebuilds/win32-ia32/usearch.node +0 -0
- package/prebuilds/win32-x64/usearch.node +0 -0
- package/.clang-format +0 -10
- package/.clang-tidy +0 -60
- package/.cmake-format.py +0 -19
- package/.dockerignore +0 -93
- package/.gitattributes +0 -9
- package/.github/ISSUE_TEMPLATE/bug_report.yml +0 -90
- package/.github/ISSUE_TEMPLATE/config.yml +0 -5
- package/.github/ISSUE_TEMPLATE/feature_request.yml +0 -64
- package/.github/workflows/Dockerfile.libs +0 -36
- package/.github/workflows/prerelease.yml +0 -331
- package/.github/workflows/release.yml +0 -722
- package/.github/workflows/update_version.sh +0 -15
- package/.gitmodules +0 -9
- package/.releaserc +0 -90
- package/CITATION.cff +0 -11
- package/CMakeLists.txt +0 -324
- package/CODE_OF_CONDUCT.md +0 -128
- package/CONTRIBUTING.md +0 -361
- package/Cargo.toml +0 -19
- package/Dockerfile +0 -16
- package/VERSION +0 -1
- package/assets/unum.png +0 -0
- package/assets/usearch-approaches-transparent.png +0 -0
- package/assets/usearch-approaches-white.png +0 -0
- package/assets/usearch-neighbor-types.png +0 -0
- package/binding.gyp +0 -87
- package/build.gradle +0 -204
- package/build.rs +0 -18
- package/c/CMakeLists.txt +0 -25
- package/c/README.md +0 -34
- package/c/lib.cpp +0 -358
- package/c/test.c +0 -382
- package/c/usearch.h +0 -311
- package/cmake/clipp.cmake +0 -7
- package/cmake/config.cmake.in +0 -15
- package/cmake/emscripten.cmake +0 -8
- package/cmake/jemalloc.cmake +0 -62
- package/cmake/pkg-config.pc.in +0 -4
- package/cmake/usearchConfigVersion.cmake.in +0 -20
- package/conanfile.py +0 -26
- package/cpp/CMakeLists.txt +0 -14
- package/cpp/README.md +0 -161
- package/cpp/bench.cpp +0 -618
- package/cpp/test.cpp +0 -323
- package/fp16/CMakeLists.txt +0 -176
- package/fp16/LICENSE +0 -11
- package/fp16/README.md +0 -20
- package/fp16/bench/alt-element.cc +0 -89
- package/fp16/bench/from-alt-array.cc +0 -175
- package/fp16/bench/from-ieee-array.cc +0 -399
- package/fp16/bench/ieee-element.cc +0 -272
- package/fp16/bench/to-alt-array.cc +0 -91
- package/fp16/bench/to-ieee-array.cc +0 -303
- package/fp16/cmake/DownloadGoogleBenchmark.cmake +0 -15
- package/fp16/cmake/DownloadGoogleTest.cmake +0 -15
- package/fp16/cmake/DownloadPSimd.cmake +0 -15
- package/fp16/configure.py +0 -65
- package/fp16/confu.yaml +0 -8
- package/fp16/include/fp16/__init__.py +0 -0
- package/fp16/include/fp16/avx.py +0 -50
- package/fp16/include/fp16/avx2.py +0 -53
- package/fp16/include/fp16/bitcasts.h +0 -92
- package/fp16/include/fp16/fp16.h +0 -451
- package/fp16/include/fp16/psimd.h +0 -131
- package/fp16/include/fp16.h +0 -11
- package/fp16/jni/Android.mk +0 -6
- package/fp16/jni/Application.mk +0 -4
- package/fp16/test/alt-from-fp32-value.cc +0 -498
- package/fp16/test/alt-to-fp32-bits.cc +0 -263
- package/fp16/test/alt-to-fp32-psimd.cc +0 -145
- package/fp16/test/alt-to-fp32-value.cc +0 -348
- package/fp16/test/alt-to-fp32x2-psimd.cc +0 -245
- package/fp16/test/bitcasts.cc +0 -56
- package/fp16/test/ieee-from-fp32-value.cc +0 -523
- package/fp16/test/ieee-to-fp32-bits.cc +0 -333
- package/fp16/test/ieee-to-fp32-psimd.cc +0 -312
- package/fp16/test/ieee-to-fp32-value.cc +0 -420
- package/fp16/test/ieee-to-fp32x2-psimd.cc +0 -541
- package/fp16/test/peachpy/alt-xmm-to-fp32-xmm-avx.cc +0 -96
- package/fp16/test/peachpy/alt-xmm-to-fp32-ymm-avx2.cc +0 -96
- package/fp16/test/peachpy/stubs.py +0 -38
- package/fp16/test/tables.cc +0 -530
- package/fp16/test/tables.h +0 -13
- package/fp16/third-party/THHalf.h +0 -92
- package/fp16/third-party/eigen-half.h +0 -170
- package/fp16/third-party/float16-compressor.h +0 -80
- package/fp16/third-party/half.hpp +0 -2916
- package/fp16/third-party/npy-halffloat.h +0 -328
- package/include/usearch/index.hpp +0 -3836
- package/include/usearch/index_dense.hpp +0 -2004
- package/include/usearch/index_plugins.hpp +0 -2213
- package/javascript/lib.cpp +0 -341
- package/javascript/types/usearch.d.ts.map +0 -1
- package/javascript/usearch.test.js +0 -74
- package/package-ci.json +0 -10
- package/pyproject.toml +0 -51
- package/setup.py +0 -201
- package/simsimd/.releaserc +0 -86
- package/simsimd/CONTRIBUTING.md +0 -88
- package/simsimd/Cargo.lock +0 -709
- package/simsimd/Cargo.toml +0 -44
- package/simsimd/LICENSE +0 -201
- package/simsimd/README.md +0 -317
- package/simsimd/binding.gyp +0 -36
- package/simsimd/build.rs +0 -20
- package/simsimd/include/simsimd/binary.h +0 -245
- package/simsimd/include/simsimd/complex.h +0 -174
- package/simsimd/include/simsimd/probability.h +0 -548
- package/simsimd/include/simsimd/simsimd.h +0 -465
- package/simsimd/include/simsimd/spatial.h +0 -1267
- package/simsimd/include/simsimd/types.h +0 -208
- package/simsimd/javascript/bench.js +0 -127
- package/simsimd/javascript/fallback.ts +0 -193
- package/simsimd/javascript/lib.c +0 -102
- package/simsimd/javascript/node-gyp-build.d.ts +0 -1
- package/simsimd/javascript/simsimd.ts +0 -124
- package/simsimd/javascript/test.mjs +0 -153
- package/simsimd/javascript/tsconfig-base.json +0 -25
- package/simsimd/javascript/tsconfig-cjs.json +0 -8
- package/simsimd/javascript/tsconfig-esm.json +0 -8
- package/simsimd/package-lock.json +0 -1688
- package/simsimd/package.json +0 -61
- package/simsimd/rust/benches/cosine.rs +0 -27
- package/simsimd/rust/benches/native.rs +0 -30
- package/simsimd/rust/benches/sqeuclidean.rs +0 -27
- package/simsimd/rust/lib.c +0 -45
- package/simsimd/rust/lib.rs +0 -550
- package/sqlite/README.md +0 -178
- package/sqlite/lib.cpp +0 -325
- package/stringzilla/CONTRIBUTING.md +0 -446
- package/stringzilla/Cargo.lock +0 -25
- package/stringzilla/Cargo.toml +0 -26
- package/stringzilla/LICENSE +0 -201
- package/stringzilla/README.md +0 -1422
- package/stringzilla/VERSION +0 -1
- package/stringzilla/assets/cover-strinzilla.jpeg +0 -0
- package/stringzilla/assets/meme-stringzilla-v2.jpeg +0 -0
- package/stringzilla/assets/meme-stringzilla-v3.jpeg +0 -0
- package/stringzilla/binding.gyp +0 -16
- package/stringzilla/build.rs +0 -87
- package/stringzilla/c/lib.c +0 -337
- package/stringzilla/cli/README.md +0 -65
- package/stringzilla/cli/__init__.py +0 -0
- package/stringzilla/cli/split.py +0 -105
- package/stringzilla/cli/wc.py +0 -117
- package/stringzilla/include/stringzilla/experimental.h +0 -845
- package/stringzilla/include/stringzilla/module.modulemap +0 -9
- package/stringzilla/include/stringzilla/stringzilla.h +0 -5370
- package/stringzilla/include/stringzilla/stringzilla.hpp +0 -3838
- package/stringzilla/javascript/lib.c +0 -125
- package/stringzilla/javascript/stringzilla.js +0 -22
- package/stringzilla/package-ci.json +0 -10
- package/stringzilla/package.json +0 -32
- package/stringzilla/rust/lib.rs +0 -873
- package/stringzilla/swift/StringProtocol+StringZilla.swift +0 -280
- package/stringzilla/swift/Test.swift +0 -60
- package/tsconfig.json +0 -21
- package/wasmer.toml +0 -12
- package/winlibinstaller.bat +0 -42
- /package/{simsimd/javascript/dist-package-cjs.json → javascript/dist/cjs/package.json} +0 -0
- /package/{simsimd/javascript/dist-package-esm.json → javascript/dist/esm/package.json} +0 -0
package/stringzilla/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
3.5.0
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/stringzilla/binding.gyp
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"targets": [
|
|
3
|
-
{
|
|
4
|
-
"target_name": "stringzilla",
|
|
5
|
-
"sources": ["javascript/lib.c"],
|
|
6
|
-
"include_dirs": ["include"],
|
|
7
|
-
"cflags": [
|
|
8
|
-
"-std=c99",
|
|
9
|
-
"-Wno-unknown-pragmas",
|
|
10
|
-
"-Wno-maybe-uninitialized",
|
|
11
|
-
"-Wno-cast-function-type",
|
|
12
|
-
"-Wno-unused-function",
|
|
13
|
-
],
|
|
14
|
-
}
|
|
15
|
-
]
|
|
16
|
-
}
|
package/stringzilla/build.rs
DELETED
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
use std::env;
|
|
2
|
-
|
|
3
|
-
fn main() {
|
|
4
|
-
let mut build = cc::Build::new();
|
|
5
|
-
build
|
|
6
|
-
.file("c/lib.c")
|
|
7
|
-
.include("include")
|
|
8
|
-
.warnings(false)
|
|
9
|
-
.flag_if_supported("-std=c99")
|
|
10
|
-
.flag_if_supported("-fPIC");
|
|
11
|
-
|
|
12
|
-
// Cargo will set different environment variables that we can use to properly configure the build.
|
|
13
|
-
// https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts
|
|
14
|
-
let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default();
|
|
15
|
-
let target_endian = env::var("CARGO_CFG_TARGET_ENDIAN").unwrap_or_default();
|
|
16
|
-
|
|
17
|
-
// To get the operating system we can use the TARGET environment variable.
|
|
18
|
-
// To check the list of available targets, run `rustc --print target-list`.
|
|
19
|
-
let target = env::var("TARGET").unwrap_or_default();
|
|
20
|
-
|
|
21
|
-
if target.contains("linux") {
|
|
22
|
-
build.flag_if_supported("-fdiagnostics-color=always");
|
|
23
|
-
build.flag_if_supported("-O3");
|
|
24
|
-
build.flag_if_supported("-pedantic");
|
|
25
|
-
|
|
26
|
-
// Set architecture-specific flags and macros
|
|
27
|
-
if target_arch == "x86_64" {
|
|
28
|
-
build.define("SZ_USE_X86_AVX512", "1");
|
|
29
|
-
build.define("SZ_USE_X86_AVX2", "1");
|
|
30
|
-
} else {
|
|
31
|
-
build.define("SZ_USE_X86_AVX512", "0");
|
|
32
|
-
build.define("SZ_USE_X86_AVX2", "0");
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
if target_arch == "aarch64" {
|
|
36
|
-
build.flag_if_supported("-march=armv8-a+simd");
|
|
37
|
-
build.define("SZ_USE_ARM_SVE", "1");
|
|
38
|
-
build.define("SZ_USE_ARM_NEON", "1");
|
|
39
|
-
} else {
|
|
40
|
-
build.define("SZ_USE_ARM_SVE", "0");
|
|
41
|
-
build.define("SZ_USE_ARM_NEON", "0");
|
|
42
|
-
}
|
|
43
|
-
} else if target.contains("darwin") {
|
|
44
|
-
build.flag_if_supported("-fcolor-diagnostics");
|
|
45
|
-
build.flag_if_supported("-O3");
|
|
46
|
-
build.flag_if_supported("-pedantic");
|
|
47
|
-
|
|
48
|
-
if target_arch == "x86_64" {
|
|
49
|
-
// Assuming no AVX-512 support for Darwin as per setup.py logic
|
|
50
|
-
build.define("SZ_USE_X86_AVX512", "0");
|
|
51
|
-
build.define("SZ_USE_X86_AVX2", "1");
|
|
52
|
-
} else {
|
|
53
|
-
build.define("SZ_USE_X86_AVX512", "0");
|
|
54
|
-
build.define("SZ_USE_X86_AVX2", "0");
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
if target_arch == "aarch64" {
|
|
58
|
-
build.define("SZ_USE_ARM_SVE", "0"); // Assuming no SVE support for Darwin
|
|
59
|
-
build.define("SZ_USE_ARM_NEON", "1");
|
|
60
|
-
} else {
|
|
61
|
-
build.define("SZ_USE_ARM_SVE", "0");
|
|
62
|
-
build.define("SZ_USE_ARM_NEON", "0");
|
|
63
|
-
}
|
|
64
|
-
} else if target.contains("windows") {
|
|
65
|
-
// Set architecture-specific flags and macros
|
|
66
|
-
if target_arch == "x86_64" {
|
|
67
|
-
build.define("SZ_USE_X86_AVX512", "1");
|
|
68
|
-
build.define("SZ_USE_X86_AVX2", "1");
|
|
69
|
-
} else {
|
|
70
|
-
build.define("SZ_USE_X86_AVX512", "0");
|
|
71
|
-
build.define("SZ_USE_X86_AVX2", "0");
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// Set endian-specific macro
|
|
76
|
-
if target_endian == "big" {
|
|
77
|
-
build.define("SZ_DETECT_BIG_ENDIAN", "1");
|
|
78
|
-
} else {
|
|
79
|
-
build.define("SZ_DETECT_BIG_ENDIAN", "0");
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
build.compile("stringzilla");
|
|
83
|
-
|
|
84
|
-
println!("cargo:rerun-if-changed=c/lib.c");
|
|
85
|
-
println!("cargo:rerun-if-changed=rust/lib.rs");
|
|
86
|
-
println!("cargo:rerun-if-changed=include/stringzilla/stringzilla.h");
|
|
87
|
-
}
|
package/stringzilla/c/lib.c
DELETED
|
@@ -1,337 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file lib.c
|
|
3
|
-
* @brief StringZilla C library with dynamic backed dispatch for the most appropriate implementation.
|
|
4
|
-
* @author Ash Vardanian
|
|
5
|
-
* @date January 16, 2024
|
|
6
|
-
* @copyright Copyright (c) 2024
|
|
7
|
-
*/
|
|
8
|
-
#if defined(_WIN32) || defined(__CYGWIN__)
|
|
9
|
-
#include <windows.h> // `DllMain`
|
|
10
|
-
#endif
|
|
11
|
-
|
|
12
|
-
// Overwrite `SZ_DYNAMIC_DISPATCH` before including StringZilla.
|
|
13
|
-
#ifdef SZ_DYNAMIC_DISPATCH
|
|
14
|
-
#undef SZ_DYNAMIC_DISPATCH
|
|
15
|
-
#endif
|
|
16
|
-
#define SZ_DYNAMIC_DISPATCH 1
|
|
17
|
-
#include <stringzilla/stringzilla.h>
|
|
18
|
-
|
|
19
|
-
#if SZ_AVOID_LIBC
|
|
20
|
-
// If we don't have the LibC, the `malloc` definition in `stringzilla.h` will be illformed.
|
|
21
|
-
#ifdef _MSC_VER
|
|
22
|
-
typedef sz_size_t size_t; // Reuse the type definition we've inferred from `stringzilla.h`
|
|
23
|
-
#else
|
|
24
|
-
typedef __SIZE_TYPE__ size_t; // For GCC/Clang
|
|
25
|
-
#endif
|
|
26
|
-
int rand(void) { return 0; }
|
|
27
|
-
void free(void *start) { sz_unused(start); }
|
|
28
|
-
void *malloc(size_t length) {
|
|
29
|
-
sz_unused(length);
|
|
30
|
-
return SZ_NULL;
|
|
31
|
-
}
|
|
32
|
-
#endif
|
|
33
|
-
|
|
34
|
-
SZ_DYNAMIC sz_capability_t sz_capabilities(void) {
|
|
35
|
-
|
|
36
|
-
#if SZ_USE_X86_AVX512 || SZ_USE_X86_AVX2
|
|
37
|
-
|
|
38
|
-
/// The states of 4 registers populated for a specific "cpuid" assembly call
|
|
39
|
-
union four_registers_t {
|
|
40
|
-
int array[4];
|
|
41
|
-
struct separate_t {
|
|
42
|
-
unsigned eax, ebx, ecx, edx;
|
|
43
|
-
} named;
|
|
44
|
-
} info1, info7;
|
|
45
|
-
|
|
46
|
-
#ifdef _MSC_VER
|
|
47
|
-
__cpuidex(info1.array, 1, 0);
|
|
48
|
-
__cpuidex(info7.array, 7, 0);
|
|
49
|
-
#else
|
|
50
|
-
__asm__ __volatile__("cpuid"
|
|
51
|
-
: "=a"(info1.named.eax), "=b"(info1.named.ebx), "=c"(info1.named.ecx), "=d"(info1.named.edx)
|
|
52
|
-
: "a"(1), "c"(0));
|
|
53
|
-
__asm__ __volatile__("cpuid"
|
|
54
|
-
: "=a"(info7.named.eax), "=b"(info7.named.ebx), "=c"(info7.named.ecx), "=d"(info7.named.edx)
|
|
55
|
-
: "a"(7), "c"(0));
|
|
56
|
-
#endif
|
|
57
|
-
|
|
58
|
-
// Check for AVX2 (Function ID 7, EBX register)
|
|
59
|
-
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L148
|
|
60
|
-
unsigned supports_avx2 = (info7.named.ebx & 0x00000020) != 0;
|
|
61
|
-
// Check for AVX512F (Function ID 7, EBX register)
|
|
62
|
-
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L155
|
|
63
|
-
unsigned supports_avx512f = (info7.named.ebx & 0x00010000) != 0;
|
|
64
|
-
// Check for AVX512BW (Function ID 7, EBX register)
|
|
65
|
-
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L166
|
|
66
|
-
unsigned supports_avx512bw = (info7.named.ebx & 0x40000000) != 0;
|
|
67
|
-
// Check for AVX512VL (Function ID 7, EBX register)
|
|
68
|
-
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L167C25-L167C35
|
|
69
|
-
unsigned supports_avx512vl = (info7.named.ebx & 0x80000000) != 0;
|
|
70
|
-
// Check for GFNI (Function ID 1, ECX register)
|
|
71
|
-
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L171C30-L171C40
|
|
72
|
-
unsigned supports_avx512vbmi = (info1.named.ecx & 0x00000002) != 0;
|
|
73
|
-
// Check for GFNI (Function ID 1, ECX register)
|
|
74
|
-
// https://github.com/llvm/llvm-project/blob/50598f0ff44f3a4e75706f8c53f3380fe7faa896/clang/lib/Headers/cpuid.h#L177C30-L177C40
|
|
75
|
-
unsigned supports_gfni = (info1.named.ecx & 0x00000100) != 0;
|
|
76
|
-
|
|
77
|
-
return (sz_capability_t)( //
|
|
78
|
-
(sz_cap_x86_avx2_k * supports_avx2) | //
|
|
79
|
-
(sz_cap_x86_avx512f_k * supports_avx512f) | //
|
|
80
|
-
(sz_cap_x86_avx512vl_k * supports_avx512vl) | //
|
|
81
|
-
(sz_cap_x86_avx512bw_k * supports_avx512bw) | //
|
|
82
|
-
(sz_cap_x86_avx512vbmi_k * supports_avx512vbmi) | //
|
|
83
|
-
(sz_cap_x86_gfni_k * (supports_gfni)) | //
|
|
84
|
-
(sz_cap_serial_k));
|
|
85
|
-
|
|
86
|
-
#endif // SIMSIMD_TARGET_X86
|
|
87
|
-
|
|
88
|
-
#if SZ_USE_ARM_NEON || SZ_USE_ARM_SVE
|
|
89
|
-
|
|
90
|
-
// Every 64-bit Arm CPU supports NEON
|
|
91
|
-
unsigned supports_neon = 1;
|
|
92
|
-
unsigned supports_sve = 0;
|
|
93
|
-
unsigned supports_sve2 = 0;
|
|
94
|
-
sz_unused(supports_sve);
|
|
95
|
-
sz_unused(supports_sve2);
|
|
96
|
-
|
|
97
|
-
return (sz_capability_t)( //
|
|
98
|
-
(sz_cap_arm_neon_k * supports_neon) | //
|
|
99
|
-
(sz_cap_serial_k));
|
|
100
|
-
|
|
101
|
-
#endif // SIMSIMD_TARGET_ARM
|
|
102
|
-
|
|
103
|
-
return sz_cap_serial_k;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
typedef struct sz_implementations_t {
|
|
107
|
-
sz_equal_t equal;
|
|
108
|
-
sz_order_t order;
|
|
109
|
-
|
|
110
|
-
sz_move_t copy;
|
|
111
|
-
sz_move_t move;
|
|
112
|
-
sz_fill_t fill;
|
|
113
|
-
|
|
114
|
-
sz_find_byte_t find_byte;
|
|
115
|
-
sz_find_byte_t rfind_byte;
|
|
116
|
-
sz_find_t find;
|
|
117
|
-
sz_find_t rfind;
|
|
118
|
-
sz_find_set_t find_from_set;
|
|
119
|
-
sz_find_set_t rfind_from_set;
|
|
120
|
-
|
|
121
|
-
sz_edit_distance_t edit_distance;
|
|
122
|
-
sz_alignment_score_t alignment_score;
|
|
123
|
-
sz_hashes_t hashes;
|
|
124
|
-
|
|
125
|
-
} sz_implementations_t;
|
|
126
|
-
static sz_implementations_t sz_dispatch_table;
|
|
127
|
-
|
|
128
|
-
/**
|
|
129
|
-
* @brief Initializes a global static "virtual table" of supported backends
|
|
130
|
-
* Run it just once to avoiding unnecessary `if`-s.
|
|
131
|
-
*/
|
|
132
|
-
static void sz_dispatch_table_init(void) {
|
|
133
|
-
sz_implementations_t *impl = &sz_dispatch_table;
|
|
134
|
-
sz_capability_t caps = sz_capabilities();
|
|
135
|
-
sz_unused(caps); //< Unused when compiling on pre-SIMD machines.
|
|
136
|
-
|
|
137
|
-
impl->equal = sz_equal_serial;
|
|
138
|
-
impl->order = sz_order_serial;
|
|
139
|
-
impl->copy = sz_copy_serial;
|
|
140
|
-
impl->move = sz_move_serial;
|
|
141
|
-
impl->fill = sz_fill_serial;
|
|
142
|
-
|
|
143
|
-
impl->find = sz_find_serial;
|
|
144
|
-
impl->rfind = sz_rfind_serial;
|
|
145
|
-
impl->find_byte = sz_find_byte_serial;
|
|
146
|
-
impl->rfind_byte = sz_rfind_byte_serial;
|
|
147
|
-
impl->find_from_set = sz_find_charset_serial;
|
|
148
|
-
impl->rfind_from_set = sz_rfind_charset_serial;
|
|
149
|
-
|
|
150
|
-
impl->edit_distance = sz_edit_distance_serial;
|
|
151
|
-
impl->alignment_score = sz_alignment_score_serial;
|
|
152
|
-
impl->hashes = sz_hashes_serial;
|
|
153
|
-
|
|
154
|
-
#if SZ_USE_X86_AVX2
|
|
155
|
-
if (caps & sz_cap_x86_avx2_k) {
|
|
156
|
-
impl->copy = sz_copy_avx2;
|
|
157
|
-
impl->move = sz_move_avx2;
|
|
158
|
-
impl->fill = sz_fill_avx2;
|
|
159
|
-
impl->find_byte = sz_find_byte_avx2;
|
|
160
|
-
impl->rfind_byte = sz_rfind_byte_avx2;
|
|
161
|
-
impl->find = sz_find_avx2;
|
|
162
|
-
impl->rfind = sz_rfind_avx2;
|
|
163
|
-
}
|
|
164
|
-
#endif
|
|
165
|
-
|
|
166
|
-
#if SZ_USE_X86_AVX512
|
|
167
|
-
if (caps & sz_cap_x86_avx512f_k) {
|
|
168
|
-
impl->equal = sz_equal_avx512;
|
|
169
|
-
impl->order = sz_order_avx512;
|
|
170
|
-
impl->copy = sz_copy_avx512;
|
|
171
|
-
impl->move = sz_move_avx512;
|
|
172
|
-
impl->fill = sz_fill_avx512;
|
|
173
|
-
|
|
174
|
-
impl->find = sz_find_avx512;
|
|
175
|
-
impl->rfind = sz_rfind_avx512;
|
|
176
|
-
impl->find_byte = sz_find_byte_avx512;
|
|
177
|
-
impl->rfind_byte = sz_rfind_byte_avx512;
|
|
178
|
-
|
|
179
|
-
impl->edit_distance = sz_edit_distance_avx512;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
if ((caps & sz_cap_x86_avx512f_k) && (caps & sz_cap_x86_avx512vl_k) && (caps & sz_cap_x86_gfni_k) &&
|
|
183
|
-
(caps & sz_cap_x86_avx512bw_k) && (caps & sz_cap_x86_avx512vbmi_k)) {
|
|
184
|
-
impl->find_from_set = sz_find_charset_avx512;
|
|
185
|
-
impl->rfind_from_set = sz_rfind_charset_avx512;
|
|
186
|
-
impl->alignment_score = sz_alignment_score_avx512;
|
|
187
|
-
}
|
|
188
|
-
#endif
|
|
189
|
-
|
|
190
|
-
#if SZ_USE_ARM_NEON
|
|
191
|
-
if (caps & sz_cap_arm_neon_k) {
|
|
192
|
-
impl->find = sz_find_neon;
|
|
193
|
-
impl->rfind = sz_rfind_neon;
|
|
194
|
-
impl->find_byte = sz_find_byte_neon;
|
|
195
|
-
impl->rfind_byte = sz_rfind_byte_neon;
|
|
196
|
-
impl->find_from_set = sz_find_charset_neon;
|
|
197
|
-
impl->rfind_from_set = sz_rfind_charset_neon;
|
|
198
|
-
}
|
|
199
|
-
#endif
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
#if defined(_MSC_VER)
|
|
203
|
-
BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved) {
|
|
204
|
-
switch (fdwReason) {
|
|
205
|
-
case DLL_PROCESS_ATTACH: sz_dispatch_table_init(); return TRUE;
|
|
206
|
-
case DLL_THREAD_ATTACH: return TRUE;
|
|
207
|
-
case DLL_THREAD_DETACH: return TRUE;
|
|
208
|
-
case DLL_PROCESS_DETACH: return TRUE;
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
#else
|
|
212
|
-
__attribute__((constructor)) static void sz_dispatch_table_init_on_gcc_or_clang(void) { sz_dispatch_table_init(); }
|
|
213
|
-
#endif
|
|
214
|
-
|
|
215
|
-
SZ_DYNAMIC sz_bool_t sz_equal(sz_cptr_t a, sz_cptr_t b, sz_size_t length) {
|
|
216
|
-
return sz_dispatch_table.equal(a, b, length);
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
SZ_DYNAMIC sz_ordering_t sz_order(sz_cptr_t a, sz_size_t a_length, sz_cptr_t b, sz_size_t b_length) {
|
|
220
|
-
return sz_dispatch_table.order(a, a_length, b, b_length);
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
SZ_DYNAMIC void sz_copy(sz_ptr_t target, sz_cptr_t source, sz_size_t length) {
|
|
224
|
-
sz_dispatch_table.copy(target, source, length);
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
SZ_DYNAMIC void sz_move(sz_ptr_t target, sz_cptr_t source, sz_size_t length) {
|
|
228
|
-
sz_dispatch_table.move(target, source, length);
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
SZ_DYNAMIC void sz_fill(sz_ptr_t target, sz_size_t length, sz_u8_t value) {
|
|
232
|
-
sz_dispatch_table.fill(target, length, value);
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
SZ_DYNAMIC sz_cptr_t sz_find_byte(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle) {
|
|
236
|
-
return sz_dispatch_table.find_byte(haystack, h_length, needle);
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
SZ_DYNAMIC sz_cptr_t sz_rfind_byte(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle) {
|
|
240
|
-
return sz_dispatch_table.rfind_byte(haystack, h_length, needle);
|
|
241
|
-
}
|
|
242
|
-
|
|
243
|
-
SZ_DYNAMIC sz_cptr_t sz_find(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle, sz_size_t n_length) {
|
|
244
|
-
return sz_dispatch_table.find(haystack, h_length, needle, n_length);
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
SZ_DYNAMIC sz_cptr_t sz_rfind(sz_cptr_t haystack, sz_size_t h_length, sz_cptr_t needle, sz_size_t n_length) {
|
|
248
|
-
return sz_dispatch_table.rfind(haystack, h_length, needle, n_length);
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
SZ_DYNAMIC sz_cptr_t sz_find_charset(sz_cptr_t text, sz_size_t length, sz_charset_t const *set) {
|
|
252
|
-
return sz_dispatch_table.find_from_set(text, length, set);
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
SZ_DYNAMIC sz_cptr_t sz_rfind_charset(sz_cptr_t text, sz_size_t length, sz_charset_t const *set) {
|
|
256
|
-
return sz_dispatch_table.rfind_from_set(text, length, set);
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
SZ_DYNAMIC sz_size_t sz_hamming_distance( //
|
|
260
|
-
sz_cptr_t a, sz_size_t a_length, //
|
|
261
|
-
sz_cptr_t b, sz_size_t b_length, //
|
|
262
|
-
sz_size_t bound) {
|
|
263
|
-
return sz_hamming_distance_serial(a, a_length, b, b_length, bound);
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
SZ_DYNAMIC sz_size_t sz_hamming_distance_utf8( //
|
|
267
|
-
sz_cptr_t a, sz_size_t a_length, //
|
|
268
|
-
sz_cptr_t b, sz_size_t b_length, //
|
|
269
|
-
sz_size_t bound) {
|
|
270
|
-
return sz_hamming_distance_utf8_serial(a, a_length, b, b_length, bound);
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
SZ_DYNAMIC sz_size_t sz_edit_distance( //
|
|
274
|
-
sz_cptr_t a, sz_size_t a_length, //
|
|
275
|
-
sz_cptr_t b, sz_size_t b_length, //
|
|
276
|
-
sz_size_t bound, sz_memory_allocator_t *alloc) {
|
|
277
|
-
return sz_dispatch_table.edit_distance(a, a_length, b, b_length, bound, alloc);
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
SZ_DYNAMIC sz_size_t sz_edit_distance_utf8( //
|
|
281
|
-
sz_cptr_t a, sz_size_t a_length, //
|
|
282
|
-
sz_cptr_t b, sz_size_t b_length, //
|
|
283
|
-
sz_size_t bound, sz_memory_allocator_t *alloc) {
|
|
284
|
-
return _sz_edit_distance_wagner_fisher_serial(a, a_length, b, b_length, bound, sz_true_k, alloc);
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
SZ_DYNAMIC sz_ssize_t sz_alignment_score(sz_cptr_t a, sz_size_t a_length, sz_cptr_t b, sz_size_t b_length,
|
|
288
|
-
sz_error_cost_t const *subs, sz_error_cost_t gap,
|
|
289
|
-
sz_memory_allocator_t *alloc) {
|
|
290
|
-
return sz_dispatch_table.alignment_score(a, a_length, b, b_length, subs, gap, alloc);
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
SZ_DYNAMIC void sz_hashes(sz_cptr_t text, sz_size_t length, sz_size_t window_length, sz_size_t step, //
|
|
294
|
-
sz_hash_callback_t callback, void *callback_handle) {
|
|
295
|
-
sz_dispatch_table.hashes(text, length, window_length, step, callback, callback_handle);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
SZ_DYNAMIC sz_cptr_t sz_find_char_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
|
|
299
|
-
sz_charset_t set;
|
|
300
|
-
sz_charset_init(&set);
|
|
301
|
-
for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
|
|
302
|
-
return sz_find_charset(h, h_length, &set);
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
SZ_DYNAMIC sz_cptr_t sz_find_char_not_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
|
|
306
|
-
sz_charset_t set;
|
|
307
|
-
sz_charset_init(&set);
|
|
308
|
-
for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
|
|
309
|
-
sz_charset_invert(&set);
|
|
310
|
-
return sz_find_charset(h, h_length, &set);
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
SZ_DYNAMIC sz_cptr_t sz_rfind_char_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
|
|
314
|
-
sz_charset_t set;
|
|
315
|
-
sz_charset_init(&set);
|
|
316
|
-
for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
|
|
317
|
-
return sz_rfind_charset(h, h_length, &set);
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
SZ_DYNAMIC sz_cptr_t sz_rfind_char_not_from(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n, sz_size_t n_length) {
|
|
321
|
-
sz_charset_t set;
|
|
322
|
-
sz_charset_init(&set);
|
|
323
|
-
for (; n_length; ++n, --n_length) sz_charset_add(&set, *n);
|
|
324
|
-
sz_charset_invert(&set);
|
|
325
|
-
return sz_rfind_charset(h, h_length, &set);
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
sz_u64_t _sz_random_generator(void *empty_state) {
|
|
329
|
-
sz_unused(empty_state);
|
|
330
|
-
return (sz_u64_t)rand();
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
SZ_DYNAMIC void sz_generate(sz_cptr_t alphabet, sz_size_t alphabet_size, sz_ptr_t result, sz_size_t result_length,
|
|
334
|
-
sz_random_generator_t generator, void *generator_user_data) {
|
|
335
|
-
if (!generator) generator = _sz_random_generator;
|
|
336
|
-
sz_generate_serial(alphabet, alphabet_size, result, result_length, generator, generator_user_data);
|
|
337
|
-
}
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
# SIMD-accelerated CLI utilities based on StringZilla
|
|
2
|
-
|
|
3
|
-
This section of the project is pretty much a work in progress.
|
|
4
|
-
The goal is to provide a set of command-line utilities that:
|
|
5
|
-
|
|
6
|
-
- ✅ that benefit the most from SIMD instructions,
|
|
7
|
-
- ✅ rely solely on core StringZilla functionality,
|
|
8
|
-
- ✅ works the same on Linux, MacOS, and Windows.
|
|
9
|
-
|
|
10
|
-
Other utilities are, of course, welcome to use StringZilla but may not be good candidates for this repository.
|
|
11
|
-
To install, pull the Python package from PyPi:
|
|
12
|
-
|
|
13
|
-
```bash
|
|
14
|
-
pip install stringzilla
|
|
15
|
-
```
|
|
16
|
-
|
|
17
|
-
Currently implemented:
|
|
18
|
-
|
|
19
|
-
- `sz_wc`: 3x faster `wc` word count.
|
|
20
|
-
- `sz_split`: 4x faster `splt` file splitting.
|
|
21
|
-
|
|
22
|
-
What other interfaces should be added?
|
|
23
|
-
Levenshtein distances?
|
|
24
|
-
Fuzzy search?
|
|
25
|
-
Are there common alternatives to mimic?
|
|
26
|
-
|
|
27
|
-
## `wc`: Word Count
|
|
28
|
-
|
|
29
|
-
The `wc` utility on Linux can be used to count the number of lines, words, and bytes in a file.
|
|
30
|
-
Using SIMD-accelerated character and character-set search, StringZilla, even with slow SSDs, it can be noticeably faster.
|
|
31
|
-
|
|
32
|
-
```bash
|
|
33
|
-
$ time wc enwik9.txt
|
|
34
|
-
13147025 129348346 1000000000 enwik9.txt
|
|
35
|
-
|
|
36
|
-
real 0m3.562s
|
|
37
|
-
user 0m3.470s
|
|
38
|
-
sys 0m0.092s
|
|
39
|
-
|
|
40
|
-
$ time sz_wc enwik9.txt
|
|
41
|
-
13147025 139132610 1000000000 enwik9.txt # Note: different word count, WIP
|
|
42
|
-
|
|
43
|
-
real 0m1.165s
|
|
44
|
-
user 0m1.121s
|
|
45
|
-
sys 0m0.044s
|
|
46
|
-
```
|
|
47
|
-
|
|
48
|
-
## `split`: Split File into Smaller Ones
|
|
49
|
-
|
|
50
|
-
The `split` utility on Linux can be used to split a file into smaller ones.
|
|
51
|
-
The current prototype only splits by line counts.
|
|
52
|
-
|
|
53
|
-
```bash
|
|
54
|
-
$ time split -l 100000 enwik9.txt ...
|
|
55
|
-
|
|
56
|
-
real 0m6.424s
|
|
57
|
-
user 0m0.179s
|
|
58
|
-
sys 0m0.663s
|
|
59
|
-
|
|
60
|
-
$ time sz_split -l 100000 enwik9.txt ...
|
|
61
|
-
|
|
62
|
-
real 0m1.482s
|
|
63
|
-
user 0m1.020s
|
|
64
|
-
sys 0m0.460s
|
|
65
|
-
```
|
|
File without changes
|
package/stringzilla/cli/split.py
DELETED
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
|
|
3
|
-
import argparse
|
|
4
|
-
import sys
|
|
5
|
-
import stringzilla
|
|
6
|
-
from stringzilla import File, Str
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def parse_arguments():
|
|
10
|
-
parser = argparse.ArgumentParser(
|
|
11
|
-
description="Output pieces of FILE to PREFIXaa, PREFIXab, ...; default size is 1000 lines, and default PREFIX is 'x'."
|
|
12
|
-
)
|
|
13
|
-
parser.add_argument(
|
|
14
|
-
"file", nargs="?", default="-", help='File to process, "-" for standard input'
|
|
15
|
-
)
|
|
16
|
-
parser.add_argument(
|
|
17
|
-
"prefix", nargs="?", default="X", help='Output file prefix, default is "x"'
|
|
18
|
-
)
|
|
19
|
-
parser.add_argument(
|
|
20
|
-
"-l",
|
|
21
|
-
"--lines",
|
|
22
|
-
type=int,
|
|
23
|
-
default=1000,
|
|
24
|
-
help="Number of lines per output file, default is 1000",
|
|
25
|
-
)
|
|
26
|
-
parser.add_argument(
|
|
27
|
-
"-t",
|
|
28
|
-
"--separator",
|
|
29
|
-
default="\n",
|
|
30
|
-
help="Use SEP instead of newline as the record separator; '\\0' (zero) specifies the NUL character",
|
|
31
|
-
)
|
|
32
|
-
parser.add_argument(
|
|
33
|
-
"-n",
|
|
34
|
-
"--number",
|
|
35
|
-
type=int,
|
|
36
|
-
default=None,
|
|
37
|
-
help="Generate N output files based on size of input",
|
|
38
|
-
)
|
|
39
|
-
parser.add_argument("--version", action="version", version=stringzilla.__version__)
|
|
40
|
-
return parser.parse_args()
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def split_file(file_path, lines_per_file, output_prefix, separator, number_of_files):
|
|
44
|
-
try:
|
|
45
|
-
if separator == "\\0":
|
|
46
|
-
separator = "\0"
|
|
47
|
-
if file_path == "-":
|
|
48
|
-
file_contents = Str(sys.stdin.read())
|
|
49
|
-
else:
|
|
50
|
-
file_mapped = File(file_path)
|
|
51
|
-
file_contents = Str(file_mapped)
|
|
52
|
-
|
|
53
|
-
if number_of_files is not None:
|
|
54
|
-
total_length = len(file_contents)
|
|
55
|
-
chunk_size = total_length // number_of_files
|
|
56
|
-
for file_part in range(number_of_files):
|
|
57
|
-
start = file_part * chunk_size
|
|
58
|
-
end = (
|
|
59
|
-
start + chunk_size
|
|
60
|
-
if file_part < number_of_files - 1
|
|
61
|
-
else total_length
|
|
62
|
-
)
|
|
63
|
-
current_slice = file_contents[start:end]
|
|
64
|
-
output_path = f"{output_prefix}{file_part}"
|
|
65
|
-
current_slice.write_to(output_path)
|
|
66
|
-
return
|
|
67
|
-
current_position = 0
|
|
68
|
-
file_part = 0
|
|
69
|
-
newline_position = -1
|
|
70
|
-
|
|
71
|
-
while current_position < len(file_contents):
|
|
72
|
-
for _ in range(lines_per_file):
|
|
73
|
-
newline_position = file_contents.find(separator, newline_position + 1)
|
|
74
|
-
if newline_position == -1:
|
|
75
|
-
break
|
|
76
|
-
|
|
77
|
-
if newline_position == -1 and current_position < len(file_contents):
|
|
78
|
-
newline_position = len(file_contents)
|
|
79
|
-
|
|
80
|
-
section_length = (
|
|
81
|
-
newline_position - current_position if newline_position != -1 else 0
|
|
82
|
-
)
|
|
83
|
-
|
|
84
|
-
if section_length > 0:
|
|
85
|
-
current_slice = file_contents[current_position : newline_position + 1]
|
|
86
|
-
output_path = f"{output_prefix}{file_part}"
|
|
87
|
-
current_slice.write_to(output_path)
|
|
88
|
-
|
|
89
|
-
file_part += 1
|
|
90
|
-
current_position = newline_position + 1
|
|
91
|
-
|
|
92
|
-
except FileNotFoundError:
|
|
93
|
-
print(f"No such file: {file_path}")
|
|
94
|
-
except Exception as e:
|
|
95
|
-
print(f"An error occurred: {e}")
|
|
96
|
-
print("Usage example: split.py [-l LINES] [file] [prefix]")
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
def main():
|
|
100
|
-
args = parse_arguments()
|
|
101
|
-
split_file(args.file, args.lines, args.prefix, args.separator, args.number)
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if __name__ == "__main__":
|
|
105
|
-
main()
|